SPC/snark
5
1
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix pandoc tests

Browse Source
This commit is contained in:
Alexander Nozik 2023-11-26 11:37:07 +03:00
parent aff7e88c7e
commit d5edf5e989
13 changed files with 89 additions and 108 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
build.gradle.kts
View File

@ -14,7 +14,7 @@ allprojects {
}
}
val dataforgeVersion by extra("0.6.2")
val dataforgeVersion by extra("0.7.0")
ksciencePublish {
pom("https://github.com/SciProgCentre/snark") {

2
gradle.properties
View File

@ -1,3 +1,3 @@
kotlin.code.style=official
toolsVersion=0.15.0-kotlin-1.9.20
toolsVersion=0.15.1-kotlin-1.9.21

10
snark-core/src/commonMain/kotlin/space/kscience/snark/SnarkParser.kt
View File

@ -1,20 +1,20 @@
package space.kscience.snark
import io.ktor.utils.io.core.Input
import io.ktor.utils.io.core.readBytes
import kotlinx.io.Source
import kotlinx.io.readByteArray
import space.kscience.dataforge.context.Context
import space.kscience.dataforge.io.IOReader
import space.kscience.dataforge.io.asBinary
import space.kscience.dataforge.io.readWith
import space.kscience.dataforge.meta.Meta
import space.kscience.dataforge.misc.Type
import space.kscience.dataforge.misc.DfId
import kotlin.reflect.KType
import kotlin.reflect.typeOf
/**
* A parser of binary content including priority flag and file extensions
*/
@Type(SnarkParser.TYPE)
@DfId(SnarkParser.TYPE)
public interface SnarkParser<out R> {
public val type: KType
@ -28,7 +28,7 @@ public interface SnarkParser<out R> {
public fun asReader(context: Context, meta: Meta): IOReader<R> = object : IOReader<R> {
override val type: KType get() = this@SnarkParser.type
override fun readObject(input: Input): R = parse(context, meta, input.readBytes())
override fun readFrom(source: Source): R = parse(context, meta, source.readByteArray())
}
public companion object {

5
snark-html/src/main/kotlin/space/kscience/snark/html/ImageIOReader.kt
View File

@ -1,7 +1,8 @@
package space.kscience.snark.html
import io.ktor.util.asStream
import io.ktor.utils.io.core.Input
import kotlinx.io.Source
import kotlinx.io.asInputStream
import space.kscience.dataforge.io.IOReader
import java.awt.image.BufferedImage
import javax.imageio.ImageIO
@ -11,5 +12,5 @@ import kotlin.reflect.typeOf
internal object ImageIOReader : IOReader<BufferedImage> {
override val type: KType get() = typeOf<BufferedImage>()
override fun readObject(input: Input): BufferedImage = ImageIO.read(input.asStream())
override fun readFrom(source: Source): BufferedImage = ImageIO.read(source.asInputStream())
}

4
snark-html/src/main/kotlin/space/kscience/snark/html/SiteLayout.kt
View File

@ -1,13 +1,13 @@
package space.kscience.snark.html
import space.kscience.dataforge.data.DataTreeItem
import space.kscience.dataforge.misc.Type
import space.kscience.dataforge.misc.DfId
import space.kscience.dataforge.names.NameToken
/**
* An abstraction to render singular data or a data tree.
*/
@Type(SiteLayout.TYPE)
@DfId(SiteLayout.TYPE)
public fun interface SiteLayout {
context(SiteBuilder)

3
snark-html/src/main/kotlin/space/kscience/snark/html/SnarkHtmlPlugin.kt
View File

@ -1,6 +1,7 @@
package space.kscience.snark.html
import io.ktor.utils.io.core.readBytes
import kotlinx.io.readByteArray
import space.kscience.dataforge.context.*
import space.kscience.dataforge.data.DataTree
import space.kscience.dataforge.data.node
@ -83,7 +84,7 @@ public class SnarkHtmlPlugin : AbstractPlugin() {
override fun build(context: Context, meta: Meta): SnarkHtmlPlugin = SnarkHtmlPlugin()
private val byteArrayIOReader = IOReader {
readBytes()
readByteArray()
}
internal val byteArraySnarkParser = SnarkParser(byteArrayIOReader)

7
snark-html/src/main/kotlin/space/kscience/snark/html/StaticSiteBuilder.kt
View File

@ -1,17 +1,16 @@
package space.kscience.snark.html
import io.ktor.utils.io.streams.asInput
import io.ktor.utils.io.streams.asOutput
import kotlinx.coroutines.runBlocking
import kotlinx.html.HTML
import kotlinx.html.html
import kotlinx.html.stream.createHTML
import kotlinx.io.asSink
import kotlinx.io.buffered
import space.kscience.dataforge.data.DataTree
import space.kscience.dataforge.data.DataTreeItem
import space.kscience.dataforge.data.await
import space.kscience.dataforge.data.getItem
import space.kscience.dataforge.io.Binary
import space.kscience.dataforge.io.toByteArray
import space.kscience.dataforge.io.writeBinary
import space.kscience.dataforge.meta.*
import space.kscience.dataforge.names.Name
@ -66,7 +65,7 @@ internal class StaticSiteBuilder(
if (datum.type != typeOf<Binary>()) error("Can't directly serve file of type ${item.data.type}")
val targetPath = outputPath.resolve(routeName.toWebPath())
val binary = datum.await() as Binary
targetPath.outputStream().asOutput().use{
targetPath.outputStream().asSink().buffered().use {
it.writeBinary(binary)
}
}

5
snark-html/src/main/kotlin/space/kscience/snark/html/TextProcessor.kt
View File

@ -1,15 +1,14 @@
package space.kscience.snark.html
import space.kscience.dataforge.meta.get
import space.kscience.dataforge.meta.string
import space.kscience.dataforge.misc.Type
import space.kscience.dataforge.misc.DfId
import space.kscience.dataforge.names.NameToken
import space.kscience.dataforge.names.parseAsName
/**
* An object that conducts page-based text transformation. Like using link replacement or templating.
*/
@Type(TextProcessor.TYPE)
@DfId(TextProcessor.TYPE)
public fun interface TextProcessor {
context(WebPage)
public fun process(text: String): String

43
snark-pandoc/src/jvmMain/kotlin/space/kscience/snark/pandoc/Pandoc.kt
View File

@ -2,12 +2,9 @@ package space.kscience.snark.pandoc
import org.slf4j.Logger
import org.slf4j.LoggerFactory
import java.io.BufferedReader
import java.io.IOException
import java.io.InputStreamReader
import java.nio.file.Files
import java.nio.file.Path
import java.nio.file.StandardOpenOption
import java.util.concurrent.TimeUnit
import kotlin.io.path.Path
@ -36,34 +33,24 @@ public object Pandoc {
redirectError: Path? = null,
pandocExecutablePath: Path = Path("./pandoc").toAbsolutePath(),
commandBuilder: PandocCommandBuilder.() -> Unit,
): Boolean {
) {
val path = getOrInstallPandoc(pandocExecutablePath)
try {
val commandLine = PandocCommandBuilder().apply(commandBuilder).build(path)
logger.info("Running pandoc: ${commandLine.joinToString(separator = " ")}")
val pandoc = ProcessBuilder(commandLine).apply {
if(redirectOutput!= null){
redirectOutput(redirectOutput.toFile())
}
if(redirectError !=null){
redirectError(redirectError.toFile())
}
}.start()
pandoc.waitFor(1, TimeUnit.SECONDS)
if (pandoc.exitValue() == 0) {
logger.info("Successfully execute")
return true
} else{
return false
val commandLine = PandocCommandBuilder().apply(commandBuilder).build(path)
logger.info("Running pandoc: ${commandLine.joinToString(separator = " ")}")
val pandoc = ProcessBuilder(commandLine).apply {
if (redirectOutput != null) {
redirectOutput(redirectOutput.toFile())
}
if (redirectError != null) {
redirectError(redirectError.toFile())
}
} catch (e: Exception) {
logger.error("Got problems with executing: " + e.message)
return false
}
}
}.start()
pandoc.waitFor()
if (pandoc.exitValue() != 0)
error("Non-zero process return for pandoc.")
}
}

2
snark-pandoc/src/jvmMain/kotlin/space/kscience/snark/pandoc/PandocInstaller.kt
View File

@ -31,7 +31,7 @@ internal object PandocInstaller {
private const val TIMEOUT_SECONDS = 2
private const val ATTEMPTS = 3
private enum class OSType(public val assetSuffix: String, public val propertySuffix: String) {
private enum class OSType(val assetSuffix: String, val propertySuffix: String) {
WINDOWS("windows-x86_64.zip", "windows"),
MAC_OS_AMD("x86_64-macOS.zip", "mac.os.amd"),
MAC_OS_ARM("arm64-macOS.zip", "mac.os.arm"),

114
snark-pandoc/src/jvmTest/kotlin/PandocTest.kt
View File

@ -1,96 +1,97 @@
import org.junit.jupiter.api.Assertions.*
import org.junit.jupiter.api.Assertions.assertFalse
import org.junit.jupiter.api.Assertions.assertTrue
import org.junit.jupiter.api.Test
import space.kscience.snark.pandoc.Pandoc
import java.io.BufferedReader
import java.io.FileReader
import java.io.IOException
import java.nio.file.Files
import java.nio.file.Path
import java.util.stream.Collectors
import kotlin.io.path.Path
import kotlin.io.path.createDirectories
import kotlin.io.path.div
import kotlin.io.path.exists
import kotlin.io.path.isDirectory
import kotlin.io.path.readText
import kotlin.io.path.writeBytes
import kotlin.test.assertContains
import kotlin.test.assertFails
class PandocTest {
@Test
fun when_gotPandocAndCorrectArgs_doConverting() {
try {
val res = Pandoc.execute {
addInputFile(CORRECT_MD)
outputFile(TEX_PATH_TO)
}
assertTrue(res)
assertTrue(TEX_PATH_TO.toFile().exists())
val reader = BufferedReader(FileReader(TEX_PATH_TO.toFile()))
val fileString = reader.lines().collect(Collectors.joining())
val inputFile = Files.createTempFile("snark-pandoc", "first_test.md")
inputFile.writeBytes(javaClass.getResourceAsStream("/first_test.md")!!.readAllBytes())
val outputFile = Files.createTempFile("snark-pandoc", "output1.tex")
assertTrue(fileString.contains("Some simple text"))
assertTrue(fileString.contains("\\subsection{Copy elision}"))
assertTrue(fileString.contains("return"))
Files.delete(TEX_PATH_TO)
} catch (ex: Exception) {
fail<Any>("Unexpected exception during test when_gotPandocAndCorrectArgs_doConverting()", ex)
Pandoc.execute {
addInputFile(inputFile)
outputFile(outputFile)
}
assertTrue(outputFile.exists())
val result = outputFile.readText()
assertContains(result, "Some simple text")
assertContains(result, "\\subsection{Copy elision}")
assertContains(result, "return")
}
@Test
fun when_gotPandocAndNotExistsFromFile_then_error() {
val outputFile = Files.createTempFile("snark-pandoc", "output2.tex")
val notExistsFile = Path.of("./src/test/testing_directory/non_exists_test.md")
assertFalse(notExistsFile.toFile().exists())
val res = Pandoc.execute {
addInputFile(notExistsFile)
outputFile(TEX_PATH_TO)
assertFalse(notExistsFile.exists())
assertFails {
Pandoc.execute {
addInputFile(notExistsFile)
outputFile(outputFile)
}
}
assertFalse(res)
}
@Test
fun when_gotPandocAndPassDirectory_then_error() {
assertTrue(TESTING_DIRECTORY.toFile().isDirectory)
val tempDir = Files.createTempDirectory("snark-pandoc")
assertTrue(tempDir.isDirectory())
val res = Pandoc.execute {
addInputFile(TESTING_DIRECTORY)
outputFile(TEX_PATH_TO)
val outputFile = Files.createTempFile("snark-pandoc", "output3.tex")
assertFails {
Pandoc.execute {
addInputFile(tempDir)
outputFile(outputFile)
}
}
assertFalse(res)
}
@Test
fun when_askVersionToFile_then_Ok() {
val outputFile = TESTING_DIRECTORY/"output.txt"
val outputFile = Files.createTempFile("snark-pandoc", "output4.tex")
val res = Pandoc.execute(redirectOutput = outputFile) {
getVersion()
}
val reader = BufferedReader(FileReader(outputFile.toFile()))
val fileString = reader.lines().collect(Collectors.joining())
assertTrue(fileString.contains("pandoc"))
assertTrue(fileString.contains("This is free software"))
assertTrue(res)
val fileContent = outputFile.readText()
assertContains(fileContent, "pandoc")
assertContains(fileContent, "This is free software")
}
@Test
fun when_error_then_writeToErrorStream() {
val outputFile = Files.createTempFile(TESTING_DIRECTORY, "output", ".txt")
val errorFile = Files.createTempFile(TESTING_DIRECTORY, "error", ".txt")
val inputFile = Files.createTempFile("snark-pandoc", "simple.txt")
inputFile.writeBytes(javaClass.getResourceAsStream("/simple.txt")!!.readAllBytes())
val outputFile = Files.createTempFile("snark-pandoc", "output.txt")
val errorFile = Files.createTempFile("snark-pandoc", "error.txt")
val res = Pandoc.execute(outputFile, errorFile) {
addInputFile(Path.of("./simple.txt"))
outputFile(TEX_PATH_TO)
formatFrom("txt")
assertFails {
Pandoc.execute(redirectError = errorFile) {
addInputFile(inputFile)
outputFile(outputFile)
formatFrom("txt")
}
}
val reader = BufferedReader(FileReader(errorFile.toFile()))
val fileString = reader.lines().collect(Collectors.joining())
assertFalse(res)
assertTrue(fileString.contains("21"))
Files.delete(outputFile)
Files.delete(errorFile)
assertContains(errorFile.readText(), "input format")
}
@ -101,11 +102,4 @@ class PandocTest {
// assertTrue(Pandoc.isPandocInstalled())
// }
companion object {
private val TESTING_DIRECTORY: Path = Path("./testing_directory").apply {
createDirectories()
}
private val CORRECT_MD: Path = TESTING_DIRECTORY.resolve("first_test.md")
private val TEX_PATH_TO: Path = TESTING_DIRECTORY.resolve("output1.tex")
}
}

0
snark-pandoc/testing_directory/first_test.md → snark-pandoc/src/jvmTest/resources/first_test.md
View File

0
snark-pandoc/testing_directory/simple.txt → snark-pandoc/src/jvmTest/resources/simple.txt
View File