fix pandoc tests

This commit is contained in:
Alexander Nozik 2023-11-26 11:37:07 +03:00
parent aff7e88c7e
commit d5edf5e989
13 changed files with 89 additions and 108 deletions

View File

@ -14,7 +14,7 @@ allprojects {
} }
} }
val dataforgeVersion by extra("0.6.2") val dataforgeVersion by extra("0.7.0")
ksciencePublish { ksciencePublish {
pom("https://github.com/SciProgCentre/snark") { pom("https://github.com/SciProgCentre/snark") {

View File

@ -1,3 +1,3 @@
kotlin.code.style=official kotlin.code.style=official
toolsVersion=0.15.0-kotlin-1.9.20 toolsVersion=0.15.1-kotlin-1.9.21

View File

@ -1,20 +1,20 @@
package space.kscience.snark package space.kscience.snark
import io.ktor.utils.io.core.Input import kotlinx.io.Source
import io.ktor.utils.io.core.readBytes import kotlinx.io.readByteArray
import space.kscience.dataforge.context.Context import space.kscience.dataforge.context.Context
import space.kscience.dataforge.io.IOReader import space.kscience.dataforge.io.IOReader
import space.kscience.dataforge.io.asBinary import space.kscience.dataforge.io.asBinary
import space.kscience.dataforge.io.readWith import space.kscience.dataforge.io.readWith
import space.kscience.dataforge.meta.Meta import space.kscience.dataforge.meta.Meta
import space.kscience.dataforge.misc.Type import space.kscience.dataforge.misc.DfId
import kotlin.reflect.KType import kotlin.reflect.KType
import kotlin.reflect.typeOf import kotlin.reflect.typeOf
/** /**
* A parser of binary content including priority flag and file extensions * A parser of binary content including priority flag and file extensions
*/ */
@Type(SnarkParser.TYPE) @DfId(SnarkParser.TYPE)
public interface SnarkParser<out R> { public interface SnarkParser<out R> {
public val type: KType public val type: KType
@ -28,7 +28,7 @@ public interface SnarkParser<out R> {
public fun asReader(context: Context, meta: Meta): IOReader<R> = object : IOReader<R> { public fun asReader(context: Context, meta: Meta): IOReader<R> = object : IOReader<R> {
override val type: KType get() = this@SnarkParser.type override val type: KType get() = this@SnarkParser.type
override fun readObject(input: Input): R = parse(context, meta, input.readBytes()) override fun readFrom(source: Source): R = parse(context, meta, source.readByteArray())
} }
public companion object { public companion object {

View File

@ -1,7 +1,8 @@
package space.kscience.snark.html package space.kscience.snark.html
import io.ktor.util.asStream import io.ktor.util.asStream
import io.ktor.utils.io.core.Input import kotlinx.io.Source
import kotlinx.io.asInputStream
import space.kscience.dataforge.io.IOReader import space.kscience.dataforge.io.IOReader
import java.awt.image.BufferedImage import java.awt.image.BufferedImage
import javax.imageio.ImageIO import javax.imageio.ImageIO
@ -11,5 +12,5 @@ import kotlin.reflect.typeOf
internal object ImageIOReader : IOReader<BufferedImage> { internal object ImageIOReader : IOReader<BufferedImage> {
override val type: KType get() = typeOf<BufferedImage>() override val type: KType get() = typeOf<BufferedImage>()
override fun readObject(input: Input): BufferedImage = ImageIO.read(input.asStream()) override fun readFrom(source: Source): BufferedImage = ImageIO.read(source.asInputStream())
} }

View File

@ -1,13 +1,13 @@
package space.kscience.snark.html package space.kscience.snark.html
import space.kscience.dataforge.data.DataTreeItem import space.kscience.dataforge.data.DataTreeItem
import space.kscience.dataforge.misc.Type import space.kscience.dataforge.misc.DfId
import space.kscience.dataforge.names.NameToken import space.kscience.dataforge.names.NameToken
/** /**
* An abstraction to render singular data or a data tree. * An abstraction to render singular data or a data tree.
*/ */
@Type(SiteLayout.TYPE) @DfId(SiteLayout.TYPE)
public fun interface SiteLayout { public fun interface SiteLayout {
context(SiteBuilder) context(SiteBuilder)

View File

@ -1,6 +1,7 @@
package space.kscience.snark.html package space.kscience.snark.html
import io.ktor.utils.io.core.readBytes import io.ktor.utils.io.core.readBytes
import kotlinx.io.readByteArray
import space.kscience.dataforge.context.* import space.kscience.dataforge.context.*
import space.kscience.dataforge.data.DataTree import space.kscience.dataforge.data.DataTree
import space.kscience.dataforge.data.node import space.kscience.dataforge.data.node
@ -83,7 +84,7 @@ public class SnarkHtmlPlugin : AbstractPlugin() {
override fun build(context: Context, meta: Meta): SnarkHtmlPlugin = SnarkHtmlPlugin() override fun build(context: Context, meta: Meta): SnarkHtmlPlugin = SnarkHtmlPlugin()
private val byteArrayIOReader = IOReader { private val byteArrayIOReader = IOReader {
readBytes() readByteArray()
} }
internal val byteArraySnarkParser = SnarkParser(byteArrayIOReader) internal val byteArraySnarkParser = SnarkParser(byteArrayIOReader)

View File

@ -1,17 +1,16 @@
package space.kscience.snark.html package space.kscience.snark.html
import io.ktor.utils.io.streams.asInput
import io.ktor.utils.io.streams.asOutput
import kotlinx.coroutines.runBlocking import kotlinx.coroutines.runBlocking
import kotlinx.html.HTML import kotlinx.html.HTML
import kotlinx.html.html import kotlinx.html.html
import kotlinx.html.stream.createHTML import kotlinx.html.stream.createHTML
import kotlinx.io.asSink
import kotlinx.io.buffered
import space.kscience.dataforge.data.DataTree import space.kscience.dataforge.data.DataTree
import space.kscience.dataforge.data.DataTreeItem import space.kscience.dataforge.data.DataTreeItem
import space.kscience.dataforge.data.await import space.kscience.dataforge.data.await
import space.kscience.dataforge.data.getItem import space.kscience.dataforge.data.getItem
import space.kscience.dataforge.io.Binary import space.kscience.dataforge.io.Binary
import space.kscience.dataforge.io.toByteArray
import space.kscience.dataforge.io.writeBinary import space.kscience.dataforge.io.writeBinary
import space.kscience.dataforge.meta.* import space.kscience.dataforge.meta.*
import space.kscience.dataforge.names.Name import space.kscience.dataforge.names.Name
@ -66,7 +65,7 @@ internal class StaticSiteBuilder(
if (datum.type != typeOf<Binary>()) error("Can't directly serve file of type ${item.data.type}") if (datum.type != typeOf<Binary>()) error("Can't directly serve file of type ${item.data.type}")
val targetPath = outputPath.resolve(routeName.toWebPath()) val targetPath = outputPath.resolve(routeName.toWebPath())
val binary = datum.await() as Binary val binary = datum.await() as Binary
targetPath.outputStream().asOutput().use{ targetPath.outputStream().asSink().buffered().use {
it.writeBinary(binary) it.writeBinary(binary)
} }
} }

View File

@ -1,15 +1,14 @@
package space.kscience.snark.html package space.kscience.snark.html
import space.kscience.dataforge.meta.get
import space.kscience.dataforge.meta.string import space.kscience.dataforge.meta.string
import space.kscience.dataforge.misc.Type import space.kscience.dataforge.misc.DfId
import space.kscience.dataforge.names.NameToken import space.kscience.dataforge.names.NameToken
import space.kscience.dataforge.names.parseAsName import space.kscience.dataforge.names.parseAsName
/** /**
* An object that conducts page-based text transformation. Like using link replacement or templating. * An object that conducts page-based text transformation. Like using link replacement or templating.
*/ */
@Type(TextProcessor.TYPE) @DfId(TextProcessor.TYPE)
public fun interface TextProcessor { public fun interface TextProcessor {
context(WebPage) context(WebPage)
public fun process(text: String): String public fun process(text: String): String

View File

@ -2,12 +2,9 @@ package space.kscience.snark.pandoc
import org.slf4j.Logger import org.slf4j.Logger
import org.slf4j.LoggerFactory import org.slf4j.LoggerFactory
import java.io.BufferedReader
import java.io.IOException import java.io.IOException
import java.io.InputStreamReader
import java.nio.file.Files import java.nio.file.Files
import java.nio.file.Path import java.nio.file.Path
import java.nio.file.StandardOpenOption
import java.util.concurrent.TimeUnit import java.util.concurrent.TimeUnit
import kotlin.io.path.Path import kotlin.io.path.Path
@ -36,11 +33,10 @@ public object Pandoc {
redirectError: Path? = null, redirectError: Path? = null,
pandocExecutablePath: Path = Path("./pandoc").toAbsolutePath(), pandocExecutablePath: Path = Path("./pandoc").toAbsolutePath(),
commandBuilder: PandocCommandBuilder.() -> Unit, commandBuilder: PandocCommandBuilder.() -> Unit,
): Boolean { ) {
val path = getOrInstallPandoc(pandocExecutablePath) val path = getOrInstallPandoc(pandocExecutablePath)
try {
val commandLine = PandocCommandBuilder().apply(commandBuilder).build(path) val commandLine = PandocCommandBuilder().apply(commandBuilder).build(path)
logger.info("Running pandoc: ${commandLine.joinToString(separator = " ")}") logger.info("Running pandoc: ${commandLine.joinToString(separator = " ")}")
val pandoc = ProcessBuilder(commandLine).apply { val pandoc = ProcessBuilder(commandLine).apply {
@ -52,18 +48,9 @@ public object Pandoc {
} }
}.start() }.start()
pandoc.waitFor(1, TimeUnit.SECONDS) pandoc.waitFor()
if (pandoc.exitValue() == 0) { if (pandoc.exitValue() != 0)
logger.info("Successfully execute") error("Non-zero process return for pandoc.")
return true
} else{
return false
}
} catch (e: Exception) {
logger.error("Got problems with executing: " + e.message)
return false
} }
} }
}

View File

@ -31,7 +31,7 @@ internal object PandocInstaller {
private const val TIMEOUT_SECONDS = 2 private const val TIMEOUT_SECONDS = 2
private const val ATTEMPTS = 3 private const val ATTEMPTS = 3
private enum class OSType(public val assetSuffix: String, public val propertySuffix: String) { private enum class OSType(val assetSuffix: String, val propertySuffix: String) {
WINDOWS("windows-x86_64.zip", "windows"), WINDOWS("windows-x86_64.zip", "windows"),
MAC_OS_AMD("x86_64-macOS.zip", "mac.os.amd"), MAC_OS_AMD("x86_64-macOS.zip", "mac.os.amd"),
MAC_OS_ARM("arm64-macOS.zip", "mac.os.arm"), MAC_OS_ARM("arm64-macOS.zip", "mac.os.arm"),

View File

@ -1,96 +1,97 @@
import org.junit.jupiter.api.Assertions.* import org.junit.jupiter.api.Assertions.assertFalse
import org.junit.jupiter.api.Assertions.assertTrue
import org.junit.jupiter.api.Test import org.junit.jupiter.api.Test
import space.kscience.snark.pandoc.Pandoc import space.kscience.snark.pandoc.Pandoc
import java.io.BufferedReader
import java.io.FileReader
import java.io.IOException
import java.nio.file.Files import java.nio.file.Files
import java.nio.file.Path import java.nio.file.Path
import java.util.stream.Collectors import kotlin.io.path.exists
import kotlin.io.path.Path import kotlin.io.path.isDirectory
import kotlin.io.path.createDirectories import kotlin.io.path.readText
import kotlin.io.path.div import kotlin.io.path.writeBytes
import kotlin.test.assertContains
import kotlin.test.assertFails
class PandocTest { class PandocTest {
@Test @Test
fun when_gotPandocAndCorrectArgs_doConverting() { fun when_gotPandocAndCorrectArgs_doConverting() {
try {
val res = Pandoc.execute { val inputFile = Files.createTempFile("snark-pandoc", "first_test.md")
addInputFile(CORRECT_MD) inputFile.writeBytes(javaClass.getResourceAsStream("/first_test.md")!!.readAllBytes())
outputFile(TEX_PATH_TO) val outputFile = Files.createTempFile("snark-pandoc", "output1.tex")
Pandoc.execute {
addInputFile(inputFile)
outputFile(outputFile)
} }
assertTrue(res)
assertTrue(TEX_PATH_TO.toFile().exists())
val reader = BufferedReader(FileReader(TEX_PATH_TO.toFile())) assertTrue(outputFile.exists())
val fileString = reader.lines().collect(Collectors.joining())
assertTrue(fileString.contains("Some simple text")) val result = outputFile.readText()
assertTrue(fileString.contains("\\subsection{Copy elision}"))
assertTrue(fileString.contains("return"))
Files.delete(TEX_PATH_TO) assertContains(result, "Some simple text")
} catch (ex: Exception) { assertContains(result, "\\subsection{Copy elision}")
fail<Any>("Unexpected exception during test when_gotPandocAndCorrectArgs_doConverting()", ex) assertContains(result, "return")
}
} }
@Test @Test
fun when_gotPandocAndNotExistsFromFile_then_error() { fun when_gotPandocAndNotExistsFromFile_then_error() {
val outputFile = Files.createTempFile("snark-pandoc", "output2.tex")
val notExistsFile = Path.of("./src/test/testing_directory/non_exists_test.md") val notExistsFile = Path.of("./src/test/testing_directory/non_exists_test.md")
assertFalse(notExistsFile.toFile().exists()) assertFalse(notExistsFile.exists())
val res = Pandoc.execute { assertFails {
Pandoc.execute {
addInputFile(notExistsFile) addInputFile(notExistsFile)
outputFile(TEX_PATH_TO) outputFile(outputFile)
}
} }
assertFalse(res)
} }
@Test @Test
fun when_gotPandocAndPassDirectory_then_error() { fun when_gotPandocAndPassDirectory_then_error() {
assertTrue(TESTING_DIRECTORY.toFile().isDirectory) val tempDir = Files.createTempDirectory("snark-pandoc")
assertTrue(tempDir.isDirectory())
val res = Pandoc.execute { val outputFile = Files.createTempFile("snark-pandoc", "output3.tex")
addInputFile(TESTING_DIRECTORY)
outputFile(TEX_PATH_TO) assertFails {
Pandoc.execute {
addInputFile(tempDir)
outputFile(outputFile)
}
} }
assertFalse(res)
} }
@Test @Test
fun when_askVersionToFile_then_Ok() { fun when_askVersionToFile_then_Ok() {
val outputFile = TESTING_DIRECTORY/"output.txt" val outputFile = Files.createTempFile("snark-pandoc", "output4.tex")
val res = Pandoc.execute(redirectOutput = outputFile) { val res = Pandoc.execute(redirectOutput = outputFile) {
getVersion() getVersion()
} }
val reader = BufferedReader(FileReader(outputFile.toFile())) val fileContent = outputFile.readText()
val fileString = reader.lines().collect(Collectors.joining()) assertContains(fileContent, "pandoc")
assertTrue(fileString.contains("pandoc")) assertContains(fileContent, "This is free software")
assertTrue(fileString.contains("This is free software"))
assertTrue(res)
} }
@Test @Test
fun when_error_then_writeToErrorStream() { fun when_error_then_writeToErrorStream() {
val outputFile = Files.createTempFile(TESTING_DIRECTORY, "output", ".txt") val inputFile = Files.createTempFile("snark-pandoc", "simple.txt")
val errorFile = Files.createTempFile(TESTING_DIRECTORY, "error", ".txt") inputFile.writeBytes(javaClass.getResourceAsStream("/simple.txt")!!.readAllBytes())
val outputFile = Files.createTempFile("snark-pandoc", "output.txt")
val errorFile = Files.createTempFile("snark-pandoc", "error.txt")
val res = Pandoc.execute(outputFile, errorFile) { assertFails {
addInputFile(Path.of("./simple.txt")) Pandoc.execute(redirectError = errorFile) {
outputFile(TEX_PATH_TO) addInputFile(inputFile)
outputFile(outputFile)
formatFrom("txt") formatFrom("txt")
} }
}
val reader = BufferedReader(FileReader(errorFile.toFile())) assertContains(errorFile.readText(), "input format")
val fileString = reader.lines().collect(Collectors.joining())
assertFalse(res)
assertTrue(fileString.contains("21"))
Files.delete(outputFile)
Files.delete(errorFile)
} }
@ -101,11 +102,4 @@ class PandocTest {
// assertTrue(Pandoc.isPandocInstalled()) // assertTrue(Pandoc.isPandocInstalled())
// } // }
companion object {
private val TESTING_DIRECTORY: Path = Path("./testing_directory").apply {
createDirectories()
}
private val CORRECT_MD: Path = TESTING_DIRECTORY.resolve("first_test.md")
private val TEX_PATH_TO: Path = TESTING_DIRECTORY.resolve("output1.tex")
}
} }