From 440f6e3b839cd055cb70d56a0fb3469f5f2c473d Mon Sep 17 00:00:00 2001 From: darksnake Date: Sat, 12 Oct 2024 11:51:11 +0300 Subject: [PATCH] Move parse action into snark html --- docs/README-TEMPLATE.md | 9 +++ examples/README.md | 4 ++ examples/document/README.md | 4 ++ .../document/data/loremIpsum/document.yaml | 2 +- examples/document/data/simple/body.md | 2 + examples/document/data/simple/document.json | 9 +++ examples/document/data/simple/footer.html | 4 ++ examples/document/src/jvmMain/kotlin/main.kt | 12 ++++ snark-core/README.md | 21 ++++++ snark-gradle-plugin/README.md | 21 ++++++ .../space/kscience/snark/html/ParseAction.kt | 68 ------------------- .../space/kscience/snark/html/SnarkHtml.kt | 59 ++++++++++++---- .../src/jvmMain/resources/application.conf | 12 ---- snark-html/src/jvmMain/resources/logback.xml | 29 -------- snark-ktor/README.md | 21 ++++++ .../space/kscience/snark/ktor/extractData.kt | 1 + .../kscience/snark/ktor/snarkApplication.kt | 4 +- 17 files changed, 155 insertions(+), 127 deletions(-) create mode 100644 examples/README.md create mode 100644 examples/document/README.md create mode 100644 examples/document/data/simple/body.md create mode 100644 examples/document/data/simple/document.json create mode 100644 examples/document/data/simple/footer.html create mode 100644 snark-core/README.md create mode 100644 snark-gradle-plugin/README.md delete mode 100644 snark-html/src/jvmMain/kotlin/space/kscience/snark/html/ParseAction.kt delete mode 100644 snark-html/src/jvmMain/resources/application.conf delete mode 100644 snark-html/src/jvmMain/resources/logback.xml create mode 100644 snark-ktor/README.md diff --git a/docs/README-TEMPLATE.md b/docs/README-TEMPLATE.md index 5daf794..188dee7 100644 --- a/docs/README-TEMPLATE.md +++ b/docs/README-TEMPLATE.md @@ -58,5 +58,14 @@ Postprocessors are functions that transform fragments of HTML wrapped in them ac Other details on HTML rendering could be found in [snark-html](./snark-html) module +## Examples + +### Scientific document builder + +The idea of [the project](examples/document) is to produce a tree of scientific documents or papers. It does that in following steps: + +1. Read data tree from `data` directory (data path could be overridden by either ktor configuration or manually). +2. Search all directories for a files called `document.yaml` or any other format that could be treated as value-tree (for example `document.json`). Use that file as a document descriptor that defines linear document structure. +3. ${modules} diff --git a/examples/README.md b/examples/README.md new file mode 100644 index 0000000..d4e1c52 --- /dev/null +++ b/examples/README.md @@ -0,0 +1,4 @@ +# Module examples + + + diff --git a/examples/document/README.md b/examples/document/README.md new file mode 100644 index 0000000..b072b95 --- /dev/null +++ b/examples/document/README.md @@ -0,0 +1,4 @@ +# Module document + + + diff --git a/examples/document/data/loremIpsum/document.yaml b/examples/document/data/loremIpsum/document.yaml index 9913ed0..acf5024 100644 --- a/examples/document/data/loremIpsum/document.yaml +++ b/examples/document/data/loremIpsum/document.yaml @@ -2,7 +2,7 @@ route: lorem.ipsum title: Lorem Ipsum authors: - name: Alexander Nozik - affiliation: MIPT + affiliation: SPC fragments: - type: image ref: SPC-logo.png diff --git a/examples/document/data/simple/body.md b/examples/document/data/simple/body.md new file mode 100644 index 0000000..dadb270 --- /dev/null +++ b/examples/document/data/simple/body.md @@ -0,0 +1,2 @@ + +This is a document body for a simple document \ No newline at end of file diff --git a/examples/document/data/simple/document.json b/examples/document/data/simple/document.json new file mode 100644 index 0000000..13e0b84 --- /dev/null +++ b/examples/document/data/simple/document.json @@ -0,0 +1,9 @@ +{ + "title": "A simple document", + "fragments": [ + { + "type": "data", + "name": "body" + } + ] +} diff --git a/examples/document/data/simple/footer.html b/examples/document/data/simple/footer.html new file mode 100644 index 0000000..86c2355 --- /dev/null +++ b/examples/document/data/simple/footer.html @@ -0,0 +1,4 @@ + +

+ This is HTML footer +

\ No newline at end of file diff --git a/examples/document/src/jvmMain/kotlin/main.kt b/examples/document/src/jvmMain/kotlin/main.kt index b6c2bf0..5d326f1 100644 --- a/examples/document/src/jvmMain/kotlin/main.kt +++ b/examples/document/src/jvmMain/kotlin/main.kt @@ -1,8 +1,12 @@ package center.sciprog.snark.documents import io.ktor.server.application.Application +import io.ktor.server.application.call import io.ktor.server.cio.CIO import io.ktor.server.engine.embeddedServer +import io.ktor.server.response.respondRedirect +import io.ktor.server.routing.get +import io.ktor.server.routing.routing import kotlinx.html.ScriptCrossorigin import kotlinx.html.link import kotlinx.html.script @@ -14,6 +18,7 @@ import space.kscience.snark.ktor.snarkApplication fun Application.renderAllDocuments() = snarkApplication { allDocuments( headers = { + //add katex headers link { rel = "stylesheet" href = "https://cdn.jsdelivr.net/npm/katex@0.16.10/dist/katex.min.css" @@ -33,6 +38,7 @@ fun Application.renderAllDocuments() = snarkApplication { crossorigin = ScriptCrossorigin.anonymous attributes["onload"] = "renderMathInElement(document.body);" } + // Auto-render latex expressions with katex script { unsafe { +""" @@ -51,6 +57,12 @@ fun Application.renderAllDocuments() = snarkApplication { } } ) + + routing { + get("/"){ + call.respondRedirect("lorem/ipsum") + } + } } diff --git a/snark-core/README.md b/snark-core/README.md new file mode 100644 index 0000000..9f07ecd --- /dev/null +++ b/snark-core/README.md @@ -0,0 +1,21 @@ +# Module snark-core + + + +## Usage + +## Artifact: + +The Maven coordinates of this project are `space.kscience:snark-core:0.2.0-dev-1`. + +**Gradle Kotlin DSL:** +```kotlin +repositories { + maven("https://repo.kotlin.link") + mavenCentral() +} + +dependencies { + implementation("space.kscience:snark-core:0.2.0-dev-1") +} +``` diff --git a/snark-gradle-plugin/README.md b/snark-gradle-plugin/README.md new file mode 100644 index 0000000..ded1d5c --- /dev/null +++ b/snark-gradle-plugin/README.md @@ -0,0 +1,21 @@ +# Module snark-gradle-plugin + + + +## Usage + +## Artifact: + +The Maven coordinates of this project are `space.kscience:snark-gradle-plugin:0.2.0-dev-1`. + +**Gradle Kotlin DSL:** +```kotlin +repositories { + maven("https://repo.kotlin.link") + mavenCentral() +} + +dependencies { + implementation("space.kscience:snark-gradle-plugin:0.2.0-dev-1") +} +``` diff --git a/snark-html/src/jvmMain/kotlin/space/kscience/snark/html/ParseAction.kt b/snark-html/src/jvmMain/kotlin/space/kscience/snark/html/ParseAction.kt deleted file mode 100644 index 448e0e7..0000000 --- a/snark-html/src/jvmMain/kotlin/space/kscience/snark/html/ParseAction.kt +++ /dev/null @@ -1,68 +0,0 @@ -package space.kscience.snark.html - -import space.kscience.dataforge.actions.AbstractAction -import space.kscience.dataforge.data.* -import space.kscience.dataforge.io.Binary -import space.kscience.dataforge.io.toByteArray -import space.kscience.dataforge.meta.Meta -import space.kscience.dataforge.meta.get -import space.kscience.dataforge.misc.DFInternal -import space.kscience.snark.SnarkReader -import space.kscience.snark.TextProcessor -import kotlin.coroutines.CoroutineContext -import kotlin.coroutines.EmptyCoroutineContext -import kotlin.reflect.KType -import kotlin.reflect.typeOf - -@OptIn(DFInternal::class) -internal fun Data.transform( - type: KType, - meta: Meta = this.meta, - coroutineContext: CoroutineContext = EmptyCoroutineContext, - block: suspend (T) -> R, -): Data { - val data = Data(type, meta, coroutineContext, listOf(this)) { - block(await()) - } - return data -} - -public class ParseAction(private val snarkHtml: SnarkHtml) : - AbstractAction(typeOf()) { - - private fun parseOne(data: NamedData): NamedData? = with(snarkHtml) { - val contentType = getContentType(data.name, data.meta) - - val parser: SnarkReader? = snark.readers.values.filter { parser -> - contentType in parser.inputContentTypes - }.maxByOrNull { - it.priority - } - - //ignore data for which parser is not found - if (parser != null) { - val preprocessor = meta[TextProcessor.TEXT_PREPROCESSOR_KEY]?.let { snark.preprocessor(it) } - data.transform(parser.outputType) { - if (preprocessor == null) { - parser.readFrom(it) - } else { - //TODO provide encoding - val string = it.toByteArray().decodeToString() - parser.readFrom(preprocessor.process(string)) - } - }.named(data.name) - } else { - null - } - } - - override fun DataSink.generate(data: DataTree, meta: Meta) { - data.forEach { - parseOne(it)?.let { put(it) } - } - } - - override fun DataSink.update(source: DataTree, meta: Meta, namedData: NamedData) { - parseOne(namedData)?.let { put(it) } - } -} \ No newline at end of file diff --git a/snark-html/src/jvmMain/kotlin/space/kscience/snark/html/SnarkHtml.kt b/snark-html/src/jvmMain/kotlin/space/kscience/snark/html/SnarkHtml.kt index 950f77a..19bbc60 100644 --- a/snark-html/src/jvmMain/kotlin/space/kscience/snark/html/SnarkHtml.kt +++ b/snark-html/src/jvmMain/kotlin/space/kscience/snark/html/SnarkHtml.kt @@ -3,7 +3,6 @@ package space.kscience.snark.html import io.ktor.http.ContentType -import kotlinx.coroutines.CoroutineScope import kotlinx.io.readByteArray import space.kscience.dataforge.actions.Action import space.kscience.dataforge.actions.mapping @@ -11,11 +10,11 @@ import space.kscience.dataforge.actions.transform import space.kscience.dataforge.context.Context import space.kscience.dataforge.context.PluginFactory import space.kscience.dataforge.context.PluginTag -import space.kscience.dataforge.data.* -import space.kscience.dataforge.io.Binary -import space.kscience.dataforge.io.IOPlugin -import space.kscience.dataforge.io.IOReader -import space.kscience.dataforge.io.JsonMetaFormat +import space.kscience.dataforge.data.DataSink +import space.kscience.dataforge.data.DataTree +import space.kscience.dataforge.data.filterByType +import space.kscience.dataforge.data.putAll +import space.kscience.dataforge.io.* import space.kscience.dataforge.io.yaml.YamlMetaFormat import space.kscience.dataforge.io.yaml.YamlPlugin import space.kscience.dataforge.meta.Meta @@ -106,7 +105,34 @@ public class SnarkHtml : WorkspacePlugin() { } } - public val parseAction: Action = ParseAction(this) + public val parseAction: Action = Action.mapping { + val contentType = getContentType(name, meta) + + val parser: SnarkReader? = snark.readers.values.filter { parser -> + contentType in parser.inputContentTypes + }.maxByOrNull { + it.priority + } + + result(parser?.outputType ?: typeOf()) { data -> + + //ignore data for which parser is not found + if (parser != null) { + val preprocessor = + meta[TextProcessor.TEXT_PREPROCESSOR_KEY]?.let { snark.preprocessor(it) } + if (preprocessor == null) { + parser.readFrom(data) + } else { + //TODO provide encoding + val string = data.toByteArray().decodeToString() + parser.readFrom(preprocessor.process(string)) + } + } else { + data + } + } + + } public val layoutAction: Action = Action.mapping { @@ -134,7 +160,7 @@ public class SnarkHtml : WorkspacePlugin() { override fun build(context: Context, meta: Meta): SnarkHtml = SnarkHtml() - private val byteArrayIOReader = IOReader { source-> + private val byteArrayIOReader = IOReader { source -> source.readByteArray() } @@ -143,21 +169,24 @@ public class SnarkHtml : WorkspacePlugin() { } } - -public fun SnarkHtml.readSiteData( +/** + * Parse raw data tree into html primitives + */ +public fun SnarkHtml.parseDataTree( binaries: DataTree, meta: Meta = Meta.EMPTY, -): DataTree = ObservableDataTree(context) { +): DataTree = DataTree { //put all binaries putAll(binaries) //override ones which could be parsed putAll(binaries.transform(parseAction, meta)) }.transform(prepareHeaderAction, meta).transform(removeIndexAction, meta) - -public fun SnarkHtml.readSiteData( - coroutineScope: CoroutineScope, +/** + * Read the parsed data tree by providing [builder] for raw binary data tree + */ +public fun SnarkHtml.parseDataTree( meta: Meta = Meta.EMPTY, //TODO add IO plugin as a context parameter builder: DataSink.() -> Unit, -): DataTree = readSiteData(ObservableDataTree(coroutineScope) { builder() }, meta) +): DataTree = parseDataTree(DataTree { builder() }, meta) diff --git a/snark-html/src/jvmMain/resources/application.conf b/snark-html/src/jvmMain/resources/application.conf deleted file mode 100644 index 714ea03..0000000 --- a/snark-html/src/jvmMain/resources/application.conf +++ /dev/null @@ -1,12 +0,0 @@ -ktor { - application { - modules = [ ru.mipt.spc.ApplicationKt.spcModule ] - } - - deployment { - port = 7080 - watch = ["classes", "data/"] - } - - development = true -} \ No newline at end of file diff --git a/snark-html/src/jvmMain/resources/logback.xml b/snark-html/src/jvmMain/resources/logback.xml deleted file mode 100644 index a5631ba..0000000 --- a/snark-html/src/jvmMain/resources/logback.xml +++ /dev/null @@ -1,29 +0,0 @@ - - - - - - %d{YYYY-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n - - - - - - - - - - logs/${bySecond}.txt - - %d{YYYY-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n - - - - - - - - - - diff --git a/snark-ktor/README.md b/snark-ktor/README.md new file mode 100644 index 0000000..eed9697 --- /dev/null +++ b/snark-ktor/README.md @@ -0,0 +1,21 @@ +# Module snark-ktor + + + +## Usage + +## Artifact: + +The Maven coordinates of this project are `space.kscience:snark-ktor:0.2.0-dev-1`. + +**Gradle Kotlin DSL:** +```kotlin +repositories { + maven("https://repo.kotlin.link") + mavenCentral() +} + +dependencies { + implementation("space.kscience:snark-ktor:0.2.0-dev-1") +} +``` diff --git a/snark-ktor/src/jvmMain/kotlin/space/kscience/snark/ktor/extractData.kt b/snark-ktor/src/jvmMain/kotlin/space/kscience/snark/ktor/extractData.kt index 5718f04..cf6adc6 100644 --- a/snark-ktor/src/jvmMain/kotlin/space/kscience/snark/ktor/extractData.kt +++ b/snark-ktor/src/jvmMain/kotlin/space/kscience/snark/ktor/extractData.kt @@ -18,6 +18,7 @@ private const val BUILD_DATE_FILE = "/buildDate" * * @return true if cache is valid and false if it is reset */ +@Deprecated("To be removed") fun Application.prepareSnarkDataCacheDirectory(dataPath: Path): Boolean { // Clear data directory if it is outdated diff --git a/snark-ktor/src/jvmMain/kotlin/space/kscience/snark/ktor/snarkApplication.kt b/snark-ktor/src/jvmMain/kotlin/space/kscience/snark/ktor/snarkApplication.kt index 31df139..17bfc81 100644 --- a/snark-ktor/src/jvmMain/kotlin/space/kscience/snark/ktor/snarkApplication.kt +++ b/snark-ktor/src/jvmMain/kotlin/space/kscience/snark/ktor/snarkApplication.kt @@ -15,7 +15,7 @@ import space.kscience.dataforge.workspace.FileData import space.kscience.dataforge.workspace.directory import space.kscience.snark.html.HtmlSite import space.kscience.snark.html.SnarkHtml -import space.kscience.snark.html.readSiteData +import space.kscience.snark.html.parseDataTree import kotlin.io.path.Path import kotlin.io.path.exists @@ -47,7 +47,7 @@ public fun Route.site( error("Data directory at $dataDirectory is not resolved") } - val siteData = snark.readSiteData(context) { + val siteData = snark.parseDataTree { directory(snark.io, Name.EMPTY, dataDirectory) }