diff --git a/docs/README-TEMPLATE.md b/docs/README-TEMPLATE.md
index 5daf794..188dee7 100644
--- a/docs/README-TEMPLATE.md
+++ b/docs/README-TEMPLATE.md
@@ -58,5 +58,14 @@ Postprocessors are functions that transform fragments of HTML wrapped in them ac
Other details on HTML rendering could be found in [snark-html](./snark-html) module
+## Examples
+
+### Scientific document builder
+
+The idea of [the project](examples/document) is to produce a tree of scientific documents or papers. It does that in following steps:
+
+1. Read data tree from `data` directory (data path could be overridden by either ktor configuration or manually).
+2. Search all directories for a files called `document.yaml` or any other format that could be treated as value-tree (for example `document.json`). Use that file as a document descriptor that defines linear document structure.
+3.
${modules}
diff --git a/examples/README.md b/examples/README.md
new file mode 100644
index 0000000..d4e1c52
--- /dev/null
+++ b/examples/README.md
@@ -0,0 +1,4 @@
+# Module examples
+
+
+
diff --git a/examples/document/README.md b/examples/document/README.md
new file mode 100644
index 0000000..b072b95
--- /dev/null
+++ b/examples/document/README.md
@@ -0,0 +1,4 @@
+# Module document
+
+
+
diff --git a/examples/document/data/loremIpsum/document.yaml b/examples/document/data/loremIpsum/document.yaml
index 9913ed0..acf5024 100644
--- a/examples/document/data/loremIpsum/document.yaml
+++ b/examples/document/data/loremIpsum/document.yaml
@@ -2,7 +2,7 @@ route: lorem.ipsum
title: Lorem Ipsum
authors:
- name: Alexander Nozik
- affiliation: MIPT
+ affiliation: SPC
fragments:
- type: image
ref: SPC-logo.png
diff --git a/examples/document/data/simple/body.md b/examples/document/data/simple/body.md
new file mode 100644
index 0000000..dadb270
--- /dev/null
+++ b/examples/document/data/simple/body.md
@@ -0,0 +1,2 @@
+
+This is a document body for a simple document
\ No newline at end of file
diff --git a/examples/document/data/simple/document.json b/examples/document/data/simple/document.json
new file mode 100644
index 0000000..13e0b84
--- /dev/null
+++ b/examples/document/data/simple/document.json
@@ -0,0 +1,9 @@
+{
+ "title": "A simple document",
+ "fragments": [
+ {
+ "type": "data",
+ "name": "body"
+ }
+ ]
+}
diff --git a/examples/document/data/simple/footer.html b/examples/document/data/simple/footer.html
new file mode 100644
index 0000000..86c2355
--- /dev/null
+++ b/examples/document/data/simple/footer.html
@@ -0,0 +1,4 @@
+
+
+ This is HTML footer
+
\ No newline at end of file
diff --git a/examples/document/src/jvmMain/kotlin/main.kt b/examples/document/src/jvmMain/kotlin/main.kt
index b6c2bf0..5d326f1 100644
--- a/examples/document/src/jvmMain/kotlin/main.kt
+++ b/examples/document/src/jvmMain/kotlin/main.kt
@@ -1,8 +1,12 @@
package center.sciprog.snark.documents
import io.ktor.server.application.Application
+import io.ktor.server.application.call
import io.ktor.server.cio.CIO
import io.ktor.server.engine.embeddedServer
+import io.ktor.server.response.respondRedirect
+import io.ktor.server.routing.get
+import io.ktor.server.routing.routing
import kotlinx.html.ScriptCrossorigin
import kotlinx.html.link
import kotlinx.html.script
@@ -14,6 +18,7 @@ import space.kscience.snark.ktor.snarkApplication
fun Application.renderAllDocuments() = snarkApplication {
allDocuments(
headers = {
+ //add katex headers
link {
rel = "stylesheet"
href = "https://cdn.jsdelivr.net/npm/katex@0.16.10/dist/katex.min.css"
@@ -33,6 +38,7 @@ fun Application.renderAllDocuments() = snarkApplication {
crossorigin = ScriptCrossorigin.anonymous
attributes["onload"] = "renderMathInElement(document.body);"
}
+ // Auto-render latex expressions with katex
script {
unsafe {
+"""
@@ -51,6 +57,12 @@ fun Application.renderAllDocuments() = snarkApplication {
}
}
)
+
+ routing {
+ get("/"){
+ call.respondRedirect("lorem/ipsum")
+ }
+ }
}
diff --git a/snark-core/README.md b/snark-core/README.md
new file mode 100644
index 0000000..9f07ecd
--- /dev/null
+++ b/snark-core/README.md
@@ -0,0 +1,21 @@
+# Module snark-core
+
+
+
+## Usage
+
+## Artifact:
+
+The Maven coordinates of this project are `space.kscience:snark-core:0.2.0-dev-1`.
+
+**Gradle Kotlin DSL:**
+```kotlin
+repositories {
+ maven("https://repo.kotlin.link")
+ mavenCentral()
+}
+
+dependencies {
+ implementation("space.kscience:snark-core:0.2.0-dev-1")
+}
+```
diff --git a/snark-gradle-plugin/README.md b/snark-gradle-plugin/README.md
new file mode 100644
index 0000000..ded1d5c
--- /dev/null
+++ b/snark-gradle-plugin/README.md
@@ -0,0 +1,21 @@
+# Module snark-gradle-plugin
+
+
+
+## Usage
+
+## Artifact:
+
+The Maven coordinates of this project are `space.kscience:snark-gradle-plugin:0.2.0-dev-1`.
+
+**Gradle Kotlin DSL:**
+```kotlin
+repositories {
+ maven("https://repo.kotlin.link")
+ mavenCentral()
+}
+
+dependencies {
+ implementation("space.kscience:snark-gradle-plugin:0.2.0-dev-1")
+}
+```
diff --git a/snark-html/src/jvmMain/kotlin/space/kscience/snark/html/ParseAction.kt b/snark-html/src/jvmMain/kotlin/space/kscience/snark/html/ParseAction.kt
deleted file mode 100644
index 448e0e7..0000000
--- a/snark-html/src/jvmMain/kotlin/space/kscience/snark/html/ParseAction.kt
+++ /dev/null
@@ -1,68 +0,0 @@
-package space.kscience.snark.html
-
-import space.kscience.dataforge.actions.AbstractAction
-import space.kscience.dataforge.data.*
-import space.kscience.dataforge.io.Binary
-import space.kscience.dataforge.io.toByteArray
-import space.kscience.dataforge.meta.Meta
-import space.kscience.dataforge.meta.get
-import space.kscience.dataforge.misc.DFInternal
-import space.kscience.snark.SnarkReader
-import space.kscience.snark.TextProcessor
-import kotlin.coroutines.CoroutineContext
-import kotlin.coroutines.EmptyCoroutineContext
-import kotlin.reflect.KType
-import kotlin.reflect.typeOf
-
-@OptIn(DFInternal::class)
-internal fun Data.transform(
- type: KType,
- meta: Meta = this.meta,
- coroutineContext: CoroutineContext = EmptyCoroutineContext,
- block: suspend (T) -> R,
-): Data {
- val data = Data(type, meta, coroutineContext, listOf(this)) {
- block(await())
- }
- return data
-}
-
-public class ParseAction(private val snarkHtml: SnarkHtml) :
- AbstractAction(typeOf()) {
-
- private fun parseOne(data: NamedData): NamedData? = with(snarkHtml) {
- val contentType = getContentType(data.name, data.meta)
-
- val parser: SnarkReader? = snark.readers.values.filter { parser ->
- contentType in parser.inputContentTypes
- }.maxByOrNull {
- it.priority
- }
-
- //ignore data for which parser is not found
- if (parser != null) {
- val preprocessor = meta[TextProcessor.TEXT_PREPROCESSOR_KEY]?.let { snark.preprocessor(it) }
- data.transform(parser.outputType) {
- if (preprocessor == null) {
- parser.readFrom(it)
- } else {
- //TODO provide encoding
- val string = it.toByteArray().decodeToString()
- parser.readFrom(preprocessor.process(string))
- }
- }.named(data.name)
- } else {
- null
- }
- }
-
- override fun DataSink.generate(data: DataTree, meta: Meta) {
- data.forEach {
- parseOne(it)?.let { put(it) }
- }
- }
-
- override fun DataSink.update(source: DataTree, meta: Meta, namedData: NamedData) {
- parseOne(namedData)?.let { put(it) }
- }
-}
\ No newline at end of file
diff --git a/snark-html/src/jvmMain/kotlin/space/kscience/snark/html/SnarkHtml.kt b/snark-html/src/jvmMain/kotlin/space/kscience/snark/html/SnarkHtml.kt
index 950f77a..19bbc60 100644
--- a/snark-html/src/jvmMain/kotlin/space/kscience/snark/html/SnarkHtml.kt
+++ b/snark-html/src/jvmMain/kotlin/space/kscience/snark/html/SnarkHtml.kt
@@ -3,7 +3,6 @@
package space.kscience.snark.html
import io.ktor.http.ContentType
-import kotlinx.coroutines.CoroutineScope
import kotlinx.io.readByteArray
import space.kscience.dataforge.actions.Action
import space.kscience.dataforge.actions.mapping
@@ -11,11 +10,11 @@ import space.kscience.dataforge.actions.transform
import space.kscience.dataforge.context.Context
import space.kscience.dataforge.context.PluginFactory
import space.kscience.dataforge.context.PluginTag
-import space.kscience.dataforge.data.*
-import space.kscience.dataforge.io.Binary
-import space.kscience.dataforge.io.IOPlugin
-import space.kscience.dataforge.io.IOReader
-import space.kscience.dataforge.io.JsonMetaFormat
+import space.kscience.dataforge.data.DataSink
+import space.kscience.dataforge.data.DataTree
+import space.kscience.dataforge.data.filterByType
+import space.kscience.dataforge.data.putAll
+import space.kscience.dataforge.io.*
import space.kscience.dataforge.io.yaml.YamlMetaFormat
import space.kscience.dataforge.io.yaml.YamlPlugin
import space.kscience.dataforge.meta.Meta
@@ -106,7 +105,34 @@ public class SnarkHtml : WorkspacePlugin() {
}
}
- public val parseAction: Action = ParseAction(this)
+ public val parseAction: Action = Action.mapping {
+ val contentType = getContentType(name, meta)
+
+ val parser: SnarkReader? = snark.readers.values.filter { parser ->
+ contentType in parser.inputContentTypes
+ }.maxByOrNull {
+ it.priority
+ }
+
+ result(parser?.outputType ?: typeOf()) { data ->
+
+ //ignore data for which parser is not found
+ if (parser != null) {
+ val preprocessor =
+ meta[TextProcessor.TEXT_PREPROCESSOR_KEY]?.let { snark.preprocessor(it) }
+ if (preprocessor == null) {
+ parser.readFrom(data)
+ } else {
+ //TODO provide encoding
+ val string = data.toByteArray().decodeToString()
+ parser.readFrom(preprocessor.process(string))
+ }
+ } else {
+ data
+ }
+ }
+
+ }
public val layoutAction: Action = Action.mapping {
@@ -134,7 +160,7 @@ public class SnarkHtml : WorkspacePlugin() {
override fun build(context: Context, meta: Meta): SnarkHtml = SnarkHtml()
- private val byteArrayIOReader = IOReader { source->
+ private val byteArrayIOReader = IOReader { source ->
source.readByteArray()
}
@@ -143,21 +169,24 @@ public class SnarkHtml : WorkspacePlugin() {
}
}
-
-public fun SnarkHtml.readSiteData(
+/**
+ * Parse raw data tree into html primitives
+ */
+public fun SnarkHtml.parseDataTree(
binaries: DataTree,
meta: Meta = Meta.EMPTY,
-): DataTree = ObservableDataTree(context) {
+): DataTree = DataTree {
//put all binaries
putAll(binaries)
//override ones which could be parsed
putAll(binaries.transform(parseAction, meta))
}.transform(prepareHeaderAction, meta).transform(removeIndexAction, meta)
-
-public fun SnarkHtml.readSiteData(
- coroutineScope: CoroutineScope,
+/**
+ * Read the parsed data tree by providing [builder] for raw binary data tree
+ */
+public fun SnarkHtml.parseDataTree(
meta: Meta = Meta.EMPTY,
//TODO add IO plugin as a context parameter
builder: DataSink.() -> Unit,
-): DataTree = readSiteData(ObservableDataTree(coroutineScope) { builder() }, meta)
+): DataTree = parseDataTree(DataTree { builder() }, meta)
diff --git a/snark-html/src/jvmMain/resources/application.conf b/snark-html/src/jvmMain/resources/application.conf
deleted file mode 100644
index 714ea03..0000000
--- a/snark-html/src/jvmMain/resources/application.conf
+++ /dev/null
@@ -1,12 +0,0 @@
-ktor {
- application {
- modules = [ ru.mipt.spc.ApplicationKt.spcModule ]
- }
-
- deployment {
- port = 7080
- watch = ["classes", "data/"]
- }
-
- development = true
-}
\ No newline at end of file
diff --git a/snark-html/src/jvmMain/resources/logback.xml b/snark-html/src/jvmMain/resources/logback.xml
deleted file mode 100644
index a5631ba..0000000
--- a/snark-html/src/jvmMain/resources/logback.xml
+++ /dev/null
@@ -1,29 +0,0 @@
-
-
-
-
-
- %d{YYYY-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n
-
-
-
-
-
-
-
-
-
- logs/${bySecond}.txt
-
- %d{YYYY-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n
-
-
-
-
-
-
-
-
-
-
diff --git a/snark-ktor/README.md b/snark-ktor/README.md
new file mode 100644
index 0000000..eed9697
--- /dev/null
+++ b/snark-ktor/README.md
@@ -0,0 +1,21 @@
+# Module snark-ktor
+
+
+
+## Usage
+
+## Artifact:
+
+The Maven coordinates of this project are `space.kscience:snark-ktor:0.2.0-dev-1`.
+
+**Gradle Kotlin DSL:**
+```kotlin
+repositories {
+ maven("https://repo.kotlin.link")
+ mavenCentral()
+}
+
+dependencies {
+ implementation("space.kscience:snark-ktor:0.2.0-dev-1")
+}
+```
diff --git a/snark-ktor/src/jvmMain/kotlin/space/kscience/snark/ktor/extractData.kt b/snark-ktor/src/jvmMain/kotlin/space/kscience/snark/ktor/extractData.kt
index 5718f04..cf6adc6 100644
--- a/snark-ktor/src/jvmMain/kotlin/space/kscience/snark/ktor/extractData.kt
+++ b/snark-ktor/src/jvmMain/kotlin/space/kscience/snark/ktor/extractData.kt
@@ -18,6 +18,7 @@ private const val BUILD_DATE_FILE = "/buildDate"
*
* @return true if cache is valid and false if it is reset
*/
+@Deprecated("To be removed")
fun Application.prepareSnarkDataCacheDirectory(dataPath: Path): Boolean {
// Clear data directory if it is outdated
diff --git a/snark-ktor/src/jvmMain/kotlin/space/kscience/snark/ktor/snarkApplication.kt b/snark-ktor/src/jvmMain/kotlin/space/kscience/snark/ktor/snarkApplication.kt
index 31df139..17bfc81 100644
--- a/snark-ktor/src/jvmMain/kotlin/space/kscience/snark/ktor/snarkApplication.kt
+++ b/snark-ktor/src/jvmMain/kotlin/space/kscience/snark/ktor/snarkApplication.kt
@@ -15,7 +15,7 @@ import space.kscience.dataforge.workspace.FileData
import space.kscience.dataforge.workspace.directory
import space.kscience.snark.html.HtmlSite
import space.kscience.snark.html.SnarkHtml
-import space.kscience.snark.html.readSiteData
+import space.kscience.snark.html.parseDataTree
import kotlin.io.path.Path
import kotlin.io.path.exists
@@ -47,7 +47,7 @@ public fun Route.site(
error("Data directory at $dataDirectory is not resolved")
}
- val siteData = snark.readSiteData(context) {
+ val siteData = snark.parseDataTree {
directory(snark.io, Name.EMPTY, dataDirectory)
}