Add pandoc-plugin and conversion to latex

Use common s3 api and '/'-separated keys to simulate directories

.dockerignore

@@ -0,0 +1,24 @@
+# .gitignore contents
+
+.gradle/
+build/
+.idea/
+/logs/
+
+!gradle/wrapper/gradle-wrapper.jar
+
+kotlin-js-store
+*.iml
+
+# addition to .gitignore
+
+README.md
+LICENSE
+gradlew.bat
+.dockerignore
+.gitignore
+Dockerfile
+
+tmp
+rundata
+credentials.json

.gitignore

@@ -2,5 +2,10 @@
 build/
 .idea/
 /logs/
+rundata/
 
 !gradle/wrapper/gradle-wrapper.jar
+
+kotlin-js-store
+*.iml
+*.json

Dockerfile

@@ -0,0 +1,25 @@
+FROM ubuntu:latest
+
+RUN apt-get update
+RUN apt-get install -y curl zip unzip
+
+ARG JAVA_VERSION="17.0.7-zulu"
+
+WORKDIR /app
+
+COPY ./snark-main/ci ./snark-main/ci
+
+RUN ./snark-main/ci/install_sdk.sh
+RUN ./snark-main/ci/install_java.sh "$JAVA_VERSION"
+
+COPY . .
+
+RUN ./requirements.sh
+
+RUN bash -c "source ~/.sdkman/bin/sdkman-init.sh && ./gradlew clean build"
+
+EXPOSE 8080
+
+RUN mkdir -p ~/.aws/ && ln -s /run/secrets/credentials.json ~/.aws/credentials.json
+
+CMD bash -c "source ~/.sdkman/bin/sdkman-init.sh && ./gradlew :snark-main:run_server"

build.gradle.kts

@@ -1,5 +1,4 @@
-import space.kscience.gradle.useApache2Licence
-import space.kscience.gradle.useSPCTeam
+import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
 
 plugins {
     id("space.kscience.gradle.project")
@@ -7,21 +6,21 @@ plugins {
 
 allprojects {
     group = "space.kscience"
-    version = "0.2.0-dev-1"
+    version = "0.1.0-dev-1"
 
-    repositories {
-        mavenCentral()
-        mavenLocal()
+    if (name != "snark-gradle-plugin") {
+        tasks.withType<KotlinCompile> {
+            kotlinOptions {
+                freeCompilerArgs = freeCompilerArgs + "-Xcontext-receivers"
+            }
+        }
     }
 }
 
-val dataforgeVersion by extra("0.8.0")
+val dataforgeVersion by extra("0.6.0-dev-15")
 
 ksciencePublish {
-    pom("https://github.com/SciProgCentre/snark") {
-        useApache2Licence()
-        useSPCTeam()
-    }
-    repository("spc","https://maven.sciprog.center/kscience")
+    github("SciProgCentre", "snark")
+    space("https://maven.pkg.jetbrains.space/mipt-npm/p/sci/maven")
 //    sonatype()
 }

docker-compose.yml

@@ -0,0 +1,19 @@
+version: '3'
+
+services:
+  snark:
+    build: .
+    ports:
+      - "8081:8080"
+    expose:
+      - 8081
+    volumes:
+      - storage:/app/rundata
+    secrets:
+      - credentials.json
+volumes:
+  storage:
+
+secrets:
+  credentials.json:
+    file: ./credentials.json

examples/document/build.gradle.kts

@@ -1,33 +0,0 @@
-plugins {
-    id("space.kscience.gradle.mpp")
-    application
-}
-
-application {
-    mainClass.set("Mainkt")
-
-    val isDevelopment: Boolean = project.ext.has("development")
-    applicationDefaultJvmArgs = listOf("-Dio.ktor.development=$isDevelopment", "-Xmx200M")
-}
-
-val snarkVersion: String by extra
-val ktorVersion = space.kscience.gradle.KScienceVersions.ktorVersion
-
-kscience {
-    jvm()
-    useContextReceivers()
-
-    jvmMain {
-        implementation(projects.snarkKtor)
-        implementation("io.ktor:ktor-server-cio:$ktorVersion")
-        implementation(spclibs.logback.classic)
-    }
-
-    jvmTest{
-        implementation("io.ktor:ktor-server-tests:$ktorVersion")
-    }
-}
-
-kotlin {
-    explicitApi = org.jetbrains.kotlin.gradle.dsl.ExplicitApiMode.Disabled
-}

examples/document/data/loremIpsum/chapter1.md

@@ -1,29 +0,0 @@
----
-contentType: markdown
----
-
-Document name: ${documentName}
-
-${documentMeta.metaValue}
-
-${documentMeta.get('metaValue')}
-
-## Chapter ${section(1)}
-
-Curabitur hendrerit hendrerit rutrum. Nullam elementum libero a nisi viverra aliquet. Sed ut urna a sem bibendum dictum. Cras non elit sit amet ex ultrices iaculis. Fusce lobortis lacinia fermentum. Fusce in metus id massa mollis consequat. Quisque non dolor quis orci gravida vulputate. Vivamus sed pellentesque orci. Sed aliquet malesuada rhoncus. Mauris id aliquet lorem.
-
-### Section ${section(2)}
-
-Maecenas at iaculis ipsum. Praesent maximus tristique magna eu faucibus. In tincidunt elementum pharetra. Nam scelerisque eros mattis, suscipit odio sit amet, efficitur mi. Etiam eleifend pulvinar erat a aliquet. Cras pellentesque tincidunt mi eget scelerisque. Proin eget ipsum a velit lobortis commodo. Nulla facilisi. Donec id pretium leo. Ut nec tortor sapien. Praesent vehicula dolor ut laoreet commodo. Pellentesque convallis, sapien et placerat luctus, tortor magna sodales sem, non tristique eros sem vel ipsum. Nulla vulputate accumsan nulla. Duis tempor, mi nec pharetra suscipit, sem odio sagittis mi, ut dignissim odio erat a dolor.
-
-### Section ${section(2)}
-
-In a quam nec turpis venenatis vehicula at ut lorem. Vestibulum tincidunt at velit laoreet sodales. Fusce fermentum enim sed lacinia fringilla. Nam et augue vitae felis sagittis consectetur in eget mauris. Fusce eget auctor turpis. Quisque at tristique nibh, id fringilla arcu. Cras sed finibus sapien.
-
-### Section ${section(2)}
-
-Praesent ullamcorper volutpat facilisis. Quisque posuere nisi sed nisl tempor, et sollicitudin justo imperdiet. Donec interdum auctor dui, quis dapibus lorem ullamcorper et. Sed aliquam, augue a tempus viverra, felis nulla convallis magna, a lacinia orci nisi id mauris. Pellentesque habitant morbi tristique senectus et netus et malesuada fames ac turpis egestas. Ut congue cursus quam, in vehicula nibh laoreet vel. Vivamus tincidunt sit amet dui quis mollis. Aliquam sed bibendum erat. Vivamus eget ante sed sapien volutpat luctus.
-
-### Section ${section(2)}
-
-Donec auctor quis libero eu cursus. Ut molestie varius massa, quis eleifend purus. Quisque elementum, magna id sollicitudin ullamcorper, arcu orci ullamcorper felis, ac tempus massa nisi vitae elit. In mollis porttitor orci. Integer gravida massa ut massa imperdiet, quis maximus libero varius. Phasellus vitae quam commodo, aliquet quam sed, euismod velit. Cras nibh ante, sodales non nulla sed, fringilla posuere orci.

examples/document/data/loremIpsum/chapter2.md

@@ -1,22 +0,0 @@
----
-contentType: markdown
----
-
-## Chapter ${section(1)}
-
-Fusce at tristique ex. Proin vehicula venenatis mattis. Fusce at congue sapien, sed interdum lacus. Vivamus scelerisque ligula pretium nisl accumsan, molestie commodo sem condimentum. Nam ullamcorper leo quis sapien commodo, feugiat pellentesque purus rhoncus. Cras feugiat, lorem sit amet sodales aliquet, ante ipsum aliquam felis, ac rhoncus risus felis non enim. Suspendisse bibendum ornare efficitur. Nam tortor dolor, imperdiet nec orci et, pellentesque elementum sem. Integer sapien urna, rhoncus et felis et, fringilla euismod elit. Quisque tellus quam, tincidunt sed velit at, aliquam mollis leo. Integer pellentesque leo in libero pretium pharetra vitae sed ipsum. Mauris auctor venenatis pharetra. Maecenas tincidunt nulla ullamcorper, faucibus ante id, bibendum turpis. In lacus risus, pretium vel accumsan non, rhoncus nec augue. Suspendisse potenti.
-
-Orci varius natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus. Ut eu aliquet leo. Duis luctus viverra ex, at tincidunt diam fringilla at. Nunc rhoncus lorem arcu. Donec et nisi erat. Interdum et malesuada fames ac ante ipsum primis in faucibus. Sed vulputate lobortis velit. Etiam nisi sem, pellentesque sit amet molestie vel, porta vel enim. Cras sed diam sit amet nibh laoreet blandit non eget dui. Suspendisse tellus metus, pretium tempor euismod non, rhoncus eu enim. Etiam pharetra diam in quam auctor viverra. Nunc rhoncus libero quis dolor elementum accumsan. Mauris sed lectus fermentum, suscipit justo ac, faucibus tortor. Cras at volutpat enim, dapibus fringilla justo. Nulla vel erat quis neque congue laoreet.
-
-Integer et metus metus. Donec fringilla nec sem sit amet bibendum. Sed ornare lobortis velit eu gravida. Maecenas tincidunt ante et elit auctor convallis. Donec vestibulum augue et nisl fringilla aliquet venenatis ut diam. Nulla vitae leo est. Donec in magna blandit, dignissim ligula ultrices, cursus tortor. Praesent fermentum lorem placerat, venenatis ipsum eget, molestie ante. Duis non congue mi. Pellentesque non sem nibh. Donec feugiat lorem metus, sit amet dapibus augue congue vitae. Donec leo neque, sollicitudin et dignissim ac, semper vel mauris. Nunc fermentum egestas massa id varius. Aliquam interdum posuere mi in scelerisque. Aenean interdum consequat ultrices. Donec elementum tristique blandit.
-
-Cras finibus vel leo id mattis. Nulla tellus augue, bibendum in ipsum vitae, aliquet convallis nisl. Sed auctor urna sit amet ante pulvinar, sit amet venenatis nibh porta. Cras vitae ultrices nisi. Vestibulum eu sapien eu nulla rhoncus porttitor ut vitae odio. Curabitur scelerisque hendrerit elit vitae laoreet. Etiam eget accumsan nibh, non vehicula ex.
-
-Quisque ut ultricies nisi, eget vehicula ipsum. Quisque tortor mauris, sagittis vitae consectetur in, fermentum quis dui. Maecenas nec risus eu ipsum eleifend ornare. Nam tempus interdum mi, eget tincidunt enim interdum et. Nam a ultricies libero. Cras vehicula, quam a egestas semper, ipsum est cursus nunc, ac mollis est velit at enim. Sed nec nibh ut leo fermentum interdum. Donec aliquam elementum metus, non fermentum eros bibendum eu. Suspendisse ut odio vel dolor blandit condimentum. Vivamus malesuada accumsan magna, a suscipit tellus vehicula ut. Duis et orci arcu. Vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia curae;
-
-```snark
-type: image
-path: myImage.png
-caption: Whatever
-index: ${label(image)}
-```

examples/document/data/loremIpsum/chapter3.md

@@ -1,15 +0,0 @@
----
-contentType: markdown
----
-
-## Chapter ${section(1)}
-
-Vestibulum mauris urna, sagittis in nibh placerat, vehicula suscipit leo. Maecenas lacinia varius tellus vel laoreet. Vestibulum sollicitudin nibh et nibh ullamcorper, at pretium orci molestie. Donec vulputate, nibh tempus maximus pretium, urna arcu consequat metus, vel ullamcorper est lectus id purus. Aliquam rutrum eu arcu nec convallis. Cras nec nisl ut massa tempus fringilla. Lorem ipsum dolor sit amet, consectetur adipiscing elit. Aliquam non augue vel nisi commodo blandit a a augue. Sed aliquet semper ipsum nec maximus. Aliquam elementum tellus eu lectus tempus eleifend. Donec sapien nisi, maximus vitae ante nec, dictum vestibulum justo. Nam at ex est. Pellentesque nisi lacus, congue non felis posuere, pulvinar cursus justo. Fusce tincidunt dui vel pharetra fringilla.
-
-Pellentesque habitant morbi tristique senectus et netus et malesuada fames ac turpis egestas. In accumsan lobortis ante, vitae convallis eros lobortis vitae. Praesent ac mi id dui tempor pellentesque ac quis ligula. Ut tristique elit non ex euismod, ac hendrerit ante porta. Sed venenatis porttitor neque quis molestie. Aenean consectetur vehicula nisl quis tincidunt. Phasellus a eros tristique, rhoncus dui non, commodo ligula. Maecenas gravida, felis nec vulputate ornare, dolor mauris aliquet arcu, nec hendrerit lacus risus vitae arcu. Nullam dignissim nulla vulputate nisi pulvinar, non volutpat libero tristique. Nullam faucibus justo et elit condimentum sagittis. Ut imperdiet molestie purus.
-
-Fusce dignissim laoreet lectus ac sollicitudin. Cras porta dapibus orci vel vehicula. Integer dapibus vitae risus non luctus. Vestibulum eu orci nec tortor rutrum convallis tincidunt non lectus. Mauris aliquam, nunc id tincidunt cursus, turpis dui aliquam arcu, quis consectetur sapien nunc eget turpis. Donec leo metus, dictum vitae egestas ut, tristique ut sapien. Mauris tempus leo sit amet justo dignissim, a ullamcorper orci luctus.
-
-Nunc eget imperdiet lorem. Integer vitae vestibulum leo, vel tincidunt metus. Integer id lorem sodales, dignissim justo a, lacinia nisl. Maecenas vel nisi aliquet, vulputate tortor quis, pretium libero. Morbi convallis ex non vulputate scelerisque. Phasellus pharetra lacus in justo volutpat, vitae hendrerit leo porttitor. Suspendisse bibendum, dui eu ullamcorper mollis, magna arcu semper ipsum, sit amet ullamcorper lacus elit non ipsum. Nullam pulvinar justo a odio fringilla, sit amet pharetra odio consectetur. Integer tempus, lorem non tristique placerat, ipsum felis eleifend eros, fringilla feugiat velit felis egestas eros. Duis purus nibh, accumsan vitae purus eget, porttitor cursus quam. Aliquam eu eros in odio fringilla rhoncus nec sed nulla. Fusce ex ex, iaculis eu porttitor a, aliquam at ex. Pellentesque enim erat, egestas sit amet sodales quis, molestie ac ante. Phasellus fringilla tellus velit, eu elementum urna convallis ut. Nam condimentum gravida erat id efficitur.
-
-Nulla cursus aliquet justo malesuada dictum. Donec euismod rhoncus ex vel tempus. Praesent ac sodales massa, eu fringilla est. Phasellus ac sapien posuere, dapibus lacus eget, commodo sapien. Aliquam justo risus, posuere vitae erat non, volutpat eleifend dui. Morbi vel lobortis mauris. Duis non tortor vitae neque maximus porttitor. Aliquam ac placerat dolor. Duis id sollicitudin felis. Suspendisse ligula ex, convallis id velit quis, semper laoreet sem.

examples/document/data/loremIpsum/document.yaml

@@ -1,22 +0,0 @@
-route: lorem.ipsum
-title: Lorem Ipsum
-authors:
-  - name: Alexander Nozik
-    affiliation: MIPT
-fragments:
-  - type: image
-    ref: SPC-logo.png
-    meta:
-      caption: SPC logo
-  - type: data
-    name: chapter1
-  - type: data
-    name: chapter2
-  - type: image
-    ref: SPC-logo.png
-    meta:
-      caption: Another SPC logo
-  - type: data
-    name: chapter3
-documentMeta:
-  metaValue: Hello world!

examples/document/src/jvmMain/kotlin/main.kt

@@ -1,41 +0,0 @@
-import io.ktor.server.application.Application
-import io.ktor.server.cio.CIO
-import io.ktor.server.engine.embeddedServer
-import kotlinx.html.ScriptCrossorigin
-import kotlinx.html.link
-import kotlinx.html.script
-import space.kscience.snark.html.document.allDocuments
-
-@Suppress("unused")
-fun Application.renderAllDocuments() = snarkApplication {
-    allDocuments(
-        headers = {
-            link {
-                rel = "stylesheet"
-                href = "https://cdn.jsdelivr.net/npm/katex@0.16.10/dist/katex.min.css"
-                attributes["integrity"] = "sha384-wcIxkf4k558AjM3Yz3BBFQUbk/zgIYC2R0QpeeYb+TwlBVMrlgLqwRjRtGZiK7ww"
-                attributes["crossorigin"] = "anonymous"
-            }
-            script {
-                defer = true
-                src = "https://cdn.jsdelivr.net/npm/katex@0.16.10/dist/katex.min.js"
-                integrity = "sha384-hIoBPJpTUs74ddyc4bFZSM1TVlQDA60VBbJS0oA934VSz82sBx1X7kSx2ATBDIyd"
-                crossorigin = ScriptCrossorigin.anonymous
-            }
-            script {
-                defer = true
-                src = "https://cdn.jsdelivr.net/npm/katex@0.16.10/dist/contrib/auto-render.min.js"
-                integrity = "sha384-43gviWU0YVjaDtb/GhzOouOXtZMP/7XUzwPTstBeZFe/+rCMvRwr4yROQP43s0Xk"
-                crossorigin = ScriptCrossorigin.anonymous
-                attributes["onload"] = "renderMathInElement(document.body);"
-            }
-        }
-    )
-}
-
-
-fun main() {
-    embeddedServer(CIO) {
-        renderAllDocuments()
-    }.start(true)
-}

examples/document/src/jvmMain/kotlin/snarkApplication.kt

@@ -1,49 +0,0 @@
-import io.ktor.server.application.Application
-import io.ktor.server.application.log
-import io.ktor.server.http.content.staticResources
-import io.ktor.server.routing.routing
-import space.kscience.dataforge.context.Context
-import space.kscience.dataforge.context.ContextBuilder
-import space.kscience.dataforge.context.request
-import space.kscience.dataforge.data.forEach
-import space.kscience.dataforge.names.Name
-import space.kscience.dataforge.workspace.FileData
-import space.kscience.dataforge.workspace.directory
-import space.kscience.snark.html.HtmlSite
-import space.kscience.snark.html.SnarkHtml
-import space.kscience.snark.html.readSiteData
-import space.kscience.snark.ktor.site
-import kotlin.io.path.Path
-import kotlin.io.path.exists
-
-
-fun Application.snarkApplication(contextBuilder: ContextBuilder.() -> Unit = {}, site: HtmlSite) {
-
-    val context = Context {
-        plugin(SnarkHtml)
-        contextBuilder()
-    }
-
-    val snark = context.request(SnarkHtml)
-
-    val dataDirectoryString = environment.config.propertyOrNull("snark.dataDirectory")?.getString() ?: "data"
-
-    val dataDirectory = Path(dataDirectoryString)
-
-    if (!dataDirectory.exists()) {
-        error("Data directory at $dataDirectory is not resolved")
-    }
-
-    val siteData = snark.readSiteData(context) {
-        directory(snark.io, Name.EMPTY, dataDirectory)
-    }
-
-    siteData.forEach { namedData ->
-        log.debug("Loading data {} from {}", namedData.name, namedData.meta[FileData.FILE_PATH_KEY])
-    }
-
-    routing {
-        staticResources("/css","css")
-        site(context, siteData, content = site)
-    }
-}

gradle.properties

@@ -1,3 +1,3 @@
 kotlin.code.style=official
 
-toolsVersion=0.15.2-kotlin-1.9.22
+toolsVersion=0.13.3-kotlin-1.7.20

gradle/wrapper/gradle-wrapper.properties

@@ -1,5 +1,5 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-8.6-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-7.6-bin.zip
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists

requirements.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+set -e
+
+apt-get update
+apt-get install -y sudo
+
+for dir in ./*/
+do
+    if [[ $dir == *'snark'* ]]
+    then
+        cd "$dir"
+
+        if [[ $(find -type d -name "ci") ]]
+        then
+
+            cd ci
+            if [[ -f "requirements.sh" ]]; then
+              echo "executing sub"
+              bash -c "./requirements.sh"
+            fi
+
+            cd ..
+        fi
+
+        cd ..
+    fi
+done
+

settings.gradle.kts

@@ -1,6 +1,7 @@
 rootProject.name = "snark"
 
 enableFeaturePreview("TYPESAFE_PROJECT_ACCESSORS")
+enableFeaturePreview("VERSION_CATALOGS")
 
 pluginManagement {
 
@@ -30,7 +31,7 @@ dependencyResolutionManagement {
     }
 
     versionCatalogs {
-        create("spclibs") {
+        create("npmlibs") {
             from("space.kscience:version-catalog:$toolsVersion")
         }
     }
@@ -41,6 +42,8 @@ include(
     ":snark-core",
     ":snark-html",
     ":snark-ktor",
-    ":snark-pandoc",
-    ":examples:document"
+    ":snark-storage-driver",
+    ":snark-document-builder",
+    ":snark-main",
+    ":snark-pandoc-plugin",
 )

snark-core/build.gradle.kts

@@ -5,11 +5,12 @@ plugins{
 
 val dataforgeVersion: String by rootProject.extra
 
-kscience{
-    jvm()
-    js()
-    useContextReceivers()
+kotlin{
+    sourceSets{
+        commonMain{
             dependencies{
                 api("space.kscience:dataforge-workspace:$dataforgeVersion")
             }
+        }
+    }
 }

snark-core/src/commonMain/kotlin/space/kscience/snark/DataTreeWithDefault.kt

@@ -1,34 +0,0 @@
-package space.kscience.snark
-
-import space.kscience.dataforge.data.Data
-import space.kscience.dataforge.data.DataTree
-import space.kscience.dataforge.data.GenericDataTree
-import space.kscience.dataforge.names.NameToken
-import kotlin.reflect.KType
-
-public class DataTreeWithDefault<T>(public val tree: DataTree<T>, public val default: DataTree<T>) :
-    DataTree<T> {
-    override val dataType: KType get() = tree.dataType
-
-    override val self: DataTreeWithDefault<T> get() = this
-
-    override val data: Data<T>? get() = tree.data ?: default.data
-
-    private fun mergeItems(
-        treeItems: Map<NameToken, DataTree<T>>,
-        defaultItems: Map<NameToken, DataTree<T>>,
-    ): Map<NameToken, DataTree<T>> {
-        val mergedKeys = treeItems.keys + defaultItems.keys
-        return mergedKeys.associateWith {
-            val treeItem = treeItems[it]
-            val defaultItem = defaultItems[it]
-            when {
-                treeItem == null -> defaultItem!!
-                defaultItem == null -> treeItem
-                else -> DataTreeWithDefault(treeItem, defaultItem)
-            }
-        }
-    }
-
-    override val items: Map<NameToken, GenericDataTree<T, *>> get() = mergeItems(tree.items, default.items)
-}

snark-core/src/commonMain/kotlin/space/kscience/snark/ReWrapAction.kt

@@ -1,46 +0,0 @@
-package space.kscience.snark
-
-import space.kscience.dataforge.actions.AbstractAction
-import space.kscience.dataforge.data.*
-import space.kscience.dataforge.meta.Meta
-import space.kscience.dataforge.meta.MutableMeta
-import space.kscience.dataforge.meta.copy
-import space.kscience.dataforge.names.*
-import kotlin.reflect.KType
-import kotlin.reflect.typeOf
-
-/**
- * An action to change header (name and meta) without changing the data itself or its computation state
- */
-public class ReWrapAction<R : Any>(
-    type: KType,
-    private val newMeta: MutableMeta.(name: Name) -> Unit = {},
-    private val newName: (name: Name, meta: Meta?, type: KType) -> Name,
-) : AbstractAction<R, R>(type) {
-    override fun DataSink<R>.generate(data: DataTree<R>, meta: Meta) {
-        data.forEach { namedData ->
-            put(
-                newName(namedData.name, namedData.meta, namedData.type),
-                namedData.data.withMeta(namedData.meta.copy { newMeta(namedData.name) })
-            )
-        }
-    }
-
-    override fun DataSink<R>.update(source: DataTree<R>, meta: Meta, namedData: NamedData<R>) {
-        put(
-            newName(namedData.name, namedData.meta, namedData.type),
-            namedData.withMeta(namedData.meta.copy { newMeta(namedData.name) })
-        )
-    }
-
-    public companion object {
-
-
-    }
-}
-
-public inline fun <reified R : Any> ReWrapAction(
-    noinline newMeta: MutableMeta.(name: Name) -> Unit = {},
-    noinline newName: (Name, Meta?, type: KType) -> Name,
-): ReWrapAction<R> = ReWrapAction(typeOf<R>(), newMeta, newName)
-

snark-core/src/commonMain/kotlin/space/kscience/snark/Snark.kt

@@ -1,56 +0,0 @@
-package space.kscience.snark
-
-import kotlinx.io.readByteArray
-import space.kscience.dataforge.context.Context
-import space.kscience.dataforge.context.PluginFactory
-import space.kscience.dataforge.context.PluginTag
-import space.kscience.dataforge.context.gather
-import space.kscience.dataforge.io.IOPlugin
-import space.kscience.dataforge.io.IOReader
-import space.kscience.dataforge.meta.Meta
-import space.kscience.dataforge.meta.get
-import space.kscience.dataforge.meta.string
-import space.kscience.dataforge.names.Name
-import space.kscience.dataforge.names.parseAsName
-import space.kscience.dataforge.workspace.WorkspacePlugin
-
-/**
- * Represents a Snark workspace plugin.
- */
-public class Snark : WorkspacePlugin() {
-    public val io: IOPlugin by require(IOPlugin)
-    override val tag: PluginTag get() = Companion.tag
-
-    public val readers: Map<Name, SnarkReader<Any>> by lazy {
-        context.gather<SnarkReader<Any>>(SnarkReader.DF_TYPE, inherit = true)
-    }
-
-    /**
-     * A lazy-initialized map of `TextProcessor` instances used for page-based text transformation.
-     *
-     * @property textProcessors The `TextProcessor` instances accessible by their names.
-     */
-    public val textProcessors: Map<Name, TextProcessor> by lazy {
-        context.gather(TextProcessor.DF_TYPE, true)
-    }
-
-    public fun preprocessor(transformationMeta: Meta): TextProcessor {
-        val transformationName = transformationMeta.string
-            ?: transformationMeta["name"].string ?: error("Transformation name not defined in $transformationMeta")
-        return textProcessors[transformationName.parseAsName()]
-            ?: error("Text transformation with name $transformationName not found in $this")
-    }
-
-    public companion object : PluginFactory<Snark> {
-        override val tag: PluginTag = PluginTag("snark")
-
-        override fun build(context: Context, meta: Meta): Snark = Snark()
-
-        private val byteArrayIOReader = IOReader {
-            readByteArray()
-        }
-
-        internal val byteArraySnarkParser = SnarkReader(byteArrayIOReader)
-
-    }
-}

snark-core/src/commonMain/kotlin/space/kscience/snark/SnarkBuilder.kt

@@ -1,4 +0,0 @@
-package space.kscience.snark
-
-@DslMarker
-public annotation class SnarkBuilder

snark-core/src/commonMain/kotlin/space/kscience/snark/SnarkContext.kt

@@ -3,5 +3,4 @@ package space.kscience.snark
 /**
  * A marker interface for Snark Page and Site builders
  */
-@SnarkBuilder
 public interface SnarkContext

snark-core/src/commonMain/kotlin/space/kscience/snark/SnarkEnvironment.kt

@@ -0,0 +1,40 @@
+package space.kscience.snark
+
+import space.kscience.dataforge.context.Context
+import space.kscience.dataforge.context.Global
+import space.kscience.dataforge.context.Plugin
+import space.kscience.dataforge.data.DataSourceBuilder
+import space.kscience.dataforge.data.DataTree
+import space.kscience.dataforge.data.DataTreeBuilder
+import space.kscience.dataforge.meta.MutableMeta
+import kotlin.reflect.typeOf
+
+public class SnarkEnvironment(public val parentContext: Context) {
+    private var _data: DataTree<*>? = null
+    public val data: DataTree<Any> get() = _data ?: DataTree.empty()
+
+    public fun data(builder: DataSourceBuilder<Any>.() -> Unit) {
+        _data = DataTreeBuilder<Any>(typeOf<Any>(), parentContext.coroutineContext).apply(builder)
+        //TODO use node meta
+    }
+
+    public val meta: MutableMeta = MutableMeta()
+
+    public fun meta(block: MutableMeta.() -> Unit) {
+        meta.apply(block)
+    }
+
+    private val _plugins = HashSet<Plugin>()
+    public val plugins: Set<Plugin> get() = _plugins
+
+    public fun registerPlugin(plugin: Plugin) {
+        _plugins.add(plugin)
+    }
+
+    public companion object{
+        public val default: SnarkEnvironment = SnarkEnvironment(Global)
+    }
+}
+
+public fun SnarkEnvironment(parentContext: Context = Global, block: SnarkEnvironment.() -> Unit): SnarkEnvironment =
+    SnarkEnvironment(parentContext).apply(block)

snark-core/src/commonMain/kotlin/space/kscience/snark/SnarkParser.kt

@@ -0,0 +1,55 @@
+package space.kscience.snark
+
+import io.ktor.utils.io.core.Input
+import io.ktor.utils.io.core.readBytes
+import space.kscience.dataforge.context.Context
+import space.kscience.dataforge.io.IOReader
+import space.kscience.dataforge.io.asBinary
+import space.kscience.dataforge.io.readWith
+import space.kscience.dataforge.meta.Meta
+import space.kscience.dataforge.misc.Type
+import kotlin.reflect.KType
+import kotlin.reflect.typeOf
+
+/**
+ * A parser of binary content including priority flag and file extensions
+ */
+@Type(SnarkParser.TYPE)
+public interface SnarkParser<out R> {
+    public val type: KType
+
+    public val fileExtensions: Set<String>
+
+    public val priority: Int get() = DEFAULT_PRIORITY
+
+    public fun parse(context: Context, meta: Meta, bytes: ByteArray): R
+
+    public fun reader(context: Context, meta: Meta): IOReader<R> = object : IOReader<R> {
+        override val type: KType get() = this@SnarkParser.type
+
+        override fun readObject(input: Input): R = parse(context, meta, input.readBytes())
+    }
+
+    public companion object {
+        public const val TYPE: String = "snark.parser"
+        public const val DEFAULT_PRIORITY: Int = 10
+    }
+}
+
+@PublishedApi
+internal class SnarkParserWrapper<R : Any>(
+    val reader: IOReader<R>,
+    override val type: KType,
+    override val fileExtensions: Set<String>,
+) : SnarkParser<R> {
+    override fun parse(context: Context, meta: Meta, bytes: ByteArray): R = bytes.asBinary().readWith(reader)
+}
+
+/**
+ * Create a generic parser from reader
+ */
+@Suppress("FunctionName")
+public inline fun <reified R : Any> SnarkParser(
+    reader: IOReader<R>,
+    vararg fileExtensions: String,
+): SnarkParser<R> = SnarkParserWrapper(reader, typeOf<R>(), fileExtensions.toSet())

snark-core/src/commonMain/kotlin/space/kscience/snark/SnarkReader.kt

@@ -1,55 +0,0 @@
-package space.kscience.snark
-
-import space.kscience.dataforge.io.IOReader
-import space.kscience.dataforge.io.asBinary
-import space.kscience.dataforge.misc.DfType
-import space.kscience.snark.SnarkReader.Companion.DEFAULT_PRIORITY
-import space.kscience.snark.SnarkReader.Companion.DF_TYPE
-import kotlin.reflect.KType
-import kotlin.reflect.typeOf
-
-@DfType(DF_TYPE)
-public interface SnarkReader<out T> : IOReader<T> {
-    public val outputType: KType
-    public val inputContentTypes: Set<String>
-    public val priority: Int get() = DEFAULT_PRIORITY
-    public fun readFrom(source: String): T
-
-    public companion object {
-        public const val DF_TYPE: String = "snark.reader"
-        public const val DEFAULT_PRIORITY: Int = 10
-    }
-}
-
-/**
- * A wrapper class for IOReader that adds priority and MIME type handling.
- *
- * @param T The type of data to be read by the IOReader.
- * @property reader The underlying IOReader instance used for reading data.
- * @property inputContentTypes The set of supported types that can be read by the SnarkIOReader.
- * @property priority The priority of the SnarkIOReader. Higher priority SnarkIOReader instances will be preferred over lower priority ones.
- */
-
-private class SnarkReaderWrapper<out T>(
-    private val reader: IOReader<T>,
-    override val outputType: KType,
-    override val inputContentTypes: Set<String>,
-    override val priority: Int = DEFAULT_PRIORITY,
-) : IOReader<T> by reader, SnarkReader<T> {
-
-    override fun readFrom(source: String): T = readFrom(source.encodeToByteArray().asBinary())
-}
-
-public fun <T : Any> SnarkReader(
-    reader: IOReader<T>,
-    outputType: KType,
-    vararg inputContentTypes: String,
-    priority: Int = DEFAULT_PRIORITY,
-): SnarkReader<T> = SnarkReaderWrapper(reader, outputType, inputContentTypes.toSet(), priority)
-
-
-public inline fun <reified T : Any> SnarkReader(
-    reader: IOReader<T>,
-    vararg inputContentTypes: String,
-    priority: Int = DEFAULT_PRIORITY,
-): SnarkReader<T> = SnarkReader(reader, typeOf<T>(), inputContentTypes = inputContentTypes, priority)

snark-core/src/commonMain/kotlin/space/kscience/snark/TextProcessor.kt

@@ -1,21 +0,0 @@
-package space.kscience.snark
-
-import space.kscience.dataforge.misc.DfType
-import space.kscience.dataforge.names.NameToken
-
-/**
- * An object that conducts page-based text transformation. Like using link replacement or templating.
- */
-@DfType(TextProcessor.DF_TYPE)
-public fun interface TextProcessor {
-
-    public fun process(text: CharSequence): String
-
-    public companion object {
-        public const val DF_TYPE: String = "snark.textTransformation"
-        public val TEXT_TRANSFORMATION_KEY: NameToken = NameToken("transformation")
-        public val TEXT_PREPROCESSOR_KEY: NameToken = NameToken("preprocessor")
-    }
-}
-
-

snark-core/src/jvmMain/kotlin/space/kscience/snark/ImageIOReader.kt

@@ -1,17 +0,0 @@
-package space.kscience.snark
-
-import kotlinx.io.Source
-import kotlinx.io.asInputStream
-import space.kscience.dataforge.io.IOReader
-import java.awt.image.BufferedImage
-import javax.imageio.ImageIO
-
-/**
- * The ImageIOReader class is an implementation of the IOReader interface specifically for reading images using the ImageIO library.
- * It reads the image data from a given source and returns a BufferedImage object.
- *
- * @property type The KType of the data to be read by the ImageIOReader.
- */
-public object ImageIOReader : IOReader<BufferedImage> {
-    override fun readFrom(source: Source): BufferedImage = ImageIO.read(source.asInputStream())
-}

snark-core/src/jvmMain/kotlin/space/kscience/snark/snarkWorkspace.kt

@@ -1,58 +0,0 @@
-@file:OptIn(DFExperimental::class)
-
-package space.kscience.snark
-
-import space.kscience.dataforge.data.branch
-import space.kscience.dataforge.meta.*
-import space.kscience.dataforge.misc.DFExperimental
-import space.kscience.dataforge.names.parseAsName
-import space.kscience.dataforge.workspace.Workspace
-import space.kscience.dataforge.workspace.WorkspaceBuilder
-import space.kscience.dataforge.workspace.directory
-import space.kscience.dataforge.workspace.resources
-import kotlin.io.path.Path
-
-//
-///**
-// * Reads the specified resources and returns a [DataTree] containing the data.
-// *
-// * @param resources The names of the resources to read.
-// * @param classLoader The class loader to use for loading the resources. By default, it uses the current thread's context class loader.
-// * @return A DataTree containing the data read from the resources.
-// */
-//private fun IOPlugin.readResources(
-//    vararg resources: String,
-//    classLoader: ClassLoader = Thread.currentThread().contextClassLoader,
-//): DataTree<Binary> = DataTree {
-//    //    require(resource.isNotBlank()) {"Can't mount root resource tree as data root"}
-//    resources.forEach { resource ->
-//        val path = classLoader.getResource(resource)?.toURI()?.toPath() ?: error(
-//            "Resource with name $resource is not resolved"
-//        )
-//        node(resource, readRawDirectory(path))
-//    }
-//}
-
-public fun Snark.workspace(
-    meta: Meta,
-    workspaceBuilder: WorkspaceBuilder.() -> Unit = {},
-): Workspace = Workspace {
-
-
-    data {
-        meta.getIndexed("directory").forEach { (index, directoryMeta) ->
-            val dataDirectory = directoryMeta["path"].string ?: error("Directory path not defined")
-            val nodeName = directoryMeta["name"].string ?: directoryMeta.string ?: index ?: ""
-            directory(io, nodeName.parseAsName(), Path((dataDirectory)))
-        }
-        meta.getIndexed("resource").forEach { (index, resourceMeta) ->
-            val resource = resourceMeta["path"]?.stringList ?: listOf("/")
-            val nodeName = resourceMeta["name"].string ?: resourceMeta.string ?: index ?: ""
-            branch(nodeName) {
-                resources(io, *resource.toTypedArray())
-            }
-        }
-    }
-
-    workspaceBuilder()
-}

snark-document-builder/build.gradle.kts

@@ -0,0 +1,21 @@
+plugins {
+    id("space.kscience.gradle.jvm")
+    `maven-publish`
+    id("kotlinx-serialization")
+}
+
+val coroutinesVersion = space.kscience.gradle.KScienceVersions.coroutinesVersion
+val jacksonVersion = "2.14.2"
+val ktorVersion = space.kscience.gradle.KScienceVersions.ktorVersion
+
+
+dependencies {
+    implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:$coroutinesVersion")
+    api("io.ktor:ktor-server-html-builder:$ktorVersion")
+
+    implementation(project(":snark-storage-driver"))
+    implementation(project(":snark-pandoc-plugin"))
+
+    implementation("com.fasterxml.jackson.module:jackson-module-kotlin:$jacksonVersion")
+    implementation("org.jetbrains.kotlinx:kotlinx-serialization-json:1.3.3")
+}

snark-document-builder/ci/requirements.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+set -e
+
+sudo apt-get install -y python3
+sudo apt-get install -y nodejs
+sudo apt-get install -y npm
+
+pushd ../src/main/nodejs
+npm install .
+popd

snark-document-builder/src/main/kotlin/Build.kt

@@ -0,0 +1,17 @@
+package documentBuilder
+
+public class GraphManager(public val graph: DependencyGraph) {
+    fun buildDocument(file: FileName) {
+        val list = graph.nodes[file]
+        if (list != null) {
+            for (element in list.dependencies) {
+                element.visit(this)
+            }
+        }
+    }
+
+    fun getAstRootDocument(file: FileName): MdAstRoot {
+        buildDocument(file)
+        return graph.nodes[file]!!.mdAst
+    }
+}

snark-document-builder/src/main/kotlin/DependencyGraph.kt

@@ -0,0 +1,71 @@
+package documentBuilder
+
+import kotlinx.coroutines.coroutineScope
+import kotlin.collections.MutableList
+
+
+public typealias FileName = String
+
+/**
+ * Node of dependency graph.
+ *
+ * One node represents one file and its dependencies
+ *
+ * @property mdAst - AST tree of current file.
+ * @property dependencies - list of tail end adjacent to this node (dependencies of current file to be resolved).
+ */
+public data class DependencyGraphNode(
+    val mdAst: MdAstRoot,
+    val dependencies: List<DependencyGraphEdge>
+)
+
+/**
+ * Interface of all dependency edges.
+ */
+public sealed interface DependencyGraphEdge {
+    public fun visit(graphManager: GraphManager)
+}
+
+/**
+ * Include dependency edge.
+ *
+ * @property parentNode - node inside AST tree, that is parent for dependent node.
+ * @property dependentNode - dependent node, i.e. node of part of document with include commands
+ * @property includeList - list of files to be included.
+ */
+public data class IncludeDependency(
+    val parentNode: MdAstParent,
+    val dependentNode: MdAstElement,
+    val includeList: List<FileName>
+) : DependencyGraphEdge {
+    override fun visit(graphManager: GraphManager) {
+        val parent = parentNode
+        val childs: MutableList<MdAstElement> = mutableListOf()
+        for (file in includeList) {
+            graphManager.buildDocument(file)
+            childs.addAll(graphManager.graph.nodes[file]!!.mdAst.children)
+        }
+        val elements: MutableList<MdAstElement> = parent.children.toMutableList()
+        val index = parent.children.indexOf(dependentNode)
+        elements.removeAt(index)
+        elements.addAll(index, childs)
+        parent.children = elements
+    }
+}
+
+// parent - List<MdAstElement> -------------------------------------
+//                |                                                  \
+//                 \                                                  \
+//                  |                                                  \
+// dependentNode - MdAstElement                                         \
+//                                                                       |
+// List<FileName> -> List<MdAstRoot> --> List<List<MdAstElement>> ===> List<MdAstElement>
+
+/**
+ * Whole dependency graph.
+ *
+ * @property nodes - map of nodes, where you can find DependencyGraphNode of file by its name.
+ */
+public data class DependencyGraph(
+    val nodes: Map<FileName, DependencyGraphNode>
+)

snark-document-builder/src/main/kotlin/DocumentBuilder.kt

@@ -0,0 +1,111 @@
+package documentBuilder
+
+import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper
+import space.kscience.snark.pandoc.PandocCommandBuilder
+import space.kscience.snark.pandoc.PandocWrapper
+import space.kscience.snark.storage.*
+import java.nio.file.Files
+import java.nio.file.Path
+import kotlin.io.path.*
+
+private val SNARK_HTML_RENDER = "snark-document-builder/src/main/nodejs/HtmlRenderer.js"
+private val SNARK_MD_RENDERER = "snark-document-builder/src/main/nodejs/MdRenderer.js"
+fun getHtml(ast_string: String): String
+{
+    return ProcessBuilder("node", SNARK_HTML_RENDER, ast_string)
+        .redirectOutput(ProcessBuilder.Redirect.PIPE)
+        .redirectError(ProcessBuilder.Redirect.INHERIT)
+        .start().inputStream.bufferedReader().readText()
+}
+
+fun getLatex(ast_string: String) : Path
+{
+    val outputMd = Files.createTempFile(Path.of("./data/"), "output", ".md")
+
+    val output = ProcessBuilder("node", SNARK_MD_RENDERER, ast_string)
+        .redirectOutput(ProcessBuilder.Redirect.PIPE)
+        .redirectError(ProcessBuilder.Redirect.INHERIT)
+        .start().inputStream.bufferedReader().readText()
+    outputMd.writeText(output)
+
+    val outputTex = Files.createTempFile(Path.of("./data/"), "output", ".tex")
+
+    val pandocWrapper = PandocWrapper()
+    pandocWrapper.use { p: PandocWrapper? ->
+        val command = PandocCommandBuilder(
+            listOf<Path>(outputMd),
+            outputTex
+        )
+        PandocWrapper.execute(command)
+    }
+
+    return outputTex
+}
+
+private val DEFAULT_DOCUMENT_ROOT = "main.md"
+
+public suspend fun buildDocument(root: Directory, path: Path): String {
+    val dependencyGraph = buildDependencyGraph(root, path)
+
+    val graphManage = GraphManager(dependencyGraph)
+
+    graphManage.buildDocument(path.toString())
+
+//    for ((key, value) in dependencyGraph.nodes) {
+//        println("Key ${key}")
+//        println("Value.mdAst ${value.mdAst}")
+//        println("Value.dependencies ${value.dependencies}")
+//    }
+
+    val root: MdAstRoot = dependencyGraph.nodes[path.toString()]!!.mdAst
+
+    return getHtml(jacksonObjectMapper().writeValueAsString(root))
+}
+
+public suspend fun buildLatex(root: Directory, path: Path) : Path {
+    val dependencyGraph = buildDependencyGraph(root, path)
+
+    val graphManage = GraphManager(dependencyGraph)
+
+    graphManage.buildDocument(path.toString())
+
+    val root: MdAstRoot = dependencyGraph.nodes[path.toString()]!!.mdAst
+
+    return getLatex(jacksonObjectMapper().writeValueAsString(root))
+}
+
+public suspend fun buildDependencyGraph(root: Directory, path: Path): DependencyGraph {
+    val nodes = HashMap<FileName, DependencyGraphNode>()
+
+    buildNodes(root, path, nodes)
+
+    return DependencyGraph(nodes)
+}
+
+private suspend fun buildNodes(root: Directory, path: Path, nodes: HashMap<FileName, DependencyGraphNode>) {
+    val pathString = path.toString()
+
+    assert(!nodes.containsKey(pathString))
+
+    val rootDcoument = (root / path).get(DEFAULT_DOCUMENT_ROOT)
+    nodes.put(pathString, buildDependencyGraphNode(rootDcoument.readAll(), path))
+
+    val dependencies = getDependencies(nodes.getValue(pathString))
+
+    for (dependency in dependencies) {
+        if (!nodes.containsKey(dependency))
+            buildNodes(root, Path(dependency), nodes)
+    }
+}
+
+public suspend fun getDependencies(node: DependencyGraphNode): Set<FileName> {
+    val dependencies = mutableListOf<FileName>()
+
+    for (dependency in node.dependencies) {
+        when (dependency) {
+            is IncludeDependency -> dependencies.addAll(dependency.includeList)
+        }
+    }
+
+    return dependencies.toSet()
+}

snark-document-builder/src/main/kotlin/MdAstElements.kt

@@ -0,0 +1,84 @@
+package documentBuilder
+
+import kotlinx.serialization.Serializable
+import kotlinx.serialization.SerialName
+import com.fasterxml.jackson.annotation.JsonSubTypes
+import com.fasterxml.jackson.annotation.JsonTypeInfo
+
+@Serializable
+public data class Point(val line: Int, val column: Int, val offset: Int)
+
+@Serializable
+public data class Position(val start: Point, val end: Point)
+
+@JsonTypeInfo(
+    use = JsonTypeInfo.Id.NAME,
+    include = JsonTypeInfo.As.PROPERTY,
+    property = "type"
+)
+
+@JsonSubTypes(
+    JsonSubTypes.Type(value = MdAstRoot::class, name = "root"),
+    JsonSubTypes.Type(value = MdAstParagraph::class, name = "paragraph"),
+    JsonSubTypes.Type(value = MdAstText::class, name = "text"),
+    JsonSubTypes.Type(value = MdAstHeading::class, name = "heading"),
+    JsonSubTypes.Type(value = MdAstCode::class, name = "code"),
+    JsonSubTypes.Type(value = MdAstBlockquote::class, name = "blockquote")
+)
+
+@Serializable
+public sealed interface MdAstElement{
+    public abstract var position: Position
+}
+
+@Serializable
+public sealed interface MdAstParent: MdAstElement{
+    public var children: List<MdAstElement>
+}
+
+@Serializable
+@SerialName("root")
+public data class MdAstRoot(
+    override var children: List<MdAstElement>,
+    override var position: Position
+): MdAstParent
+
+
+@Serializable
+@SerialName("paragraph")
+public data class MdAstParagraph(
+    override var children: List<MdAstElement>,
+    override var position: Position
+): MdAstParent
+
+
+@Serializable
+@SerialName("text")
+public data class MdAstText(
+    val value: String,
+    override var position: Position
+): MdAstElement
+
+@Serializable
+@SerialName("heading")
+public data class MdAstHeading(
+    val depth: Int, 
+    override var children: List<MdAstElement>,
+    override var position: Position
+): MdAstParent
+
+@Serializable
+@SerialName("code")
+public data class MdAstCode(
+    var lang: String? = null,
+    var meta: String? = null,
+    var value: String,
+    override var position: Position,
+) : MdAstElement
+
+@Serializable
+@SerialName("blockquote")
+public data class MdAstBlockquote(
+    override var children: List<MdAstElement>,
+    override var position: Position
+): MdAstParent

snark-document-builder/src/main/kotlin/MdParser.kt

@@ -0,0 +1,60 @@
+package documentBuilder
+
+import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper
+import com.fasterxml.jackson.module.kotlin.readValue
+import java.nio.file.Path
+
+// snark-main/build.gradle.kts depends on these pathes
+private val MARKDOWN_PARSER = "snark-document-builder/src/main/nodejs/MarkdownParser.js"
+private val SNARK_PARSER = "snark-document-builder/src/main/python/SnarkParser.py"
+
+public suspend fun parseMd(mdFile: ByteArray, parserPath: String = MARKDOWN_PARSER): MdAstRoot {
+    val process = ProcessBuilder("node", parserPath, String(mdFile))
+    val result = process
+        .redirectOutput(ProcessBuilder.Redirect.PIPE)
+        .redirectError(ProcessBuilder.Redirect.INHERIT)
+        .start().inputStream.bufferedReader().readText()
+
+    return jacksonObjectMapper().readValue<MdAstRoot>(result)
+}
+
+public suspend fun buildDependencyGraphNode(mdFile: ByteArray, path: Path): DependencyGraphNode {
+    val treeRoot = parseMd(mdFile)
+    val dependencies = mutableListOf<DependencyGraphEdge>()
+
+    fillDependencies(treeRoot, dependencies, path)
+
+    return DependencyGraphNode(treeRoot, dependencies)
+}
+
+internal suspend fun fillDependencies(
+        currentNode: MdAstElement,
+        dependencies: MutableList<DependencyGraphEdge>,
+        path: Path) {
+    when (currentNode) {
+        is MdAstParent -> {
+            for (child in currentNode.children) {
+                if (child is MdAstText) {
+                    val includeList = getIncludeFiles(child.value).toMutableList()
+
+                    if (includeList.size > 0) {
+                        includeList.replaceAll { path.toString() + "/" + it }
+
+                        dependencies += IncludeDependency(currentNode, child, includeList)
+                    }
+                } else {
+                    fillDependencies(child, dependencies, path)
+                }
+            }
+        }
+        else -> {}
+    }
+}
+
+public suspend fun getIncludeFiles(string: String): List<FileName> {
+    return jacksonObjectMapper()
+        .readValue<List<FileName>>(ProcessBuilder("python3", SNARK_PARSER, string)
+        .redirectOutput(ProcessBuilder.Redirect.PIPE)
+        .redirectError(ProcessBuilder.Redirect.INHERIT)
+        .start().inputStream.bufferedReader().readText())
+}

snark-document-builder/src/main/nodejs/.gitignore

@@ -0,0 +1 @@
+node_modules

snark-document-builder/src/main/nodejs/HtmlRenderer.js

@@ -0,0 +1,16 @@
+import {toHast} from 'mdast-util-to-hast'
+import {toHtml} from 'hast-util-to-html'
+
+main()
+
+function main()
+{
+    if (process.argv.length < 3)
+        throw "No input"
+
+    const md_ast = JSON.parse(process.argv[2])
+    const hast = toHast(md_ast)
+    const html = toHtml(hast)
+
+    console.log(html)
+}

snark-document-builder/src/main/nodejs/MarkdownParser.js

@@ -0,0 +1,14 @@
+import {fromMarkdown} from 'mdast-util-from-markdown'
+
+main()
+
+function main()
+{
+    if (process.argv.length < 3)
+        throw "No input"
+
+    const markdown_string = process.argv[2] 
+    const mdast = fromMarkdown(markdown_string)
+
+    console.log(JSON.stringify(mdast))
+}

snark-document-builder/src/main/nodejs/MdRenderer.js

@@ -0,0 +1,14 @@
+import {toMarkdown} from 'mdast-util-to-markdown'
+
+main()
+
+function main()
+{
+    if (process.argv.length < 3)
+        throw "No input"
+
+    const md_ast = JSON.parse(process.argv[2])
+    const markdown = toMarkdown(md_ast)
+
+    console.log(markdown)
+}

snark-document-builder/src/main/nodejs/package.json

@@ -0,0 +1,30 @@
+{
+
+  "type": "module",
+
+  "dependencies": {
+
+    "fs": "^0.0.1-security",
+
+    "hast-util-to-html": "^8.0.4",
+
+    "mdast-util-to-hast": "^12.3.0",
+
+    "node-fetch": "^3.3.1",
+
+    "remark-html": "^15.0.2",
+
+    "remark-parse": "^10.0.1",
+
+    "require": "^2.4.20",
+
+    "to-vfile": "^7.2.4",
+
+    "unified": "^10.1.2",
+
+    "whatwg-fetch": "^3.6.2"
+
+  }
+
+}
+

snark-document-builder/src/main/python/SnarkParser.py

@@ -0,0 +1,26 @@
+import sys
+import re
+import json
+
+assert(len(sys.argv) >= 2)
+
+string = sys.argv[1]
+
+outputfile = sys.argv[2] if len(sys.argv) >= 3 else None
+
+pattern = r'^([\n|\t| ]*@include\([a-z|0-9|.|_|\/]*\)[\n|\t| ]*)*$'
+
+files = []
+
+if re.search("@include", string, re.IGNORECASE):
+    if re.match(pattern, string):
+        matches = re.findall(r'@include\((.*?)\)', string)
+        files.extend(matches)
+    else:
+        sys.exit("Illformed string")
+
+if outputfile is None:
+    print(json.dumps(files))
+else:
+    with open(outputfile, 'w+') as f:
+        json.dump(files, f)

snark-document-builder/src/test/kotlin/DocumentBuilder.kt

@@ -0,0 +1,12 @@
+package documentBuilder
+
+import org.junit.jupiter.api.Test
+import kotlinx.coroutines.runBlocking
+import space.kscience.snark.storage.local.localStorage
+
+class SomeTest {
+    @Test
+    fun justWorks() = runBlocking {
+        // buildDocument(Directory("../example"))
+    }
+}

snark-document-builder/src/test/resources/no_dependencies_example/main.md

@@ -0,0 +1,3 @@
+# Hello
+
+I'm almost empty test document without any dependencies

snark-gradle-plugin/build.gradle.kts

@@ -10,8 +10,8 @@ repositories{
 }
 
 dependencies{
-    implementation(spclibs.kotlin.gradle)
-    implementation("com.github.mwiede:jsch:0.2.17")
+    implementation(npmlibs.kotlin.gradle)
+    implementation("com.github.mwiede:jsch:0.2.1")
 }
 
 gradlePlugin{

snark-gradle-plugin/src/main/kotlin/SnarkGradlePlugin.kt

@@ -2,9 +2,6 @@ package space.kscience.snark.plugin
 
 import org.gradle.api.Plugin
 import org.gradle.api.Project
-import org.gradle.api.distribution.DistributionContainer
-import org.gradle.kotlin.dsl.findByType
-import org.gradle.kotlin.dsl.named
 import org.gradle.kotlin.dsl.withType
 import java.io.File
 import java.time.LocalDateTime
@@ -31,13 +28,11 @@ public class SnarkGradlePlugin : Plugin<Project> {
 
         plugins.withId("org.jetbrains.kotlin.jvm") {
             val writeBuildDate = tasks.register("writeBuildDate") {
-                val outputFile = project.layout.buildDirectory.file("resources/main/buildDate")
+                val outputFile = File(project.buildDir, "resources/main/buildDate")
                 doLast {
                     val deployDate = LocalDateTime.now()
-                    outputFile.get().asFile.run {
-                        parentFile.mkdirs()
-                        writeText(deployDate.toString())
-                    }
+                    outputFile.parentFile.mkdirs()
+                    outputFile.writeText(deployDate.toString())
                 }
                 outputs.file(outputFile)
                 outputs.upToDateWhen { false }
@@ -45,16 +40,10 @@ public class SnarkGradlePlugin : Plugin<Project> {
 
             tasks.getByName("processResources").dependsOn(writeBuildDate)
 
-        }
-
-        plugins.withId("org.gradle.application"){
-            extensions.findByType<DistributionContainer>()?.apply{
-                named<org.gradle.api.distribution.Distribution>("main"){
-                    contents {
-                        from(snarkExtension.dataDirectory){
-                            into("data")
-                        }
-                    }
+            extensions.configure<org.gradle.api.tasks.SourceSetContainer>("sourceSets") {
+                getByName("main") {
+                    logger.info("Adding ${snarkExtension.dataDirectory} to resources")
+                    resources.srcDir(snarkExtension.dataDirectory)
                 }
             }
         }

snark-gradle-plugin/src/main/kotlin/uploads.kt

@@ -14,7 +14,7 @@ private fun ChannelSftp.recursiveFolderUpload(sourceFile: File, destinationPath:
         cd(destinationPath)
         if (!sourceFile.name.startsWith(".")) put(
             FileInputStream(sourceFile),
-            sourceFile.name,
+            sourceFile.getName(),
             ChannelSftp.OVERWRITE
         )
     } else {
@@ -32,13 +32,13 @@ private fun ChannelSftp.recursiveFolderUpload(sourceFile: File, destinationPath:
 
             // else create a directory
             if (attrs != null) {
-                println("Directory $directoryPath exists IsDir=${attrs.isDir}")
+                println("Directory $directoryPath exists IsDir=${attrs.isDir()}")
             } else {
                 println("Creating directory $directoryPath")
-                mkdir(sourceFile.name)
+                mkdir(sourceFile.getName())
             }
             for (f in files) {
-                recursiveFolderUpload(f, destinationPath + "/" + sourceFile.name)
+                recursiveFolderUpload(f, destinationPath + "/" + sourceFile.getName())
             }
         }
     }

snark-html/build.gradle.kts

@@ -1,31 +1,23 @@
 plugins {
-    id("space.kscience.gradle.mpp")
+    id("space.kscience.gradle.jvm")
     `maven-publish`
 }
 
 val dataforgeVersion: String by rootProject.extra
 val ktorVersion = space.kscience.gradle.KScienceVersions.ktorVersion
 
-kscience{
-    jvm()
-    useSerialization()
-    useContextReceivers()
-    commonMain{
+dependencies {
     api(projects.snarkCore)
 
-        api(spclibs.kotlinx.html)
+    api("org.jetbrains.kotlinx:kotlinx-html:0.8.0")
     api("org.jetbrains.kotlin-wrappers:kotlin-css")
 
-        api("io.ktor:ktor-http:$ktorVersion")
+    api("io.ktor:ktor-utils:$ktorVersion")
+
     api("space.kscience:dataforge-io-yaml:$dataforgeVersion")
-        api("org.jetbrains:markdown:0.7.0")
-        api("org.freemarker:freemarker:2.3.32")
-
-    }
-
+    api("org.jetbrains:markdown:0.3.5")
 }
 
-
 readme {
     maturity = space.kscience.gradle.Maturity.EXPERIMENTAL
     feature("data") { "Data-based processing. Instead of traditional layout-based" }

snark-html/src/jvmMain/kotlin/space/kscience/snark/html/HtmlPage.kt

@@ -1,67 +0,0 @@
-package space.kscience.snark.html
-
-import kotlinx.html.HTML
-import kotlinx.html.stream.createHTML
-import kotlinx.html.visitTagAndFinalize
-import space.kscience.dataforge.data.DataSink
-import space.kscience.dataforge.data.DataTree
-import space.kscience.dataforge.data.wrap
-import space.kscience.dataforge.meta.Meta
-import space.kscience.dataforge.names.Name
-
-public fun interface HtmlPage {
-
-    context(PageContextWithData, HTML)
-    public fun renderPage()
-
-    public companion object {
-        public fun createHtmlString(
-            pageContext: PageContext,
-            dataSet: DataTree<*>?,
-            page: HtmlPage,
-        ): String = createHTML().run {
-            HTML(kotlinx.html.emptyMap, this, null).visitTagAndFinalize(this) {
-                with(PageContextWithData(pageContext, dataSet ?: DataTree.EMPTY)) {
-                    with(page) {
-                        renderPage()
-                    }
-                }
-            }
-        }
-    }
-}
-
-
-// data builders
-
-public fun DataSink<Any>.page(
-    name: Name,
-    pageMeta: Meta = Meta.EMPTY,
-    block: context(PageContextWithData) HTML.() -> Unit,
-) {
-    val page = HtmlPage(block)
-    wrap<HtmlPage>(name, page, pageMeta)
-}
-
-
-//                if (data.type == typeOf<HtmlData>()) {
-//                    val languageMeta: Meta = Language.forName(name)
-//
-//                    val dataMeta: Meta = if (languageMeta.isEmpty()) {
-//                        data.meta
-//                    } else {
-//                        data.meta.toMutableMeta().apply {
-//                            "languages" put languageMeta
-//                        }
-//                    }
-//
-//                    page(name, dataMeta) { pageContext->
-//                        head {
-//                            title = dataMeta["title"].string ?: "Untitled page"
-//                        }
-//                        body {
-//                            @Suppress("UNCHECKED_CAST")
-//                            htmlData(pageContext, data as HtmlData)
-//                        }
-//                    }
-//                }

snark-html/src/jvmMain/kotlin/space/kscience/snark/html/HtmlSite.kt

@@ -1,37 +0,0 @@
-package space.kscience.snark.html
-
-import space.kscience.dataforge.data.DataSink
-import space.kscience.dataforge.data.DataTree
-import space.kscience.dataforge.data.wrap
-import space.kscience.dataforge.meta.Meta
-import space.kscience.dataforge.meta.getIndexed
-import space.kscience.dataforge.meta.string
-import space.kscience.dataforge.names.Name
-import space.kscience.dataforge.names.asName
-import space.kscience.dataforge.names.parseAsName
-
-public fun interface HtmlSite {
-    context(SiteContextWithData)
-    public fun renderSite()
-}
-
-public fun DataSink<Any>.site(
-    name: Name,
-    siteMeta: Meta,
-    block: (siteContext: SiteContext, data: DataTree<*>?) -> Unit,
-) {
-    wrap(name, HtmlSite { block(site, siteData) }, siteMeta)
-}
-
-//public fun DataSetBuilder<Any>.site(name: Name, block: DataSetBuilder<Any>.() -> Unit) {
-//    node(name, block)
-//}
-
-internal fun DataSink<Any>.assetsFrom(rootMeta: Meta) {
-    rootMeta.getIndexed("file".asName()).forEach { (_, meta) ->
-        val webName: String? by meta.string()
-        val name by meta.string { error("File path is not provided") }
-        val fileName = name.parseAsName()
-        wrap(fileName, webName?.parseAsName() ?: fileName)
-    }
-}

snark-html/src/jvmMain/kotlin/space/kscience/snark/html/Language.kt

@@ -1,197 +0,0 @@
-package space.kscience.snark.html
-
-import space.kscience.dataforge.actions.AbstractAction
-import space.kscience.dataforge.actions.transform
-import space.kscience.dataforge.data.*
-import space.kscience.dataforge.meta.*
-import space.kscience.dataforge.names.*
-import space.kscience.snark.DataTreeWithDefault
-import space.kscience.snark.SnarkBuilder
-import space.kscience.snark.html.Language.Companion.SITE_LANGUAGE_KEY
-import space.kscience.snark.html.Language.Companion.SITE_LANGUAGE_MAP_KEY
-import kotlin.reflect.typeOf
-
-
-public class Language : Scheme() {
-
-    /**
-     * Language key override
-     */
-    public var key: String by string { error("Language key is not defined") }
-
-    /**
-     * Data location
-     */
-    public var dataPath: Name by value<Name>(
-        reader = { (it?.string ?: key).parseAsName(true) },
-        writer = { it.toString().asValue() }
-    )
-
-    /**
-     * Page name prefix override
-     */
-    public var route: Name by value<Name>(
-        reader = { (it?.string ?: key).parseAsName(true) },
-        writer = { it.toString().asValue() }
-    )
-
-//    /**
-//     * An override for data path. By default uses [prefix]
-//     */
-//    public var dataPath: String? by string()
-//
-//    /**
-//     * Target page name with a given language key
-//     */
-//    public var target: Name?
-//        get() = meta["target"].string?.parseAsName(false)
-//        set(value) {
-//            meta["target"] = value?.toString()?.asValue()
-//        }
-
-    public companion object : SchemeSpec<Language>(::Language) {
-
-        public val LANGUAGE_KEY: Name = "language".asName()
-
-        public val LANGUAGE_MAP_KEY: Name = "languageMap".asName()
-
-        public val SITE_LANGUAGE_KEY: Name = SiteContext.SITE_META_KEY + LANGUAGE_KEY
-
-        public val SITE_LANGUAGE_MAP_KEY: Name = SiteContext.SITE_META_KEY + LANGUAGE_MAP_KEY
-
-        public const val DEFAULT_LANGUAGE: String = "en"
-    }
-}
-
-public fun Language(
-    key: String,
-    route: Name = key.parseAsName(true),
-    modifier: Language.() -> Unit = {},
-): Language = Language {
-    this.key = key
-    this.route = route
-    modifier()
-}
-
-internal val Data<*>.language: String?
-    get() = meta[Language.LANGUAGE_KEY].string?.lowercase()
-
-public val SiteContext.languageMap: Map<String, Language>
-    get() = siteMeta[SITE_LANGUAGE_MAP_KEY]?.items?.map {
-        it.key.toStringUnescaped() to Language.read(it.value)
-    }?.toMap() ?: emptyMap()
-
-public val SiteContext.language: String
-    get() = siteMeta[SITE_LANGUAGE_KEY].string ?: Language.DEFAULT_LANGUAGE
-
-/**
- * Walk the data tree depth-first
- */
-private fun <T, TR : GenericDataTree<T, TR>> TR.walk(
-    namePrefix: Name = Name.EMPTY,
-): Sequence<Pair<Name, TR>> = sequence {
-    yield(namePrefix to this@walk)
-    items.forEach { (token, tree) ->
-        yieldAll(tree.walk(namePrefix + token))
-    }
-}
-
-private class LanguageMapAction(val languages: Set<Language>) : AbstractAction<Any, Any>(typeOf<Any>()) {
-    override fun DataSink<Any>.generate(data: DataTree<Any>, meta: Meta) {
-        val languageMapCache = mutableMapOf<Name, MutableMap<Language, DataTree<Any>>>()
-
-        data.walk().forEach { (name, node) ->
-            val language = node.data?.language?.let { itemLanguage -> languages.find { it.key == itemLanguage } }
-            if (language == null) {
-                // put data without a language into all buckets
-                languageMapCache[name] = languages.associateWithTo(HashMap()) { node }
-            } else {
-                // collect data with language markers
-                val nameWithoutPrefix = if (name.startsWith(language.dataPath)) name.cutFirst() else name
-                languageMapCache.getOrPut(nameWithoutPrefix) { mutableMapOf() }[language] = node
-            }
-        }
-
-        languageMapCache.forEach { (nodeName, languageMap) ->
-            val languageMapMeta = Meta {
-                languageMap.keys.forEach { language ->
-                    set(language.key, (language.route + nodeName).toString())
-                }
-            }
-            languageMap.forEach { (language, node) ->
-                val languagePrefix = language.dataPath
-                val nodeData = node.data
-                if (nodeData != null) {
-                    put(
-                        languagePrefix + nodeName,
-                        nodeData.withMeta { set(Language.LANGUAGE_MAP_KEY, languageMapMeta) }
-                    )
-                } else {
-                    wrap(languagePrefix + nodeName, Unit, Meta { set(Language.LANGUAGE_MAP_KEY, languageMapMeta) })
-                }
-            }
-        }
-
-    }
-
-    override fun DataSink<Any>.update(source: DataTree<Any>, meta: Meta, namedData: NamedData<Any>) {
-        TODO("Not yet implemented")
-    }
-}
-
-
-/**
- * Create a multiple sites for different languages. All sites use the same [content], but rely on different data
- */
-@SnarkBuilder
-public fun SiteContextWithData.multiLanguageSite(
-    defaultLanguage: Language,
-    vararg languages: Language,
-    content: HtmlSite,
-) {
-    val languageSet = setOf(defaultLanguage, *languages)
-
-    val languageMappedData = siteData.filterByType<Any>().transform(
-        LanguageMapAction(languageSet)
-    )
-
-    languageSet.forEach { language ->
-        val languageSiteMeta = Meta {
-            SITE_LANGUAGE_KEY put language.key
-            SITE_LANGUAGE_MAP_KEY put Meta {
-                languageSet.forEach {
-                    it.key put it
-                }
-            }
-        }
-
-        val overlayData = DataTreeWithDefault<Any>(
-            languageMappedData.branch(language.dataPath)!!,
-            languageMappedData.branch(defaultLanguage.dataPath)!!
-        )
-
-        site(
-            language.route,
-            overlayData,
-            siteMeta = Laminate(languageSiteMeta, siteMeta),
-            content
-        )
-    }
-}
-
-/**
- * The language key of this page
- */
-public val PageContext.language: String
-    get() = pageMeta[Language.LANGUAGE_KEY]?.string ?: pageMeta[SITE_LANGUAGE_KEY]?.string ?: Language.DEFAULT_LANGUAGE
-
-/**
- * Mapping of language keys to other language versions of this page
- */
-public val PageContext.languageMap: Map<String, Meta>
-    get() = pageMeta[Language.LANGUAGE_MAP_KEY]?.items?.mapKeys { it.key.toStringUnescaped() } ?: emptyMap()
-
-public fun PageContext.localisedPageRef(pageName: Name): String {
-    val prefix = languageMap[language]?.get(Language::dataPath.name)?.string?.parseAsName() ?: Name.EMPTY
-    return resolvePageRef(prefix + pageName)
-}

snark-html/src/jvmMain/kotlin/space/kscience/snark/html/MarkdownReader.kt

@@ -1,91 +0,0 @@
-package space.kscience.snark.html
-
-import kotlinx.io.Source
-import kotlinx.io.readString
-import org.intellij.markdown.IElementType
-import org.intellij.markdown.MarkdownElementTypes
-import org.intellij.markdown.ast.ASTNode
-import org.intellij.markdown.ast.findChildOfType
-import org.intellij.markdown.ast.getTextInNode
-import org.intellij.markdown.flavours.gfm.GFMFlavourDescriptor
-import org.intellij.markdown.flavours.space.SFMFlavourDescriptor
-import org.intellij.markdown.html.*
-import org.intellij.markdown.parser.LinkMap
-import org.intellij.markdown.parser.MarkdownParser
-import space.kscience.snark.SnarkReader
-
-private class SnarkInlineLinkGeneratingProvider(
-    baseURI: URI?,
-    resolveAnchors: Boolean = false,
-) : LinkGeneratingProvider(baseURI, resolveAnchors) {
-
-    override fun getRenderInfo(text: String, node: ASTNode): RenderInfo? {
-        return RenderInfo(
-            label = node.findChildOfType(MarkdownElementTypes.LINK_TEXT) ?: return null,
-            destination = node.findChildOfType(MarkdownElementTypes.LINK_DESTINATION)?.getTextInNode(text)?.let { raw ->
-                val processedLink = WebPageTextProcessor.functionRegex.replace(raw) { match ->
-                    when (match.groups["target"]?.value) {
-                        "homeRef" -> "snark://homeRef"
-                        "resolveRef" -> "snark://ref/${match.groups["name"]?.value ?: ""}"
-                        "resolvePageRef" -> "snark://page/${match.groups["name"]?.value ?: ""}"
-                        "pageMeta.get" -> "snark://meta/${match.groups["name"]?.value ?: ""}"
-                        else -> match.value
-                    }
-                }
-                LinkMap.normalizeDestination(processedLink, true)
-            } ?: "",
-            title = node.findChildOfType(MarkdownElementTypes.LINK_TITLE)?.getTextInNode(text)?.let {
-                LinkMap.normalizeTitle(it)
-            }
-        )
-    }
-}
-
-private class SnarkImageGeneratingProvider(
-    linkMap: LinkMap,
-    baseURI: URI?,
-) : ImageGeneratingProvider(linkMap, baseURI) {
-
-    val snarkInlineLinkProvider = SnarkInlineLinkGeneratingProvider(baseURI)
-    override fun getRenderInfo(text: String, node: ASTNode): RenderInfo? {
-        node.findChildOfType(MarkdownElementTypes.INLINE_LINK)?.let { linkNode ->
-            return snarkInlineLinkProvider.getRenderInfo(text, linkNode)
-        }
-        (node.findChildOfType(MarkdownElementTypes.FULL_REFERENCE_LINK)
-            ?: node.findChildOfType(MarkdownElementTypes.SHORT_REFERENCE_LINK))
-            ?.let { linkNode ->
-                return referenceLinkProvider.getRenderInfo(text, linkNode)
-            }
-        return null
-    }
-}
-
-public object SnarkFlavorDescriptor : GFMFlavourDescriptor(false) {
-    override fun createHtmlGeneratingProviders(linkMap: LinkMap, baseURI: URI?): Map<IElementType, GeneratingProvider> =
-        super.createHtmlGeneratingProviders(linkMap, baseURI) + mapOf(
-            MarkdownElementTypes.INLINE_LINK to SnarkInlineLinkGeneratingProvider(baseURI, absolutizeAnchorLinks)
-                .makeXssSafe(useSafeLinks),
-            MarkdownElementTypes.IMAGE to SnarkImageGeneratingProvider(linkMap, baseURI)
-                .makeXssSafe(useSafeLinks),
-        )
-}
-
-public object MarkdownReader : SnarkHtmlReader {
-    override val inputContentTypes: Set<String> = setOf("text/markdown", "md", "markdown")
-
-    override fun readFrom(source: String): PageFragment = PageFragment {
-        val parsedTree = markdownParser.parse(IElementType("ROOT"), source)
-//        markdownParser.buildMarkdownTreeFromString(source)
-        val htmlString = HtmlGenerator(source, parsedTree, markdownFlavor).generateHtml()
-
-        consumer.onTagContentUnsafe {
-            +htmlString
-        }
-
-    }
-
-    private val markdownFlavor = SnarkFlavorDescriptor
-    private val markdownParser = MarkdownParser(markdownFlavor)
-
-    override fun readFrom(source: Source): PageFragment = readFrom(source.readString())
-}

snark-html/src/jvmMain/kotlin/space/kscience/snark/html/MetaMaskData.kt

@@ -1,20 +0,0 @@
-package space.kscience.snark.html
-
-import space.kscience.dataforge.data.Data
-import space.kscience.dataforge.meta.Meta
-import space.kscience.dataforge.meta.MutableMeta
-import space.kscience.dataforge.meta.copy
-
-
-private class MetaMaskData<T>(val origin: Data<T>, override val meta: Meta) : Data<T> by origin
-
-/**
- * Data with overridden meta. It reflects original data computed state.
- */
-public fun <T> Data<T>.withMeta(newMeta: Meta): Data<T> = if (this is MetaMaskData) {
-    MetaMaskData(origin, newMeta)
-} else {
-    MetaMaskData(this, newMeta)
-}
-
-public inline fun <T> Data<T>.withMeta(block: MutableMeta.() -> Unit): Data<T> = withMeta(meta.copy(block))

snark-html/src/jvmMain/kotlin/space/kscience/snark/html/PageContext.kt

@@ -1,90 +0,0 @@
-package space.kscience.snark.html
-
-import io.ktor.http.URLBuilder
-import io.ktor.http.Url
-import io.ktor.http.appendPathSegments
-import space.kscience.dataforge.context.Context
-import space.kscience.dataforge.context.ContextAware
-import space.kscience.dataforge.data.DataTree
-import space.kscience.dataforge.meta.Meta
-import space.kscience.dataforge.meta.get
-import space.kscience.dataforge.meta.string
-import space.kscience.dataforge.names.Name
-import space.kscience.dataforge.names.hasIndex
-import space.kscience.dataforge.names.parseAsName
-import space.kscience.dataforge.names.plus
-import space.kscience.snark.SnarkBuilder
-import space.kscience.snark.SnarkContext
-
-public fun Name.toWebPath(): String = tokens.joinToString(separator = "/") {
-    if (it.hasIndex()) {
-        "${it.body}[${it.index}]"
-    } else {
-        it.body
-    }
-}
-
-/**
- *  A context for building a single page
- */
-@SnarkBuilder
-public interface PageContext : SnarkContext, ContextAware {
-
-    public val site: SiteContext
-
-    override val context: Context get() = site.context
-
-
-    public val host: Url
-
-    /**
-     * A metadata for a page. It should include site metadata
-     */
-    public val pageMeta: Meta
-
-    /**
-     * A route relative to parent site. Includes [SiteContext.siteRoute].
-     */
-    public val pageRoute: Name
-
-    /**
-     * Resolve absolute url for given relative [ref]
-     *
-     */
-    public fun resolveRef(ref: String, targetSite: SiteContext = site): String {
-        val pageUrl = URLBuilder(host)
-            .appendPathSegments(targetSite.path, true)
-            .appendPathSegments(ref)
-
-        return pageUrl.buildString().removeSuffix("/")
-    }
-
-    /**
-     * Resolve absolute url for a page with given [pageName].
-     */
-    public fun resolvePageRef(pageName: Name, targetSite: SiteContext = site): String
-
-    public fun resolveRelativePageRef(pageName: Name): String = resolvePageRef(pageRoute + pageName)
-
-}
-
-context(PageContext)
-public val page: PageContext
-    get() = this@PageContext
-
-context(PageContextWithData)
-public val page: PageContextWithData
-    get() = this@PageContextWithData
-
-public fun PageContext.resolvePageRef(pageName: String, targetSite: SiteContext = site): String =
-    resolvePageRef(pageName.parseAsName(), targetSite)
-
-public val PageContext.homeRef: String get() = resolvePageRef(Name.EMPTY)
-
-public val PageContext.name: Name? get() = pageMeta["name"].string?.parseAsName()
-
-
-public class PageContextWithData(
-    private val pageContext: PageContext,
-    public val data: DataTree<*>,
-) : PageContext by pageContext

snark-html/src/jvmMain/kotlin/space/kscience/snark/html/PageFragment.kt

@@ -1,84 +0,0 @@
-package space.kscience.snark.html
-
-import kotlinx.coroutines.runBlocking
-import kotlinx.html.FlowContent
-import space.kscience.dataforge.data.*
-import space.kscience.dataforge.meta.Meta
-import space.kscience.dataforge.meta.get
-import space.kscience.dataforge.meta.int
-import space.kscience.dataforge.meta.string
-import space.kscience.dataforge.names.Name
-import space.kscience.dataforge.names.parseAsName
-import space.kscience.dataforge.names.plus
-import space.kscience.dataforge.names.startsWith
-import space.kscience.snark.SnarkContext
-
-public fun interface PageFragment {
-
-    context(PageContextWithData, FlowContent) public fun renderFragment()
-}
-
-context(PageContextWithData, FlowContent)
-public fun fragment(fragment: PageFragment): Unit {
-    with(fragment) {
-        renderFragment()
-    }
-}
-
-
-context(PageContextWithData, FlowContent)
-public fun fragment(data: Data<PageFragment>): Unit = runBlocking {
-    fragment(data.await())
-}
-
-
-context(SnarkContext)
-public val Data<*>.id: String
-    get() = meta["id"]?.string ?: "block[${hashCode()}]"
-
-context(SnarkContext)
-public val Data<*>.order: Int?
-    get() = meta["order"]?.int
-
-context(SnarkContext)
-public val Data<*>.published: Boolean
-    get() = meta["published"].string != "false"
-
-
-/**
- * Resolve a Html builder by its full name
- */
-context(SnarkContext)
-public fun DataTree<*>.resolveHtmlOrNull(name: Name): Data<PageFragment>? {
-    val resolved = (getByType<PageFragment>(name) ?: getByType<PageFragment>(name + SiteContext.INDEX_PAGE_TOKEN))
-
-    return resolved?.takeIf {
-        it.published //TODO add language confirmation
-    }
-}
-
-context(SnarkContext)
-public fun DataTree<*>.resolveHtmlOrNull(name: String): Data<PageFragment>? = resolveHtmlOrNull(name.parseAsName())
-
-context(SnarkContext)
-public fun DataTree<*>.resolveHtml(name: String): Data<PageFragment> = resolveHtmlOrNull(name)
-    ?: error("Html fragment with name $name is not resolved")
-
-/**
- * Find all Html blocks using given name/meta filter
- */
-context(SnarkContext)
-public fun DataTree<*>.resolveAllHtml(
-    predicate: (name: Name, meta: Meta) -> Boolean,
-): Map<Name, Data<PageFragment>> = filterByType<PageFragment> { name, meta, _ ->
-    predicate(name, meta) && meta["published"].string != "false"
-    //TODO add language confirmation
-}.asSequence().associate { it.name to it.data }
-
-context(SnarkContext)
-public fun DataTree<*>.findHtmlByContentType(
-    contentType: String,
-    baseName: Name = Name.EMPTY,
-): Map<Name, Data<PageFragment>> = resolveAllHtml { name, meta ->
-    name.startsWith(baseName) && meta["content_type"].string == contentType
-}

snark-html/src/jvmMain/kotlin/space/kscience/snark/html/ParseAction.kt

@@ -1,68 +0,0 @@
-package space.kscience.snark.html
-
-import space.kscience.dataforge.actions.AbstractAction
-import space.kscience.dataforge.data.*
-import space.kscience.dataforge.io.Binary
-import space.kscience.dataforge.io.toByteArray
-import space.kscience.dataforge.meta.Meta
-import space.kscience.dataforge.meta.get
-import space.kscience.dataforge.misc.DFInternal
-import space.kscience.snark.SnarkReader
-import space.kscience.snark.TextProcessor
-import kotlin.coroutines.CoroutineContext
-import kotlin.coroutines.EmptyCoroutineContext
-import kotlin.reflect.KType
-import kotlin.reflect.typeOf
-
-@OptIn(DFInternal::class)
-internal fun <T, R> Data<T>.transform(
-    type: KType,
-    meta: Meta = this.meta,
-    coroutineContext: CoroutineContext = EmptyCoroutineContext,
-    block: suspend (T) -> R,
-): Data<R> {
-    val data = Data(type, meta, coroutineContext, listOf(this)) {
-        block(await())
-    }
-    return data
-}
-
-public class ParseAction(private val snarkHtml: SnarkHtml) :
-    AbstractAction<Binary, Any>(typeOf<PageFragment>()) {
-
-    private fun parseOne(data: NamedData<Binary>): NamedData<Any>? = with(snarkHtml) {
-        val contentType = getContentType(data.name, data.meta)
-
-        val parser: SnarkReader<Any>? = snark.readers.values.filter { parser ->
-            contentType in parser.inputContentTypes
-        }.maxByOrNull {
-            it.priority
-        }
-
-        //ignore data for which parser is not found
-        if (parser != null) {
-            val preprocessor = meta[TextProcessor.TEXT_PREPROCESSOR_KEY]?.let { snark.preprocessor(it) }
-            data.transform(parser.outputType) {
-                if (preprocessor == null) {
-                    parser.readFrom(it)
-                } else {
-                    //TODO provide encoding
-                    val string = it.toByteArray().decodeToString()
-                    parser.readFrom(preprocessor.process(string))
-                }
-            }.named(data.name)
-        } else {
-            null
-        }
-    }
-
-    override fun DataSink<Any>.generate(data: DataTree<Binary>, meta: Meta) {
-        data.forEach {
-            parseOne(it)?.let { put(it) }
-        }
-    }
-
-    override fun DataSink<Any>.update(source: DataTree<Binary>, meta: Meta, namedData: NamedData<Binary>) {
-        parseOne(namedData)?.let { put(it) }
-    }
-}

snark-html/src/jvmMain/kotlin/space/kscience/snark/html/Postprocessor.kt

@@ -1,111 +0,0 @@
-package space.kscience.snark.html
-
-import kotlinx.html.*
-import space.kscience.dataforge.meta.string
-import space.kscience.dataforge.names.parseAsName
-import space.kscience.snark.TextProcessor
-import java.net.URI
-
-public class WebPageTextProcessor(private val page: PageContext) : TextProcessor {
-
-    /**
-     * A basic [TextProcessor] that replaces `${...}` expressions in text. The following expressions are recognised:
-     * * `homeRef` resolves to [homeRef]
-     * * `resolveRef("...")` -> [PageContext.resolveRef]
-     * * `resolvePageRef("...")` -> [PageContext.resolvePageRef]
-     * * `pageMeta.get("...") -> [PageContext.pageMeta] get string method
-     * Otherwise return unchanged string
-     */
-    override fun process(text: CharSequence): String = text.replace(functionRegex) { match ->
-        when (match.groups["target"]?.value) {
-            "homeRef" -> page.homeRef
-            "resolveRef" -> {
-                val refString = match.groups["name"]?.value ?: error("resolveRef requires a string (quoted) argument")
-                page.resolveRef(refString)
-            }
-
-            "resolvePageRef" -> {
-                val refString = match.groups["name"]?.value
-                    ?: error("resolvePageRef requires a string (quoted) argument")
-                page.localisedPageRef(refString.parseAsName())
-            }
-
-            "pageMeta.get" -> {
-                val nameString = match.groups["name"]?.value
-                    ?: error("resolvePageRef requires a string (quoted) argument")
-                page.pageMeta[nameString.parseAsName()].string ?: "@null"
-            }
-
-            "siteMeta.get" -> {
-                val nameString = match.groups["name"]?.value
-                    ?: error("resolvePageRef requires a string (quoted) argument")
-                page.site.siteMeta[nameString.parseAsName()].string ?: "@null"
-            }
-
-            else -> match.value
-        }
-    }.replace(attributeRegex) { match ->
-        val uri = URI(match.groups["uri"]!!.value)
-        val snarkUrl = when (uri.authority) {
-            "homeRef" -> page.homeRef
-            "ref" -> page.resolveRef(uri.path)
-            "page" -> page.localisedPageRef(uri.path.parseAsName())
-            "pageMeta" -> page.pageMeta[uri.path.parseAsName()].string ?: "@null"
-            "siteMeta" -> page.site.siteMeta[uri.path.parseAsName()].string ?: "@null"
-            else -> match.value
-        }
-        "=\"$snarkUrl\""
-    }
-
-    public companion object {
-        internal val functionRegex = """\$\{(?<target>[\w.]*)(?:\((?:"|&quot;)(?<name>.*)(?:"|&quot;)\))?\}""".toRegex()
-        private val attributeRegex = """="(?<uri>snark://([^"]*))"""".toRegex()
-    }
-}
-
-/**
- * A tag consumer wrapper that wraps existing [TagConsumer] and adds postprocessing.
- *
- */
-public class Postprocessor<out R>(
-    public val page: PageContext,
-    private val consumer: TagConsumer<R>,
-    private val textProcessor: TextProcessor,
-) : TagConsumer<R> by consumer {
-
-    override fun onTagAttributeChange(tag: Tag, attribute: String, value: String?) {
-        if (tag is A && attribute == "href" && value != null) {
-            consumer.onTagAttributeChange(tag, attribute, textProcessor.process(value))
-        } else if (tag is IMG && attribute == "src" && value != null) {
-            consumer.onTagAttributeChange(tag, attribute, textProcessor.process(value))
-        } else {
-            consumer.onTagAttributeChange(tag, attribute, value)
-        }
-    }
-
-    override fun onTagContent(content: CharSequence) {
-        consumer.onTagContent(textProcessor.process(content))
-    }
-
-    override fun onTagContentUnsafe(block: Unsafe.() -> Unit) {
-        val proxy = object : Unsafe {
-            override fun String.unaryPlus() {
-                consumer.onTagContentUnsafe {
-                    textProcessor.process(this@unaryPlus).unaryPlus()
-                }
-            }
-        }
-        proxy.block()
-    }
-}
-
-context(PageContext)
-public inline fun FlowContent.postprocess(
-    processor: TextProcessor = WebPageTextProcessor(page),
-    block: FlowContent.() -> Unit,
-) {
-    val fc = object : FlowContent by this {
-        override val consumer: TagConsumer<*> = Postprocessor(page, this@postprocess.consumer, processor)
-    }
-    fc.block()
-}

snark-html/src/jvmMain/kotlin/space/kscience/snark/html/SiteContext.kt

@@ -1,237 +0,0 @@
-package space.kscience.snark.html
-
-import space.kscience.dataforge.context.Context
-import space.kscience.dataforge.context.ContextAware
-import space.kscience.dataforge.data.*
-import space.kscience.dataforge.io.Binary
-import space.kscience.dataforge.meta.Meta
-import space.kscience.dataforge.meta.get
-import space.kscience.dataforge.meta.string
-import space.kscience.dataforge.names.*
-import space.kscience.snark.SnarkBuilder
-import space.kscience.snark.SnarkContext
-
-
-/**
- * An abstraction, which is used to render sites to the different rendering engines
- */
-@SnarkBuilder
-public interface SiteContext : SnarkContext, ContextAware {
-
-    public val parent: SiteContext?
-
-    /**
-     * A context path segments for this site
-     */
-    public val path: List<String>
-
-    /**
-     * Route name of this [SiteContext] relative to the site root
-     */
-    public val siteRoute: Name
-
-    /**
-     * Site configuration
-     */
-    public val siteMeta: Meta
-
-    /**
-     * Renders a static file or resource for the given route and data.
-     *
-     * @param route The route name of the static file relative to the site root.
-     * @param data The data object containing the binary data for the static file.
-     */
-    public fun static(route: Name, data: Data<Binary>)
-
-
-    /**
-     * Create a single page at given [route]. If the route is empty, create an index page the current route.
-     *
-     * @param pageMeta additional page meta. [PageContext] will use both it and [siteMeta]
-     */
-    @SnarkBuilder
-    public fun page(
-        route: Name,
-        data: DataTree<*>?,
-        pageMeta: Meta = Meta.EMPTY,
-        content: HtmlPage,
-    )
-
-    /**
-     * Create a route block with its own data. Does not change the context path
-     */
-    @SnarkBuilder
-    public fun route(
-        route: Name,
-        data: DataTree<*>?,
-        siteMeta: Meta = Meta.EMPTY,
-        content: HtmlSite,
-    )
-
-    /**
-     * Creates a sub-site and changes context path to match [name]
-     * @param route mount site at [rootName]
-     */
-    @SnarkBuilder
-    public fun site(
-        route: Name,
-        data: DataTree<*>?,
-        siteMeta: Meta = Meta.EMPTY,
-        content: HtmlSite,
-    )
-
-
-    public companion object {
-        public val SITE_META_KEY: Name = "site".asName()
-        public val INDEX_PAGE_TOKEN: NameToken = NameToken("index")
-        public val UP_PAGE_TOKEN: NameToken = NameToken("..")
-    }
-}
-
-public fun SiteContext.static(dataSet: DataTree<Binary>, prefix: Name = Name.EMPTY) {
-    dataSet.forEach { (name, data) ->
-        static(prefix + name, data)
-    }
-}
-
-public fun SiteContext.static(dataSet: DataTree<*>, branch: String, prefix: String = branch) {
-    val branchName = branch.parseAsName()
-    val prefixName = prefix.parseAsName()
-    dataSet.branch(branchName)?.filterByType<Binary>()?.forEach {
-        static(prefixName + it.name, it.data)
-    }
-}
-
-
-context(SiteContext)
-public val site: SiteContext
-    get() = this@SiteContext
-
-context(SiteContextWithData)
-public val site: SiteContextWithData
-    get() = this@SiteContextWithData
-
-/**
- * A wrapper for site context that allows convenient site building experience
- */
-public class SiteContextWithData(private val site: SiteContext, public val siteData: DataTree<*>) : SiteContext by site
-
-
-@SnarkBuilder
-public fun SiteContextWithData.static(branch: String, prefix: String = branch): Unit = static(siteData, branch, prefix)
-
-
-@SnarkBuilder
-public fun SiteContextWithData.page(
-    route: Name = Name.EMPTY,
-    pageMeta: Meta = Meta.EMPTY,
-    content: HtmlPage,
-): Unit = page(route, siteData, pageMeta, content)
-
-@SnarkBuilder
-public fun SiteContextWithData.route(
-    route: String,
-    data: DataTree<*>? = siteData,
-    siteMeta: Meta = Meta.EMPTY,
-    content: HtmlSite,
-): Unit = route(route.parseAsName(), data, siteMeta, content)
-
-@SnarkBuilder
-public fun SiteContextWithData.site(
-    route: String,
-    data: DataTree<*>? = siteData,
-    siteMeta: Meta = Meta.EMPTY,
-    content: HtmlSite,
-): Unit = site(route.parseAsName(), data, siteMeta, content)
-
-/**
- * Render all pages and sites found in the data
- */
-public suspend fun SiteContext.renderPages(data: DataTree<*>): Unit {
-
-    // Render all sub-sites
-    data.filterByType<HtmlSite>().forEach { siteData: NamedData<HtmlSite> ->
-        // generate a sub-site context and render the data in sub-site context
-        val dataPrefix = siteData.meta["site.dataPath"].string?.asName() ?: Name.EMPTY
-        site(
-            route = siteData.meta["site.route"].string?.asName() ?: siteData.name,
-            data.branch(dataPrefix) ?: DataTree.EMPTY,
-            siteMeta = siteData.meta,
-            siteData.await()
-        )
-    }
-
-    // Render all stand-alone pages in default site
-    data.filterByType<HtmlPage>().forEach { pageData: NamedData<HtmlPage> ->
-        val dataPrefix = pageData.meta["page.dataPath"].string?.asName() ?: Name.EMPTY
-        page(
-            route = pageData.meta["page.route"].string?.asName() ?: pageData.name,
-            data.branch(dataPrefix) ?: DataTree.EMPTY,
-            pageMeta = pageData.meta,
-            pageData.await()
-        )
-    }
-}
-
-
-//
-///**
-// * Recursively renders the data items in [data]. If [LAYOUT_KEY] is defined in an item, use it to load
-// * layout from the context, otherwise render children nodes as name segments and individual data items using [dataRenderer].
-// */
-//public fun SiteContext.pages(
-//    dataRenderer: DataRenderer = DataRenderer.DEFAULT,
-//) {
-//    val layoutMeta = siteData().meta[LAYOUT_KEY]
-//    if (layoutMeta != null) {
-//        //use layout if it is defined
-//        snark.siteLayout(layoutMeta).render(siteData())
-//    } else {
-//        when (siteData()) {
-//            is DataTreeItem.Node -> {
-//                siteData().tree.items.forEach { (token, item) ->
-//                    //Don't apply index token
-//                    if (token == SiteLayout.INDEX_PAGE_TOKEN) {
-//                        pages(item, dataRenderer)
-//                    } else if (item is DataTreeItem.Leaf) {
-//                        dataRenderer(token.asName(), item.data)
-//                    } else {
-//                        route(token.asName()) {
-//                            pages(item, dataRenderer)
-//                        }
-//                    }
-//                }
-//            }
-//
-//            is DataTreeItem.Leaf -> {
-//                dataRenderer(Name.EMPTY, siteData().data)
-//            }
-//        }
-//        siteData().meta[SiteLayout.ASSETS_KEY]?.let {
-//            assetsFrom(it)
-//        }
-//    }
-//    //TODO watch for changes
-//}
-//
-///**
-// * Render all pages in a node with given name
-// */
-//public fun SiteContext.pages(
-//    dataPath: Name,
-//    remotePath: Name = dataPath,
-//    dataRenderer: DataRenderer = DataRenderer.DEFAULT,
-//) {
-//    val item = resolveData.getItem(dataPath) ?: error("No data found by name $dataPath")
-//    route(remotePath) {
-//        pages(item, dataRenderer)
-//    }
-//}
-//
-//public fun SiteContext.pages(
-//    dataPath: String,
-//    remotePath: Name = dataPath.parseAsName(),
-//    dataRenderer: DataRenderer = DataRenderer.DEFAULT,
-//) {
-//    pages(dataPath.parseAsName(), remotePath, dataRenderer = dataRenderer)
-//}

snark-html/src/jvmMain/kotlin/space/kscience/snark/html/SnarkHtml.kt

@@ -1,162 +0,0 @@
-@file:OptIn(DFExperimental::class)
-
-package space.kscience.snark.html
-
-import io.ktor.http.ContentType
-import kotlinx.coroutines.CoroutineScope
-import kotlinx.io.readByteArray
-import space.kscience.dataforge.actions.Action
-import space.kscience.dataforge.actions.mapping
-import space.kscience.dataforge.actions.transform
-import space.kscience.dataforge.context.Context
-import space.kscience.dataforge.context.PluginFactory
-import space.kscience.dataforge.context.PluginTag
-import space.kscience.dataforge.data.*
-import space.kscience.dataforge.io.Binary
-import space.kscience.dataforge.io.IOPlugin
-import space.kscience.dataforge.io.IOReader
-import space.kscience.dataforge.io.JsonMetaFormat
-import space.kscience.dataforge.io.yaml.YamlMetaFormat
-import space.kscience.dataforge.io.yaml.YamlPlugin
-import space.kscience.dataforge.meta.Meta
-import space.kscience.dataforge.meta.get
-import space.kscience.dataforge.meta.set
-import space.kscience.dataforge.meta.string
-import space.kscience.dataforge.misc.DFExperimental
-import space.kscience.dataforge.names.*
-import space.kscience.dataforge.provider.dfType
-import space.kscience.dataforge.workspace.*
-import space.kscience.snark.ReWrapAction
-import space.kscience.snark.Snark
-import space.kscience.snark.SnarkReader
-import space.kscience.snark.TextProcessor
-import java.net.URLConnection
-import kotlin.io.path.Path
-import kotlin.io.path.extension
-import kotlin.reflect.typeOf
-
-
-public fun <T : Any, R : Any> DataTree<T>.transform(action: Action<T, R>, meta: Meta = Meta.EMPTY): DataTree<R> =
-    action.execute(this, meta)
-
-/**
- * A plugin used for rendering a [DataTree] as HTML
- */
-public class SnarkHtml : WorkspacePlugin() {
-    public val snark: Snark by require(Snark)
-    private val yaml by require(YamlPlugin)
-    public val io: IOPlugin get() = snark.io
-
-    override val tag: PluginTag get() = Companion.tag
-
-    override fun content(target: String): Map<Name, Any> = when (target) {
-        SnarkReader::class.dfType -> mapOf(
-            "html".asName() to HtmlReader,
-            "markdown".asName() to MarkdownReader,
-            "json".asName() to SnarkReader<Meta>(JsonMetaFormat, ContentType.Application.Json.toString()),
-            "yaml".asName() to SnarkReader<Meta>(YamlMetaFormat, "text/yaml", "yaml"),
-        )
-
-        else -> super.content(target)
-    }
-
-    internal fun getContentType(name: Name, meta: Meta): String = meta[CONTENT_TYPE_KEY].string ?: run {
-        val filePath = meta[FileData.FILE_PATH_KEY]?.string ?: name.toString()
-        URLConnection.guessContentTypeFromName(filePath) ?: Path(filePath).extension
-    }
-
-    internal val prepareHeaderAction: ReWrapAction<Any> = ReWrapAction(
-        type = typeOf<Any>(),
-        newMeta = { name ->
-            val contentType = getContentType(name, this)
-            set(FILE_NAME_KEY, name.last().toStringUnescaped())
-            set(CONTENT_TYPE_KEY, contentType)
-        }
-    ) { name, _, type ->
-        name.replaceLast { token ->
-            val extension = token.body.substringAfterLast('.')
-            if (type != typeOf<Binary>()) {
-                NameToken(token.body.removeSuffix(".$extension"))
-            } else {
-                token
-            }
-        }
-    }
-
-    public val removeIndexAction: ReWrapAction<Any> = ReWrapAction(typeOf<Any>()) { name, _, _ ->
-        if (name.endsWith("index")) name.cutLast() else name
-    }
-
-    public fun parseMarkup(name: Name, markup: String, meta: Meta): PageFragment {
-        val contentType = getContentType(name, meta)
-
-        val parser = snark.readers.values.filterIsInstance<SnarkHtmlReader>().filter { parser ->
-            contentType in parser.inputContentTypes
-        }.maxByOrNull {
-            it.priority
-        } ?: error("Parser for name $name and meta $meta not found")
-
-        //ignore data for which parser is not found
-
-        val preprocessor = meta[TextProcessor.TEXT_PREPROCESSOR_KEY]?.let { snark.preprocessor(it) }
-        return if (preprocessor == null) {
-            parser.readFrom(markup)
-        } else {
-            parser.readFrom(preprocessor.process(markup))
-        }
-    }
-
-    public val parseAction: Action<Binary, Any> = ParseAction(this)
-
-    public val layoutAction: Action<Any, Any> = Action.mapping {
-
-    }
-
-    private val allDataNotNull: DataSelector<Any>
-        get() = DataSelector { workspace, _ -> workspace.data.filterByType() }
-
-    public val parse: TaskReference<Any> by task<Any>({
-        description = "Parse all data for which reader is resolved"
-    }) {
-        //put all data
-        putAll(from(allDataNotNull))
-        //override parsed data
-        putAll(from(allDataNotNull).filterByType<Binary>().transform(parseAction))
-    }
-
-
-    public companion object : PluginFactory<SnarkHtml> {
-        override val tag: PluginTag = PluginTag("snark.html")
-
-        public val FILE_NAME_KEY: Name = "contentType".asName()
-
-        public val CONTENT_TYPE_KEY: Name = "contentType".asName()
-
-        override fun build(context: Context, meta: Meta): SnarkHtml = SnarkHtml()
-
-        private val byteArrayIOReader = IOReader {
-            readByteArray()
-        }
-
-        internal val byteArraySnarkParser = SnarkReader(byteArrayIOReader)
-
-    }
-}
-
-
-public fun SnarkHtml.readSiteData(
-    binaries: DataTree<Binary>,
-    meta: Meta = Meta.EMPTY,
-): DataTree<Any> = ObservableDataTree(context) {
-    //put all binaries
-    putAll(binaries)
-    //override ones which could be parsed
-    putAll(binaries.transform(parseAction, meta))
-}.transform(prepareHeaderAction, meta).transform(removeIndexAction, meta)
-
-
-public fun SnarkHtml.readSiteData(
-    coroutineScope: CoroutineScope,
-    meta: Meta = Meta.EMPTY,
-    builder: DataSink<Binary>.() -> Unit,
-): DataTree<Any> = readSiteData(ObservableDataTree(coroutineScope) { builder() }, meta)

snark-html/src/jvmMain/kotlin/space/kscience/snark/html/SnarkHtmlReader.kt

@@ -1,27 +0,0 @@
-package space.kscience.snark.html
-
-import kotlinx.html.div
-import kotlinx.html.unsafe
-import kotlinx.io.Source
-import kotlinx.io.readString
-import space.kscience.snark.SnarkReader
-import kotlin.reflect.KType
-import kotlin.reflect.typeOf
-
-
-public interface SnarkHtmlReader : SnarkReader<PageFragment>{
-    override val outputType: KType get() = typeOf<PageFragment>()
-}
-
-public object HtmlReader : SnarkHtmlReader {
-    override val inputContentTypes: Set<String> = setOf("html")
-
-    override fun readFrom(source: String): PageFragment = PageFragment {
-        div {
-            unsafe { +source }
-        }
-    }
-
-    override fun readFrom(source: Source): PageFragment = readFrom(source.readString())
-}
-

snark-html/src/jvmMain/kotlin/space/kscience/snark/html/document/DocumentBuilder.kt

@@ -1,174 +0,0 @@
-package space.kscience.snark.html.document
-
-import kotlinx.coroutines.launch
-import kotlinx.coroutines.runBlocking
-import kotlinx.html.*
-import space.kscience.dataforge.context.info
-import space.kscience.dataforge.context.logger
-import space.kscience.dataforge.context.request
-import space.kscience.dataforge.data.*
-import space.kscience.dataforge.io.Binary
-import space.kscience.dataforge.meta.Laminate
-import space.kscience.dataforge.meta.Meta
-import space.kscience.dataforge.meta.get
-import space.kscience.dataforge.meta.string
-import space.kscience.dataforge.names.*
-import space.kscience.snark.SnarkBuilder
-import space.kscience.snark.SnarkContext
-import space.kscience.snark.html.*
-import kotlin.reflect.typeOf
-
-/**
- *  A context for building a single document
- */
-@SnarkBuilder
-public interface DocumentBuilder : SnarkContext {
-
-    public val route: Name
-
-    public val documentMeta: Meta
-
-    public val data: DataTree<*>
-
-    public suspend fun fragment(fragment: Data<*>, overrideMeta: Meta? = null)
-
-    public suspend fun fragment(fragment: DocumentFragment, overrideMeta: Meta? = null)
-}
-
-context(SiteContextWithData)
-public suspend fun DocumentBuilder.fragment(fragmentName: Name) {
-    fragment(site.siteData[fragmentName] ?: error("Can't find data fragment for $fragmentName in site data."))
-}
-
-context(SiteContextWithData)
-public suspend fun DocumentBuilder.fragment(fragmentName: String) {
-    fragment(site.siteData[fragmentName] ?: error("Can't find data fragment for $fragmentName in site data."))
-}
-
-private class PageBasedDocumentBuilder(
-    val page: PageContextWithData,
-    private val dataRootName: Name,
-) : DocumentBuilder {
-    override val route: Name get() = page.pageRoute
-    override val documentMeta: Meta get() = page.pageMeta
-    override val data: DataTree<*> = page.data.branch(dataRootName) ?: DataTree.EMPTY
-
-    val fragments = mutableListOf<PageFragment>()
-
-    fun fragment(pageFragment: PageFragment) {
-        fragments.add(pageFragment)
-    }
-
-    override suspend fun fragment(fragment: DocumentFragment, overrideMeta: Meta?) {
-        when (fragment) {
-            is ImageDocumentFragment -> fragment {
-                figure("snark-figure") {
-                    img(classes = "snark-image") {
-                        src = resolveRef(this@PageBasedDocumentBuilder.route.toWebPath() + "/" + fragment.ref)
-                        alt = fragment.meta["alt"].string ?: ""
-                    }
-                    fragment.meta["caption"].string?.let { caption ->
-                        figcaption("snark-figure-caption") { +caption }
-                    }
-                }
-            }
-
-            is MarkupDocumentFragment -> {
-                val snarkHtml = page.context.request(SnarkHtml)
-                snarkHtml.parseMarkup(Name.EMPTY, fragment.text, fragment.meta)
-            }
-
-            is DataDocumentFragment -> {
-                val data = data[fragment.name]
-                    ?: error("Can't find data with name ${fragment.name} for $fragment")
-                fragment(data)
-            }
-
-            is ListDocumentFragment -> {
-                val meta = Laminate(overrideMeta, fragment.meta)
-                fragment.fragments.forEach { fragment(it, meta) }
-            }
-
-            is LayoutDocumentFragment -> TODO("Layouts are not implemented")
-        }
-    }
-
-    override suspend fun fragment(fragment: Data<*>, overrideMeta: Meta?) {
-        when (fragment.type) {
-            typeOf<PageFragment>() -> fragment(fragment.await() as PageFragment)
-            typeOf<DocumentFragment>() -> fragment(
-                fragment.await() as DocumentFragment,
-                Laminate(overrideMeta, data.meta)
-            )
-
-            typeOf<String>() -> fragment(
-                MarkupDocumentFragment(fragment.await() as String, fragment.meta),
-                overrideMeta
-            )
-
-            else -> error("Unsupported data type: ${fragment.type}")
-        }
-    }
-}
-
-public fun SiteContextWithData.document(
-    dataName: Name,
-    descriptor: DocumentDescriptor = DocumentDescriptor.empty(),
-    route: Name = dataName,
-    headers: MetaDataContent.() -> Unit = {},
-    documentBlock: DocumentBuilder.() -> Unit = {},
-): Unit {
-    siteData.branch(dataName)?.filterByType<Binary>()?.forEach {
-        static(route + it.name.last(), it.data)
-    }
-    page(route, descriptor.documentMeta ?: Meta.EMPTY) {
-        //TODO think about avoiding blocking
-        val documentBuilder = runBlocking {
-            PageBasedDocumentBuilder(page, dataName).apply {
-                descriptor.fragments.forEach {
-                    fragment(it)
-                }
-                documentBlock()
-            }
-        }
-        head {
-            title(descriptor.title ?: "Snark document")
-            headers()
-        }
-        body {
-            h1("title") { +(descriptor.title ?: dataName.toString()) }
-            descriptor.authors.forEach {
-                div("author") {
-                    div("author-name") { +it.name }
-                    it.affiliation?.let { affiliation -> div("author-affiliation") { +affiliation } }
-                }
-            }
-            postprocess(FtlDocumentProcessor(this@document.context, documentBuilder)) {
-                documentBuilder.fragments.forEach {
-                    fragment(it)
-                }
-            }
-        }
-    }
-}
-
-public fun SiteContextWithData.allDocuments(
-    headers: MetaDataContent.() -> Unit = {},
-) {
-    siteData.forEach { documentData ->
-        if (documentData.type == typeOf<Meta>() && documentData.name.endsWith("document")) {
-            context.launch {
-                val descriptor = DocumentDescriptor.read(documentData.data.await() as Meta)
-                val directory = documentData.name.cutLast()
-                val route = descriptor.route?.parseAsName(false) ?: directory
-                context.logger.info { "Loading document $route" }
-                document(
-                    dataName = directory,
-                    descriptor = descriptor,
-                    route = route,
-                    headers = headers
-                )
-            }
-        }
-    }
-}

snark-html/src/jvmMain/kotlin/space/kscience/snark/html/document/DocumentDescriptor.kt

@@ -1,28 +0,0 @@
-package space.kscience.snark.html.document
-
-import space.kscience.dataforge.meta.*
-import space.kscience.dataforge.misc.DFExperimental
-
-
-public class Author : Scheme() {
-    public var name: String by string { error("Name is required") }
-    public var affiliation: String? by string()
-
-    public companion object : SchemeSpec<Author>(::Author)
-}
-
-public class DocumentDescriptor : Scheme() {
-
-    public var route: String? by string()
-
-    public var title: String? by string()
-
-    public var documentMeta: Meta? by node()
-
-    public var authors: List<Author> by listOfScheme(Author)
-
-    @OptIn(DFExperimental::class)
-    public var fragments: List<DocumentFragment> by meta.listOfSerializable<DocumentFragment>()
-
-    public companion object : SchemeSpec<DocumentDescriptor>(::DocumentDescriptor)
-}

snark-html/src/jvmMain/kotlin/space/kscience/snark/html/document/DocumentFragment.kt

@@ -1,46 +0,0 @@
-package space.kscience.snark.html.document
-
-import kotlinx.serialization.SerialName
-import kotlinx.serialization.Serializable
-import space.kscience.dataforge.meta.Meta
-import space.kscience.dataforge.names.Name
-
-@Serializable
-public sealed interface DocumentFragment {
-    public val meta: Meta
-}
-
-@Serializable
-@SerialName("markup")
-public class MarkupDocumentFragment(
-    public val text: String,
-    override val meta: Meta = Meta.EMPTY,
-) : DocumentFragment
-
-@Serializable
-@SerialName("image")
-public class ImageDocumentFragment(
-    public val ref: String,
-    override val meta: Meta = Meta.EMPTY,
-) : DocumentFragment
-
-@Serializable
-@SerialName("data")
-public class DataDocumentFragment(
-    public val name: Name,
-    override val meta: Meta = Meta.EMPTY,
-) : DocumentFragment
-
-@Serializable
-@SerialName("list")
-public class ListDocumentFragment(
-    public val fragments: List<DocumentFragment>,
-    override val meta: Meta = Meta.EMPTY,
-) : DocumentFragment
-
-@Serializable
-@SerialName("layout")
-public class LayoutDocumentFragment(
-    public val fragments: Map<String, DocumentFragment>,
-    override val meta: Meta = Meta.EMPTY,
-) : DocumentFragment

snark-html/src/jvmMain/kotlin/space/kscience/snark/html/document/FtlDocumentProcessor.kt

@@ -1,98 +0,0 @@
-package space.kscience.snark.html.document
-
-import freemarker.template.*
-import space.kscience.dataforge.context.Context
-import space.kscience.dataforge.context.ContextAware
-import space.kscience.dataforge.context.error
-import space.kscience.dataforge.context.logger
-import space.kscience.dataforge.meta.int
-import space.kscience.dataforge.meta.string
-import space.kscience.dataforge.meta.toMap
-import space.kscience.dataforge.names.*
-import space.kscience.snark.TextProcessor
-import java.io.StringReader
-import java.io.StringWriter
-
-
-public class FtlDocumentProcessor(
-    override val context: Context,
-    private val document: DocumentBuilder,
-    counters: Map<String, Int> = emptyMap(),
-) : TextProcessor, ContextAware {
-
-    private val counters = counters.toMutableMap()
-
-    private fun getCounter(counter: String): Int = counters[counter] ?: 1
-
-    private fun getAndIncrementCounter(counter: String): Int {
-        val currentCounter = counters[counter] ?: document.documentMeta[NameToken("counter", counter).asName()].int ?: 1
-        counters[counter] = currentCounter + 1
-        return currentCounter
-    }
-
-    private fun resetAndGet(counter: String, value: Int = 1): Int {
-        counters[counter] = value
-        return value
-    }
-
-    private fun ref(counter: String, value: Int): String {
-        //TODO replace by name parse in future
-        val token = Name.parse(counter).last()
-        return when (token.body) {
-            "section" -> {
-                val level = token.index?.toIntOrNull() ?: 1
-                (1..level).joinToString(separator = "_") {
-                    if (it == level) value.toString() else getCounter("section[$it]").toString()
-                }
-            }
-
-            else -> "snark_counter_${counter}_$value"
-        }
-    }
-
-
-    private val ftlConfig = Configuration(Configuration.VERSION_2_3_32).apply {
-        defaultEncoding = "UTF-8"
-        templateExceptionHandler = TemplateExceptionHandler { te: TemplateException, env, out ->
-            logger.error(te) { "An exception while rendering template" }
-        }
-    }
-
-    private val data = mapOf(
-        "documentName" to document.route.toStringUnescaped(),
-
-        "label" to TemplateMethodModelEx { args: List<Any?> ->
-            val counter = args.getOrNull(0)?.toString() ?: "@default"
-            val value = getAndIncrementCounter(counter)
-            val ref = ref(counter, value)
-            //language=HTML
-            """<a class="snark-label" id="$ref" href="#$ref">$value</a>"""
-        },
-
-        "section" to TemplateMethodModelEx { args: List<Any?> ->
-            val level: Int = args.getOrNull(0)?.toString()?.toIntOrNull() ?: 1
-            val counter = getAndIncrementCounter("section[$level]")
-            val ref = ref("section[$level]", counter)
-            //language=HTML
-            """<a class="snark-section snark-label" id="$ref" href = "#$ref">$counter</a>"""
-        },
-
-        "documentMeta" to document.documentMeta.toMap().let {
-            it.plus(
-                "get" to TemplateMethodModelEx { args: List<Any?> ->
-                    val nameString = args.getOrNull(0)?.toString() ?: ""
-                    document.documentMeta[nameString.parseAsName()].string ?: "@null"
-                }
-            )
-        }
-
-    )
-
-
-    override fun process(text: CharSequence): String {
-        val template = Template("fragment", StringReader(text.toString()), ftlConfig)
-        return StringWriter().also {
-            template.process(data, it)
-        }.toString()
-    }
-}

snark-html/src/jvmMain/kotlin/space/kscience/snark/html/document/RegexDocumentProcessor.kt

@@ -1,94 +0,0 @@
-package space.kscience.snark.html.document
-
-import space.kscience.dataforge.meta.int
-import space.kscience.dataforge.meta.string
-import space.kscience.dataforge.names.*
-import space.kscience.snark.TextProcessor
-import java.net.URI
-
-public class RegexDocumentProcessor(public val document: DocumentBuilder) : TextProcessor {
-
-    private val counters = mutableMapOf<String, Int>()
-
-    private fun getCounter(counter: String): Int = counters[counter] ?: 1
-
-    private fun getAndIncrementCounter(counter: String): Int {
-        val currentCounter = counters[counter] ?: document.documentMeta[NameToken("counter", counter).asName()].int ?: 1
-        counters[counter] = currentCounter + 1
-        return currentCounter
-    }
-
-    private fun resetAndGet(counter: String, value: Int = 1): Int {
-        counters[counter] = value
-        return value
-    }
-
-    private fun ref(counter: String, value: Int): String {
-        //TODO replace by name parse in future
-        val token = Name.parse(counter).last()
-        return when (token.body) {
-            "section" -> {
-                val level = token.index?.toIntOrNull() ?: 1
-                (1..level).joinToString(separator = "_") {
-                    if (it == level) value.toString() else getCounter("section[$it]").toString()
-                }
-            }
-
-            else -> "snark_counter_${counter}_$value"
-        }
-    }
-
-    override fun process(text: CharSequence): String = text.replace(functionRegex) { match ->
-        when (match.groups["function"]?.value) {
-
-            "documentName" -> {
-                document.route.toStringUnescaped()
-            }
-
-            "label" -> {
-                val counter = match.groups["arg1"]?.value ?: "@default"
-                val value = getAndIncrementCounter(counter)
-                val ref = ref(counter, value)
-                //language=HTML
-                """<a class="snark-label" id="$ref" href="#$ref">$value</a>"""
-            }
-
-//            "ref" -> {
-//                val target = match.groups["arg1"]?.value
-//                when
-//            }
-
-            "section" -> {
-                val level: Int = match.groups["arg1"]?.value?.toIntOrNull() ?: 1
-                val counter = getAndIncrementCounter("section[$level]")
-                val ref = ref("section[$level]", counter)
-                //language=HTML
-                """<a class="snark-section snark-label" id="$ref" href = "#$ref">$counter</a>"""
-            }
-
-            "documentMeta.get" -> {
-                val nameString = match.groups["arg1"]?.value
-                    ?: error("resolvePageRef requires a string (quoted) argument")
-                document.documentMeta[nameString.parseAsName()].string ?: "@null"
-            }
-
-            else -> match.value
-        }
-    }.replace(attributeRegex) { match ->
-        val uri = URI(match.groups["uri"]!!.value)
-        val snarkUrl = when (uri.authority) {
-            "documentName" -> document.route.toStringUnescaped()
-//            "ref" -> page.resolveRef(uri.path)
-            "meta" -> document.documentMeta[uri.path.parseAsName()].string ?: "@null"
-            else -> match.value
-        }
-        "=\"$snarkUrl\""
-    }
-
-
-    public companion object {
-        internal val functionRegex =
-            """\$\{(?<function>\w*)(?:\((?<arg1>[^(),]*)(\s*,\s*(?<arg2>[^(),]*))?\))?\}""".toRegex()
-        private val attributeRegex = """="(?<uri>snark://([^"]*))"""".toRegex()
-    }
-}

snark-html/src/jvmMain/kotlin/space/kscience/snark/html/static/StaticSiteContext.kt

@@ -1,192 +0,0 @@
-package space.kscience.snark.html.static
-
-import io.ktor.http.Url
-import kotlinx.coroutines.Dispatchers
-import kotlinx.coroutines.runBlocking
-import kotlinx.io.asSink
-import kotlinx.io.buffered
-import space.kscience.dataforge.context.Context
-import space.kscience.dataforge.data.*
-import space.kscience.dataforge.io.Binary
-import space.kscience.dataforge.io.writeBinary
-import space.kscience.dataforge.meta.*
-import space.kscience.dataforge.names.Name
-import space.kscience.dataforge.names.isEmpty
-import space.kscience.dataforge.workspace.FileData
-import space.kscience.snark.html.*
-import java.nio.file.Path
-import kotlin.io.path.*
-import kotlin.reflect.typeOf
-
-
-/**
- * An implementation of [SiteContext] to render site as a static directory [outputPath]
- */
-internal class StaticSiteContext(
-    override val context: Context,
-    override val siteMeta: Meta,
-    private val baseUrl: Url,
-    override val path: List<String>,
-    override val siteRoute: Name,
-    override val parent: SiteContext?,
-    private val outputPath: Path,
-) : SiteContext {
-
-
-//    @OptIn(ExperimentalPathApi::class)
-//    private suspend fun files(item: DataTreeItem<Any>, routeName: Name) {
-//        //try using direct file rendering
-//        item.meta[FileData.FILE_PATH_KEY]?.string?.let {
-//            val file = Path.of(it)
-//            val targetPath = outputPath.resolve(routeName.toWebPath())
-//            targetPath.parent.createDirectories()
-//            file.copyToRecursively(targetPath, followLinks = false)
-//            //success, don't do anything else
-//            return@files
-//        }
-//
-//        when (item) {
-//            is DataTreeItem.Leaf -> {
-//                val datum = item.data
-//                if (datum.type != typeOf<Binary>()) error("Can't directly serve file of type ${item.data.type}")
-//                val targetPath = outputPath.resolve(routeName.toWebPath())
-//                val binary = datum.await() as Binary
-//                targetPath.outputStream().asSink().buffered().use {
-//                    it.writeBinary(binary)
-//                }
-//            }
-//
-//            is DataTreeItem.Node -> {
-//                item.tree.items.forEach { (token, childItem) ->
-//                    files(childItem, routeName + token)
-//                }
-//            }
-//        }
-//    }
-
-    @OptIn(ExperimentalPathApi::class)
-    override fun static(route: Name, data: Data<Binary>) {
-        //if data is a file, copy it
-        data.meta[FileData.FILE_PATH_KEY]?.string?.let {
-            val file = Path.of(it)
-            val targetPath = outputPath.resolve(route.toWebPath())
-            targetPath.parent.createDirectories()
-            file.copyToRecursively(targetPath, followLinks = false)
-            //success, don't do anything else
-            return
-        }
-
-        if (data.type != typeOf<Binary>()) error("Can't directly serve file of type ${data.type}")
-        val targetPath = outputPath.resolve(route.toWebPath())
-        runBlocking(Dispatchers.IO) {
-            val binary = data.await()
-            targetPath.outputStream().asSink().buffered().use {
-                it.writeBinary(binary)
-            }
-        }
-    }
-
-//    private fun resolveRef(baseUrl: String, ref: String) = if (baseUrl.isEmpty()) {
-//        ref
-//    } else if (ref.isEmpty()) {
-//        baseUrl
-//    } else {
-//        "${baseUrl.removeSuffix("/")}/$ref"
-//    }
-
-    class StaticPageContext(
-        override val site: StaticSiteContext,
-        override val host: Url,
-        override val pageRoute: Name,
-        override val pageMeta: Meta,
-    ) : PageContext {
-
-        override fun resolvePageRef(pageName: Name, targetSite: SiteContext): String = resolveRef(
-            pageName.toWebPath() + ".html",
-            targetSite
-        )
-    }
-
-    override fun page(route: Name, data: DataTree<*>?, pageMeta: Meta, content: HtmlPage) {
-
-
-        val modifiedPageMeta = pageMeta.toMutableMeta().apply {
-            "name" put route.toString()
-        }
-
-        val newPath = if (route.isEmpty()) {
-            outputPath.resolve("index.html")
-        } else {
-            outputPath.resolve(route.toWebPath() + ".html")
-        }
-
-        newPath.parent.createDirectories()
-
-        val pageContext = StaticPageContext(this, baseUrl, route, Laminate(modifiedPageMeta, siteMeta))
-        newPath.writeText(HtmlPage.createHtmlString(pageContext, data, content))
-    }
-
-    override fun route(route: Name, data: DataTree<*>?, siteMeta: Meta, content: HtmlSite) {
-        val siteContextWithData = SiteContextWithData(
-            StaticSiteContext(
-                context = context,
-                siteMeta = Laminate(siteMeta, this@StaticSiteContext.siteMeta),
-                baseUrl = baseUrl,
-                path = emptyList(),
-                siteRoute = route,
-                parent = parent,
-                outputPath = outputPath.resolve(route.toWebPath())
-            ),
-            data ?: DataTree.EMPTY
-        )
-        with(content) {
-            with(siteContextWithData) {
-                renderSite()
-            }
-        }
-    }
-
-    override fun site(route: Name, data: DataTree<*>?, siteMeta: Meta, content: HtmlSite) {
-        val siteContextWithData = SiteContextWithData(
-            StaticSiteContext(
-                context = context,
-                siteMeta = Laminate(siteMeta, this@StaticSiteContext.siteMeta),
-                baseUrl = baseUrl,
-                path = emptyList(),
-                siteRoute = route,
-                parent = this,
-                outputPath = outputPath.resolve(route.toWebPath())
-            ),
-            data ?: DataTree.EMPTY
-        )
-        with(content) {
-            with(siteContextWithData) {
-                renderSite()
-            }
-        }
-    }
-
-}
-
-/**
- * Create a static site using given [content] in provided [outputPath].
- * Use [siteUrl] as a base for all resolved URLs. By default, use [outputPath] absolute path as a base.
- *
- */
-public suspend fun SnarkHtml.staticSite(
-    data: DataTree<*>?,
-    outputPath: Path,
-    siteUrl: Url = Url(outputPath.absolutePathString()),
-    siteMeta: Meta = data?.meta ?: Meta.EMPTY,
-    content: HtmlSite,
-) {
-    val siteContextWithData = SiteContextWithData(
-        StaticSiteContext(context, siteMeta, siteUrl, emptyList(), Name.EMPTY, null, outputPath),
-        data ?: DataTree.EMPTY
-    )
-    with(content) {
-        with(siteContextWithData) {
-            renderSite()
-        }
-    }
-}

snark-html/src/main/kotlin/space/kscience/snark/html/DataRenderer.kt

@@ -0,0 +1,48 @@
+package space.kscience.snark.html
+
+import kotlinx.html.body
+import kotlinx.html.head
+import kotlinx.html.title
+import space.kscience.dataforge.data.Data
+import space.kscience.dataforge.meta.*
+import space.kscience.dataforge.names.Name
+import kotlin.reflect.typeOf
+
+/**
+ * Render (or don't) given data piece
+ */
+public interface DataRenderer {
+
+    context(SiteBuilder)
+    public operator fun invoke(name: Name, data: Data<Any>)
+
+    public companion object {
+        public val DEFAULT: DataRenderer = object : DataRenderer {
+
+            context(SiteBuilder)
+            override fun invoke(name: Name, data: Data<Any>) {
+                if (data.type == typeOf<HtmlData>()) {
+                    val languageMeta: Meta = Language.forName(name)
+
+                    val dataMeta: Meta = if (languageMeta.isEmpty()) {
+                        data.meta
+                    } else {
+                        data.meta.toMutableMeta().apply {
+                            "languages" put languageMeta
+                        }
+                    }
+
+                    page(name, dataMeta) {
+                        head {
+                            title = dataMeta["title"].string ?: "Untitled page"
+                        }
+                        body {
+                            @Suppress("UNCHECKED_CAST")
+                            htmlData(data as HtmlData)
+                        }
+                    }
+                }
+            }
+        }
+    }
+}

snark-html/src/main/kotlin/space/kscience/snark/html/HtmlData.kt

@@ -0,0 +1,48 @@
+package space.kscience.snark.html
+
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.runBlocking
+import kotlinx.html.FlowContent
+import kotlinx.html.TagConsumer
+import space.kscience.dataforge.data.Data
+import space.kscience.dataforge.data.await
+import space.kscience.dataforge.meta.get
+import space.kscience.dataforge.meta.int
+import space.kscience.dataforge.meta.string
+import space.kscience.snark.SnarkContext
+
+
+//TODO replace by VisionForge type
+//typealias HtmlFragment = context(PageBuilder, TagConsumer<*>) () -> Unit
+
+public fun interface HtmlFragment {
+    public fun TagConsumer<*>.renderFragment(page: WebPage)
+    //TODO move pageBuilder to a context receiver after KT-52967 is fixed
+}
+
+public typealias HtmlData = Data<HtmlFragment>
+
+//fun HtmlData(meta: Meta, content: context(PageBuilder) TagConsumer<*>.() -> Unit): HtmlData =
+//    Data(HtmlFragment(content), meta)
+
+
+context(WebPage)
+public fun FlowContent.htmlData(data: HtmlData): Unit = runBlocking(Dispatchers.IO) {
+    with(data.await()) { consumer.renderFragment(page) }
+}
+
+context(SnarkContext)
+public val Data<*>.id: String
+    get() = meta["id"]?.string ?: "block[${hashCode()}]"
+
+context(SnarkContext)
+public val Data<*>.language: String?
+    get() = meta["language"].string?.lowercase()
+
+context(SnarkContext)
+public val Data<*>.order: Int?
+    get() = meta["order"]?.int
+
+context(SnarkContext)
+public val Data<*>.published: Boolean
+    get() = meta["published"].string != "false"

snark-html/src/main/kotlin/space/kscience/snark/html/Language.kt

@@ -0,0 +1,147 @@
+package space.kscience.snark.html
+
+import space.kscience.dataforge.data.getItem
+import space.kscience.dataforge.meta.*
+import space.kscience.dataforge.names.*
+import space.kscience.snark.html.Language.Companion.SITE_LANGUAGES_KEY
+import space.kscience.snark.html.Language.Companion.SITE_LANGUAGE_KEY
+
+
+
+public class Language : Scheme() {
+    /**
+     * Language key override
+     */
+    public var key: String? by string()
+
+    /**
+     * Page name prefix
+     */
+    public var prefix: String? by string()
+
+    /**
+     * Target page name with a given language key
+     */
+    public var target: Name?
+        get() = meta["target"].string?.parseAsName(false)
+        set(value) {
+            meta["target"] = value?.toString()?.asValue()
+        }
+
+    public companion object : SchemeSpec<Language>(::Language) {
+
+        public val LANGUAGE_KEY: Name = "language".asName()
+
+        public val LANGUAGES_KEY: Name = "languages".asName()
+
+        public val SITE_LANGUAGE_KEY: Name = SiteBuilder.SITE_META_KEY + LANGUAGE_KEY
+
+        public val SITE_LANGUAGES_KEY: Name = SiteBuilder.SITE_META_KEY + LANGUAGES_KEY
+
+        public const val DEFAULT_LANGUAGE: String = "en"
+
+        /**
+         * Automatically build a language map for a data piece with given [name] based on existence of appropriate data nodes.
+         */
+        context(SiteBuilder)
+        public fun forName(name: Name): Meta = Meta {
+            val currentLanguagePrefix = languages[language]?.get(Language::prefix.name)?.string ?: language
+            val fullName = (route.removeHeadOrNull(currentLanguagePrefix.asName()) ?: route) + name
+            languages.forEach { (key, meta) ->
+                val languagePrefix: String = meta[Language::prefix.name].string ?: key
+                val nameWithLanguage: Name = if (languagePrefix.isBlank()) {
+                    fullName
+                } else {
+                    languagePrefix.asName() + fullName
+                }
+                if (data.getItem(name) != null) {
+                    key put meta.asMutableMeta().apply {
+                        Language::target.name put nameWithLanguage.toString()
+                    }
+                }
+            }
+        }
+    }
+}
+
+public val SiteBuilder.languages: Map<String, Meta>
+    get() = siteMeta[SITE_LANGUAGES_KEY]?.items?.mapKeys { it.key.toStringUnescaped() } ?: emptyMap()
+
+public val SiteBuilder.language: String
+    get() = siteMeta[SITE_LANGUAGE_KEY].string ?: Language.DEFAULT_LANGUAGE
+
+public val SiteBuilder.languagePrefix: Name
+    get() = languages[language]?.let { it[Language::prefix.name].string ?: language }?.parseAsName() ?: Name.EMPTY
+
+public fun SiteBuilder.withLanguages(languageMap: Map<String, Meta>, block: SiteBuilder.(language: String) -> Unit) {
+    languageMap.forEach { (languageKey, languageMeta) ->
+        val prefix = languageMeta[Language::prefix.name].string ?: languageKey
+        val routeMeta = Meta {
+            SITE_LANGUAGE_KEY put languageKey
+            SITE_LANGUAGES_KEY put Meta {
+                languageMap.forEach {
+                    it.key put it.value
+                }
+            }
+        }
+        route(prefix, routeMeta = routeMeta) {
+            block(languageKey)
+        }
+    }
+}
+
+public fun SiteBuilder.withLanguages(
+    vararg language: Pair<String, String>,
+    block: SiteBuilder.(language: String) -> Unit,
+) {
+    val languageMap = language.associate {
+        it.first to Meta {
+            Language::prefix.name put it.second
+        }
+    }
+    withLanguages(languageMap, block)
+}
+
+
+
+/**
+ * The language key of this page
+ */
+public val WebPage.language: String
+    get() = pageMeta[Language.LANGUAGE_KEY]?.string ?: pageMeta[SITE_LANGUAGE_KEY]?.string ?: Language.DEFAULT_LANGUAGE
+
+/**
+ * Mapping of language keys to other language versions of this page
+ */
+public val WebPage.languages: Map<String, Meta>
+    get() = pageMeta[Language.LANGUAGES_KEY]?.items?.mapKeys { it.key.toStringUnescaped() } ?: emptyMap()
+
+public fun WebPage.localisedPageRef(pageName: Name, relative: Boolean = false): String {
+    val prefix = languages[language]?.get(Language::prefix.name)?.string?.parseAsName() ?: Name.EMPTY
+    return resolvePageRef(prefix + pageName, relative)
+}
+
+
+/**
+ * Render all pages in a node with given name. Use localization prefix if appropriate data is available.
+ */
+public fun SiteBuilder.localizedPages(
+    dataPath: Name,
+    remotePath: Name = dataPath,
+    dataRenderer: DataRenderer = DataRenderer.DEFAULT,
+) {
+    val item = data.getItem(languagePrefix + dataPath)
+        ?: data.getItem(dataPath)
+        ?: error("No data found by name $dataPath")
+    route(remotePath) {
+        pages(item, dataRenderer)
+    }
+}
+
+public fun SiteBuilder.localizedPages(
+    dataPath: String,
+    remotePath: Name = dataPath.parseAsName(),
+    dataRenderer: DataRenderer = DataRenderer.DEFAULT,
+) {
+    localizedPages(dataPath.parseAsName(), remotePath, dataRenderer = dataRenderer)
+}

snark-html/src/main/kotlin/space/kscience/snark/html/SiteBuilder.kt

@@ -0,0 +1,246 @@
+package space.kscience.snark.html
+
+import kotlinx.html.HTML
+import space.kscience.dataforge.context.Context
+import space.kscience.dataforge.context.ContextAware
+import space.kscience.dataforge.data.DataTree
+import space.kscience.dataforge.data.DataTreeItem
+import space.kscience.dataforge.data.getItem
+import space.kscience.dataforge.meta.Meta
+import space.kscience.dataforge.meta.get
+import space.kscience.dataforge.meta.getIndexed
+import space.kscience.dataforge.meta.string
+import space.kscience.dataforge.names.Name
+import space.kscience.dataforge.names.NameToken
+import space.kscience.dataforge.names.asName
+import space.kscience.dataforge.names.parseAsName
+import space.kscience.snark.SnarkContext
+import space.kscience.snark.html.SiteLayout.Companion.LAYOUT_KEY
+import java.nio.file.Path
+
+
+/**
+ * An abstraction, which is used to render sites to the different rendering engines
+ */
+public interface SiteBuilder : ContextAware, SnarkContext {
+
+    /**
+     * Route name of this [SiteBuilder] relative to the site root
+     */
+    public val route: Name
+
+    /**
+     * Data used for site construction. The type of the data is not limited
+     */
+    public val data: DataTree<*>
+
+    /**
+     * Snark plugin and context used for layout resolution, preprocessors, etc
+     */
+    public val snark: SnarkHtmlPlugin
+
+    override val context: Context get() = snark.context
+
+    /**
+     * Site configuration
+     */
+    public val siteMeta: Meta
+
+    /**
+     * Add a static file or directory to this site/route at [remotePath]
+     */
+    public fun file(file: Path, remotePath: String = file.fileName.toString())
+
+    /**
+     * Add a static file (single) from resources
+     */
+    public fun resourceFile(remotePath: String, resourcesPath: String)
+
+    /**
+     * Add a resource directory to route
+     */
+    public fun resourceDirectory(resourcesPath: String)
+
+    /**
+     * Create a single page at given [route]. If route is empty, create an index page at current route.
+     *
+     * @param pageMeta additional page meta. [WebPage] will use both it and [siteMeta]
+     */
+    public fun page(route: Name = Name.EMPTY, pageMeta: Meta = Meta.EMPTY, content: context(WebPage, HTML) () -> Unit)
+
+    /**
+     * Create a route with optional data tree override. For example one could use a subtree of the initial tree.
+     * By default, the same data tree is used for route.
+     */
+    public fun route(
+        routeName: Name,
+        dataOverride: DataTree<*>? = null,
+        routeMeta: Meta = Meta.EMPTY,
+    ): SiteBuilder
+
+    /**
+     * Creates a route and sets it as site base url
+     */
+    public fun site(
+        routeName: Name,
+        dataOverride: DataTree<*>? = null,
+        routeMeta: Meta = Meta.EMPTY,
+    ): SiteBuilder
+
+
+    public companion object {
+        public val SITE_META_KEY: Name = "site".asName()
+        public val INDEX_PAGE_TOKEN: NameToken = NameToken("index")
+        public val UP_PAGE_TOKEN: NameToken = NameToken("..")
+    }
+}
+
+context(SiteBuilder)
+public val siteBuilder: SiteBuilder
+    get() = this@SiteBuilder
+
+public inline fun SiteBuilder.route(
+    route: Name,
+    dataOverride: DataTree<*>? = null,
+    routeMeta: Meta = Meta.EMPTY,
+    block: SiteBuilder.() -> Unit,
+) {
+    route(route, dataOverride, routeMeta).apply(block)
+}
+
+public inline fun SiteBuilder.route(
+    route: String,
+    dataOverride: DataTree<*>? = null,
+    routeMeta: Meta = Meta.EMPTY,
+    block: SiteBuilder.() -> Unit,
+) {
+    route(route.parseAsName(), dataOverride, routeMeta).apply(block)
+}
+
+public inline fun SiteBuilder.site(
+    route: Name,
+    dataOverride: DataTree<*>? = null,
+    routeMeta: Meta = Meta.EMPTY,
+    block: SiteBuilder.() -> Unit,
+) {
+    site(route, dataOverride, routeMeta).apply(block)
+}
+
+public inline fun SiteBuilder.site(
+    route: String,
+    dataOverride: DataTree<*>? = null,
+    routeMeta: Meta = Meta.EMPTY,
+    block: SiteBuilder.() -> Unit,
+) {
+    site(route.parseAsName(), dataOverride, routeMeta).apply(block)
+}
+
+
+///**
+// * Create a stand-alone site at a given node
+// */
+//public fun SiteBuilder.site(route: Name, dataRoot: DataTree<*>, block: SiteBuilder.() -> Unit) {
+//    val mountedData = data.copy(
+//        data = dataRoot,
+//        baseUrlPath = data.resolveRef(route.tokens.joinToString(separator = "/")),
+//        meta = Laminate(dataRoot.meta, data.meta) //layering dataRoot meta over existing data
+//    )
+//    route(route) {
+//        withData(mountedData).block()
+//    }
+//}
+
+
+
+internal fun SiteBuilder.assetsFrom(rootMeta: Meta) {
+    rootMeta.getIndexed("resource".asName()).forEach { (_, meta) ->
+
+        val path by meta.string()
+        val remotePath by meta.string()
+
+        path?.let { resourcePath ->
+            //If remote path provided, use a single resource
+            remotePath?.let {
+                resourceFile(it, resourcePath)
+                return@forEach
+            }
+
+            //otherwise use package resources
+            resourceDirectory(resourcePath)
+        }
+    }
+
+    rootMeta.getIndexed("file".asName()).forEach { (_, meta) ->
+        val remotePath by meta.string { error("File remote path is not provided") }
+        val path by meta.string { error("File path is not provided") }
+        file(Path.of(path), remotePath)
+    }
+
+    rootMeta.getIndexed("directory".asName()).forEach { (_, meta) ->
+        val path by meta.string { error("Directory path is not provided") }
+        file(Path.of(path), "")
+    }
+}
+
+
+/**
+ * Recursively renders the data items in [data]. If [LAYOUT_KEY] is defined in an item, use it to load
+ * layout from the context, otherwise render children nodes as name segments and individual data items using [dataRenderer].
+ */
+public fun SiteBuilder.pages(
+    data: DataTreeItem<*>,
+    dataRenderer: DataRenderer = DataRenderer.DEFAULT,
+) {
+    val layoutMeta = data.meta[LAYOUT_KEY]
+    if (layoutMeta != null) {
+        //use layout if it is defined
+        snark.siteLayout(layoutMeta).render(data)
+    } else {
+        when (data) {
+            is DataTreeItem.Node -> {
+                data.tree.items.forEach { (token, item) ->
+                    //Don't apply index token
+                    if (token == SiteLayout.INDEX_PAGE_TOKEN) {
+                        pages(item, dataRenderer)
+                    } else if (item is DataTreeItem.Leaf) {
+                        dataRenderer(token.asName(), item.data)
+                    } else {
+                        route(token.asName()) {
+                            pages(item, dataRenderer)
+                        }
+                    }
+                }
+            }
+
+            is DataTreeItem.Leaf -> {
+                dataRenderer(Name.EMPTY, data.data)
+            }
+        }
+        data.meta[SiteLayout.ASSETS_KEY]?.let {
+            assetsFrom(it)
+        }
+    }
+    //TODO watch for changes
+}
+
+/**
+ * Render all pages in a node with given name
+ */
+public fun SiteBuilder.pages(
+    dataPath: Name,
+    remotePath: Name = dataPath,
+    dataRenderer: DataRenderer = DataRenderer.DEFAULT,
+) {
+    val item = data.getItem(dataPath) ?: error("No data found by name $dataPath")
+    route(remotePath) {
+        pages(item, dataRenderer)
+    }
+}
+
+public fun SiteBuilder.pages(
+    dataPath: String,
+    remotePath: Name = dataPath.parseAsName(),
+    dataRenderer: DataRenderer = DataRenderer.DEFAULT,
+) {
+    pages(dataPath.parseAsName(), remotePath, dataRenderer = dataRenderer)
+}

snark-html/src/main/kotlin/space/kscience/snark/html/SiteLayout.kt

@@ -0,0 +1,32 @@
+package space.kscience.snark.html
+
+import space.kscience.dataforge.data.DataTreeItem
+import space.kscience.dataforge.misc.Type
+import space.kscience.dataforge.names.NameToken
+
+/**
+ * An abstraction to render singular data or a data tree.
+ */
+@Type(SiteLayout.TYPE)
+public fun interface SiteLayout {
+
+    context(SiteBuilder)
+    public fun render(item: DataTreeItem<*>)
+
+    public companion object {
+        public const val TYPE: String = "snark.layout"
+        public const val LAYOUT_KEY: String = "layout"
+        public const val ASSETS_KEY: String = "assets"
+        public val INDEX_PAGE_TOKEN: NameToken = NameToken("index")
+    }
+}
+
+
+/**
+ * The default [SiteLayout]. It renders all [HtmlData] pages with simple headers via [SiteLayout.defaultDataRenderer]
+ */
+public object DefaultSiteLayout : SiteLayout {
+    context(SiteBuilder) override fun render(item: DataTreeItem<*>) {
+        pages(item)
+    }
+}

snark-html/src/main/kotlin/space/kscience/snark/html/SnarkHtmlPlugin.kt

@@ -0,0 +1,116 @@
+package space.kscience.snark.html
+
+import io.ktor.utils.io.core.readBytes
+import space.kscience.dataforge.context.*
+import space.kscience.dataforge.data.DataTree
+import space.kscience.dataforge.io.IOPlugin
+import space.kscience.dataforge.io.IOReader
+import space.kscience.dataforge.io.JsonMetaFormat
+import space.kscience.dataforge.io.yaml.YamlMetaFormat
+import space.kscience.dataforge.io.yaml.YamlPlugin
+import space.kscience.dataforge.meta.Meta
+import space.kscience.dataforge.meta.get
+import space.kscience.dataforge.meta.string
+import space.kscience.dataforge.misc.DFExperimental
+import space.kscience.dataforge.names.Name
+import space.kscience.dataforge.names.asName
+import space.kscience.dataforge.names.parseAsName
+import space.kscience.dataforge.workspace.FileData
+import space.kscience.dataforge.workspace.readDataDirectory
+import space.kscience.snark.SnarkEnvironment
+import space.kscience.snark.SnarkParser
+import java.nio.file.Path
+import kotlin.io.path.extension
+import kotlin.reflect.KClass
+
+/**
+ * A plugin used for rendering a [DataTree] as HTML
+ */
+public class SnarkHtmlPlugin : AbstractPlugin() {
+    private val yaml by require(YamlPlugin)
+    public val io: IOPlugin get() = yaml.io
+
+    override val tag: PluginTag get() = Companion.tag
+
+    internal val parsers: Map<Name, SnarkParser<Any>> by lazy {
+        context.gather(SnarkParser.TYPE, true)
+    }
+
+    private val siteLayouts: Map<Name, SiteLayout> by lazy {
+        context.gather(SiteLayout.TYPE, true)
+    }
+
+    private val textProcessors: Map<Name, TextProcessor> by lazy {
+        context.gather(TextProcessor.TYPE, true)
+    }
+
+    internal fun siteLayout(layoutMeta: Meta): SiteLayout {
+        val layoutName = layoutMeta.string
+            ?: layoutMeta["name"].string ?: error("Layout name not defined in $layoutMeta")
+        return siteLayouts[layoutName.parseAsName()] ?: error("Layout with name $layoutName not found in $this")
+    }
+
+    internal fun textProcessor(transformationMeta: Meta): TextProcessor {
+        val transformationName = transformationMeta.string
+            ?: transformationMeta["name"].string ?: error("Transformation name not defined in $transformationMeta")
+        return textProcessors[transformationName.parseAsName()]
+            ?: error("Text transformation with name $transformationName not found in $this")
+    }
+
+    override fun content(target: String): Map<Name, Any> = when (target) {
+        SnarkParser.TYPE -> mapOf(
+            "html".asName() to SnarkHtmlParser,
+            "markdown".asName() to SnarkMarkdownParser,
+            "json".asName() to SnarkParser(JsonMetaFormat, "json"),
+            "yaml".asName() to SnarkParser(YamlMetaFormat, "yaml", "yml"),
+            "png".asName() to SnarkParser(ImageIOReader, "png"),
+            "jpg".asName() to SnarkParser(ImageIOReader, "jpg", "jpeg"),
+            "gif".asName() to SnarkParser(ImageIOReader, "gif"),
+        )
+        TextProcessor.TYPE -> mapOf(
+            "basic".asName() to BasicTextProcessor
+        )
+        else -> super.content(target)
+    }
+
+    public companion object : PluginFactory<SnarkHtmlPlugin> {
+        override val tag: PluginTag = PluginTag("snark")
+        override val type: KClass<out SnarkHtmlPlugin> = SnarkHtmlPlugin::class
+
+        override fun build(context: Context, meta: Meta): SnarkHtmlPlugin = SnarkHtmlPlugin()
+
+        private val byteArrayIOReader = IOReader {
+            readBytes()
+        }
+
+        internal val byteArraySnarkParser = SnarkParser(byteArrayIOReader)
+    }
+}
+
+/**
+ * Load necessary dependencies and return a [SnarkHtmlPlugin] in a finalized context
+ */
+public fun SnarkEnvironment.buildHtmlPlugin(): SnarkHtmlPlugin {
+    val context = parentContext.buildContext("snark".asName()) {
+        plugin(SnarkHtmlPlugin)
+        plugins.forEach {
+            plugin(it)
+        }
+    }
+    return context.fetch(SnarkHtmlPlugin)
+}
+
+@OptIn(DFExperimental::class)
+public fun SnarkHtmlPlugin.readDirectory(path: Path): DataTree<Any> = io.readDataDirectory(path) { dataPath, meta ->
+    val fileExtension = meta[FileData.META_FILE_EXTENSION_KEY].string ?: dataPath.extension
+    val parser: SnarkParser<Any> = parsers.values.filter { parser ->
+        fileExtension in parser.fileExtensions
+    }.maxByOrNull {
+        it.priority
+    } ?: run {
+        logger.warn { "The parser is not found for file $dataPath with meta $meta" }
+        SnarkHtmlPlugin.byteArraySnarkParser
+    }
+
+    parser.reader(context, meta)
+}

snark-html/src/main/kotlin/space/kscience/snark/html/SnarkTextParser.kt

@@ -0,0 +1,68 @@
+package space.kscience.snark.html
+
+import io.ktor.util.asStream
+import io.ktor.utils.io.core.Input
+import kotlinx.html.div
+import kotlinx.html.unsafe
+import org.intellij.markdown.flavours.commonmark.CommonMarkFlavourDescriptor
+import org.intellij.markdown.html.HtmlGenerator
+import org.intellij.markdown.parser.MarkdownParser
+import space.kscience.dataforge.context.Context
+import space.kscience.dataforge.io.IOReader
+import space.kscience.dataforge.meta.Meta
+import space.kscience.dataforge.meta.get
+import space.kscience.snark.SnarkParser
+import java.awt.image.BufferedImage
+import javax.imageio.ImageIO
+import kotlin.reflect.KType
+import kotlin.reflect.typeOf
+
+public abstract class SnarkTextParser<R> : SnarkParser<R> {
+    public abstract fun parseText(text: String, meta: Meta): R
+
+    override fun parse(context: Context, meta: Meta, bytes: ByteArray): R =
+        parseText(bytes.decodeToString(), meta)
+
+    public fun transformText(text: String, meta: Meta, page: WebPage): String =
+        meta[TextProcessor.TEXT_TRANSFORMATION_KEY]?.let {
+            with(page) { page.snark.textProcessor(it).process(text) }
+        } ?: text
+}
+
+
+internal object SnarkHtmlParser : SnarkTextParser<HtmlFragment>() {
+    override val fileExtensions: Set<String> = setOf("html")
+    override val type: KType = typeOf<HtmlFragment>()
+
+    override fun parseText(text: String, meta: Meta): HtmlFragment = HtmlFragment { page ->
+        div {
+            unsafe { +transformText(text, meta, page) }
+        }
+    }
+}
+
+internal object SnarkMarkdownParser : SnarkTextParser<HtmlFragment>() {
+    override val fileExtensions: Set<String> = setOf("markdown", "mdown", "mkdn", "mkd", "md")
+    override val type: KType = typeOf<HtmlFragment>()
+
+    private val markdownFlavor = CommonMarkFlavourDescriptor()
+    private val markdownParser = MarkdownParser(markdownFlavor)
+
+    override fun parseText(text: String, meta: Meta): HtmlFragment = HtmlFragment { page ->
+        val transformedText = SnarkHtmlParser.transformText(text, meta, page)
+        val parsedTree = markdownParser.buildMarkdownTreeFromString(transformedText)
+        val htmlString = HtmlGenerator(transformedText, parsedTree, markdownFlavor).generateHtml()
+
+        div {
+            unsafe {
+                +htmlString
+            }
+        }
+    }
+}
+
+internal object ImageIOReader : IOReader<BufferedImage> {
+    override val type: KType get() = typeOf<BufferedImage>()
+
+    override fun readObject(input: Input): BufferedImage = ImageIO.read(input.asStream())
+}

snark-html/src/main/kotlin/space/kscience/snark/html/StaticSiteBuilder.kt

@@ -0,0 +1,151 @@
+package space.kscience.snark.html
+
+import kotlinx.html.HTML
+import kotlinx.html.html
+import kotlinx.html.stream.createHTML
+import space.kscience.dataforge.data.DataTree
+import space.kscience.dataforge.meta.Laminate
+import space.kscience.dataforge.meta.Meta
+import space.kscience.dataforge.meta.toMutableMeta
+import space.kscience.dataforge.names.Name
+import space.kscience.dataforge.names.isEmpty
+import space.kscience.dataforge.names.plus
+import space.kscience.snark.SnarkEnvironment
+import java.nio.file.Files
+import java.nio.file.Path
+import kotlin.contracts.InvocationKind
+import kotlin.contracts.contract
+import kotlin.io.path.*
+
+
+/**
+ * An implementation of [SiteBuilder] to render site as a static directory [outputPath]
+ */
+internal class StaticSiteBuilder(
+    override val snark: SnarkHtmlPlugin,
+    override val data: DataTree<*>,
+    override val siteMeta: Meta,
+    private val baseUrl: String,
+    override val route: Name,
+    private val outputPath: Path,
+) : SiteBuilder {
+    private fun Path.copyRecursively(target: Path) {
+        Files.walk(this).forEach { source: Path ->
+            val destination: Path = target.resolve(source.relativeTo(this))
+            if (!destination.isDirectory()) {
+                //avoid re-creating directories
+                source.copyTo(destination, true)
+            }
+        }
+    }
+
+    override fun file(file: Path, remotePath: String) {
+        val targetPath = outputPath.resolve(remotePath)
+        if (file.isDirectory()) {
+            targetPath.parent.createDirectories()
+            file.copyRecursively(targetPath)
+        } else if (remotePath.isBlank()) {
+            error("Can't mount file to an empty route")
+        } else {
+            targetPath.parent.createDirectories()
+            file.copyTo(targetPath, true)
+        }
+    }
+
+    override fun resourceFile(remotePath: String, resourcesPath: String) {
+        val targetPath = outputPath.resolve(remotePath)
+        targetPath.parent.createDirectories()
+        javaClass.getResource(resourcesPath)?.let { Path.of(it.toURI()) }?.copyTo(targetPath, true)
+    }
+
+    override fun resourceDirectory(resourcesPath: String) {
+        outputPath.parent.createDirectories()
+        javaClass.getResource(resourcesPath)?.let { Path.of(it.toURI()) }?.copyRecursively(outputPath)
+    }
+
+    private fun resolveRef(baseUrl: String, ref: String) = if (baseUrl.isEmpty()) {
+        ref
+    } else if (ref.isEmpty()) {
+        baseUrl
+    } else {
+        "${baseUrl.removeSuffix("/")}/$ref"
+    }
+
+    inner class StaticWebPage(override val pageMeta: Meta) : WebPage {
+        override val data: DataTree<*> get() = this@StaticSiteBuilder.data
+
+        override val snark: SnarkHtmlPlugin get() = this@StaticSiteBuilder.snark
+
+
+        override fun resolveRef(ref: String): String = resolveRef(baseUrl, ref)
+
+        override fun resolvePageRef(pageName: Name, relative: Boolean): String = resolveRef(
+            (if (relative) route + pageName else pageName).toWebPath() + ".html"
+        )
+    }
+
+
+    override fun page(route: Name, pageMeta: Meta, content: context(WebPage, HTML) () -> Unit) {
+        val htmlBuilder = createHTML()
+
+        val modifiedPageMeta = pageMeta.toMutableMeta().apply {
+            "name" put route.toString()
+        }
+
+        htmlBuilder.html {
+            content(StaticWebPage(Laminate(modifiedPageMeta, siteMeta)), this)
+        }
+
+        val newPath = if (route.isEmpty()) {
+            outputPath.resolve("index.html")
+        } else {
+            outputPath.resolve(route.toWebPath() + ".html")
+        }
+
+        newPath.parent.createDirectories()
+        newPath.writeText(htmlBuilder.finalize())
+    }
+
+    override fun route(
+        routeName: Name,
+        dataOverride: DataTree<*>?,
+        routeMeta: Meta,
+    ): SiteBuilder = StaticSiteBuilder(
+        snark = snark,
+        data = dataOverride ?: data,
+        siteMeta = Laminate(routeMeta, siteMeta),
+        baseUrl = baseUrl,
+        route = route + routeName,
+        outputPath = outputPath.resolve(routeName.toWebPath())
+    )
+
+    override fun site(
+        routeName: Name,
+        dataOverride: DataTree<*>?,
+        routeMeta: Meta,
+    ): SiteBuilder = StaticSiteBuilder(
+        snark = snark,
+        data = dataOverride ?: data,
+        siteMeta = Laminate(routeMeta, siteMeta),
+        baseUrl = resolveRef(baseUrl, routeName.toWebPath()),
+        route = Name.EMPTY,
+        outputPath = outputPath.resolve(routeName.toWebPath())
+    )
+}
+
+/**
+ * Create a static site using given [SnarkEnvironment] in provided [outputPath].
+ * Use [siteUrl] as a base for all resolved URLs. By default, use [outputPath] absolute path as a base.
+ *
+ */
+public fun SnarkEnvironment.static(
+    outputPath: Path,
+    siteUrl: String = outputPath.absolutePathString().replace("\\", "/"),
+    block: SiteBuilder.() -> Unit,
+) {
+    contract {
+        callsInPlace(block, InvocationKind.EXACTLY_ONCE)
+    }
+    val plugin = buildHtmlPlugin()
+    StaticSiteBuilder(plugin, data, meta, siteUrl, Name.EMPTY, outputPath).block()
+}

snark-html/src/main/kotlin/space/kscience/snark/html/TextProcessor.kt

@@ -0,0 +1,61 @@
+package space.kscience.snark.html
+
+import space.kscience.dataforge.meta.get
+import space.kscience.dataforge.meta.string
+import space.kscience.dataforge.misc.Type
+import space.kscience.dataforge.names.NameToken
+import space.kscience.dataforge.names.parseAsName
+
+/**
+ * An object that conducts page-based text transformation. Like using link replacement or templating.
+ */
+@Type(TextProcessor.TYPE)
+public fun interface TextProcessor {
+    context(WebPage)
+    public fun process(text: String): String
+
+    public companion object {
+        public const val TYPE: String = "snark.textTransformation"
+        public val TEXT_TRANSFORMATION_KEY: NameToken = NameToken("transformation")
+    }
+}
+
+/**
+ * A basic [TextProcessor] that replaces `${...}` expressions in text. The following expressions are recognised:
+ * * `homeRef` resolves to [homeRef]
+ * * `resolveRef("...")` -> [WebPage.resolveRef]
+ * * `resolvePageRef("...")` -> [WebPage.resolvePageRef]
+ * * `pageMeta.get("...") -> [WebPage.pageMeta] get string method
+ * Otherwise return unchanged string
+ */
+public object BasicTextProcessor : TextProcessor {
+
+    private val regex = """\$\{([\w.]*)(?>\("(.*)"\))?}""".toRegex()
+
+    context(WebPage)
+    override fun process(text: String): String = text.replace(regex) { match ->
+        when (match.groups[1]!!.value) {
+            "homeRef" -> homeRef
+            "resolveRef" -> {
+                val refString = match.groups[2]?.value ?: error("resolveRef requires a string (quoted) argument")
+                resolveRef(refString)
+            }
+
+            "resolvePageRef" -> {
+                val refString = match.groups[2]?.value
+                    ?: error("resolvePageRef requires a string (quoted) argument")
+                localisedPageRef(refString.parseAsName())
+            }
+
+            "pageMeta.get" -> {
+                val nameString = match.groups[2]?.value
+                    ?: error("resolvePageRef requires a string (quoted) argument")
+                pageMeta[nameString.parseAsName()].string ?: "@null"
+            }
+
+            else -> match.value
+        }
+    }
+}
+
+

snark-html/src/main/kotlin/space/kscience/snark/html/WebPage.kt

@@ -0,0 +1,94 @@
+package space.kscience.snark.html
+
+import space.kscience.dataforge.context.Context
+import space.kscience.dataforge.context.ContextAware
+import space.kscience.dataforge.data.*
+import space.kscience.dataforge.meta.Meta
+import space.kscience.dataforge.meta.get
+import space.kscience.dataforge.meta.string
+import space.kscience.dataforge.names.*
+import space.kscience.snark.SnarkContext
+
+context(SnarkContext)
+public fun Name.toWebPath(): String = tokens.joinToString(separator = "/") {
+    if (it.hasIndex()) {
+        "${it.body}[${it.index}]"
+    } else {
+        it.body
+    }
+}
+
+/**
+ *  A context for building a single page
+ */
+public interface WebPage : ContextAware, SnarkContext {
+
+    public val snark: SnarkHtmlPlugin
+
+    override val context: Context get() = snark.context
+
+    public val data: DataTree<*>
+
+    /**
+     * A metadata for a page. It should include site metadata
+     */
+    public val pageMeta: Meta
+
+    /**
+     * Resolve absolute url for given [ref]
+     *
+     */
+    public fun resolveRef(ref: String): String
+
+    /**
+     * Resolve absolute url for a page with given [pageName].
+     *
+     * @param relative if true, add [SiteBuilder] route to the absolute page name
+     */
+    public fun resolvePageRef(pageName: Name, relative: Boolean = false): String
+}
+
+context(WebPage)
+public val page: WebPage
+    get() = this@WebPage
+
+public fun WebPage.resolvePageRef(pageName: String): String = resolvePageRef(pageName.parseAsName())
+
+public val WebPage.homeRef: String get() = resolvePageRef(SiteBuilder.INDEX_PAGE_TOKEN.asName())
+
+public val WebPage.name: Name? get() = pageMeta["name"].string?.parseAsName()
+
+/**
+ * Resolve a Html builder by its full name
+ */
+context(SnarkContext)
+public fun DataTree<*>.resolveHtml(name: Name): HtmlData? {
+    val resolved = (getByType<HtmlFragment>(name) ?: getByType<HtmlFragment>(name + SiteBuilder.INDEX_PAGE_TOKEN))
+
+    return resolved?.takeIf {
+        it.published //TODO add language confirmation
+    }
+}
+
+context(SnarkContext)
+public fun DataTree<*>.resolveHtml(name: String): HtmlData? = resolveHtml(name.parseAsName())
+
+/**
+ * Find all Html blocks using given name/meta filter
+ */
+context(SnarkContext)
+public fun DataTree<*>.resolveAllHtml(predicate: (name: Name, meta: Meta) -> Boolean): Map<Name, HtmlData> =
+    filterByType<HtmlFragment> { name, meta ->
+        predicate(name, meta)
+                && meta["published"].string != "false"
+        //TODO add language confirmation
+    }.asSequence().associate { it.name to it.data }
+
+
+context(SnarkContext)
+public fun DataTree<*>.findByContentType(
+    contentType: String,
+    baseName: Name = Name.EMPTY,
+): Map<Name, Data<HtmlFragment>> = resolveAllHtml { name, meta ->
+    name.startsWith(baseName) && meta["content_type"].string == contentType
+}

snark-html/src/main/kotlin/space/kscience/snark/html/htmlEnvironment.kt

@@ -0,0 +1,42 @@
+package space.kscience.snark.html
+
+import space.kscience.dataforge.context.AbstractPlugin
+import space.kscience.dataforge.context.PluginTag
+import space.kscience.dataforge.data.DataTreeItem
+import space.kscience.dataforge.names.Name
+import space.kscience.dataforge.names.parseAsName
+import space.kscience.snark.SnarkEnvironment
+import kotlin.contracts.InvocationKind
+import kotlin.contracts.contract
+
+public class SnarkHtmlEnvironmentBuilder {
+    public val layouts: HashMap<Name, SiteLayout> = HashMap()
+
+    public fun layout(name: String, body: context(SiteBuilder) (DataTreeItem<*>) -> Unit) {
+        layouts[name.parseAsName()] = object : SiteLayout {
+            context(SiteBuilder) override fun render(item: DataTreeItem<*>) = body(siteBuilder, item)
+        }
+    }
+}
+
+
+public fun SnarkEnvironment.html(block: SnarkHtmlEnvironmentBuilder.() -> Unit) {
+    contract {
+        callsInPlace(block, InvocationKind.EXACTLY_ONCE)
+    }
+
+    val envBuilder = SnarkHtmlEnvironmentBuilder().apply(block)
+
+    val plugin = object : AbstractPlugin() {
+        val snark by require(SnarkHtmlPlugin)
+
+        override val tag: PluginTag = PluginTag("@extension[${hashCode()}]")
+
+
+        override fun content(target: String): Map<Name, Any> = when (target) {
+            SiteLayout.TYPE -> envBuilder.layouts
+            else -> super.content(target)
+        }
+    }
+    registerPlugin(plugin)
+}

snark-ktor/build.gradle.kts

@@ -1,23 +1,21 @@
 plugins {
-    id("space.kscience.gradle.mpp")
+    id("space.kscience.gradle.jvm")
     `maven-publish`
 }
 
 val dataforgeVersion: String by rootProject.extra
 val ktorVersion = space.kscience.gradle.KScienceVersions.ktorVersion
 
-kscience{
-    jvm()
-    useContextReceivers()
-
-    jvmMain{
+dependencies {
     api(projects.snarkHtml)
 
     api("io.ktor:ktor-server-core:$ktorVersion")
     api("io.ktor:ktor-server-html-builder:$ktorVersion")
     api("io.ktor:ktor-server-host-common:$ktorVersion")
-    }
-    jvmTest{
-        api("io.ktor:ktor-server-tests:$ktorVersion")
-    }
+    implementation("io.ktor:ktor-server-netty:2.3.0")
+    implementation(project(":snark-storage-driver"))
+    implementation("io.ktor:ktor-server-partial-content:$ktorVersion")
+    implementation("io.ktor:ktor-server-auto-head-response:$ktorVersion")
+
+    testApi("io.ktor:ktor-server-tests:$ktorVersion")
 }

snark-ktor/snark-ktor.iml

@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="JAVA_MODULE" version="4">
+  <component name="NewModuleRootManager" inherit-compiler-output="true">
+    <exclude-output />
+    <content url="file://$MODULE_DIR$">
+      <sourceFolder url="file://$MODULE_DIR$/src/main/kotlin" isTestSource="false" />
+    </content>
+    <orderEntry type="inheritedJdk" />
+    <orderEntry type="sourceFolder" forTests="false" />
+    <orderEntry type="library" name="KotlinJavaRuntime" level="project" />
+  </component>
+</module>

snark-ktor/src/jvmMain/kotlin/space/kscience/snark/ktor/KtorSiteContext.kt

@@ -1,200 +0,0 @@
-package space.kscience.snark.ktor
-
-import io.ktor.http.*
-import io.ktor.http.content.TextContent
-import io.ktor.server.application.call
-import io.ktor.server.http.content.staticFiles
-import io.ktor.server.plugins.origin
-import io.ktor.server.response.respond
-import io.ktor.server.response.respondBytes
-import io.ktor.server.routing.Route
-import io.ktor.server.routing.createRouteFromPath
-import io.ktor.server.routing.get
-import space.kscience.dataforge.context.Context
-import space.kscience.dataforge.context.ContextAware
-import space.kscience.dataforge.context.error
-import space.kscience.dataforge.context.logger
-import space.kscience.dataforge.data.Data
-import space.kscience.dataforge.data.DataTree
-import space.kscience.dataforge.data.await
-import space.kscience.dataforge.data.meta
-import space.kscience.dataforge.io.Binary
-import space.kscience.dataforge.io.toByteArray
-import space.kscience.dataforge.meta.*
-import space.kscience.dataforge.names.Name
-import space.kscience.dataforge.names.cutLast
-import space.kscience.dataforge.names.endsWith
-import space.kscience.dataforge.names.plus
-import space.kscience.dataforge.workspace.FileData
-import space.kscience.snark.html.*
-import java.nio.file.Path
-import kotlin.reflect.typeOf
-
-//public fun CommonAttributeGroupFacade.css(block: CssBuilder.() -> Unit) {
-//    style = CssBuilder().block().toString()
-//}
-
-internal class KtorSiteContext(
-    override val context: Context,
-    override val siteMeta: Meta,
-    override val path: List<String>,
-    override val siteRoute: Name,
-    override val parent: SiteContext?,
-    private val ktorRoute: Route,
-) : SiteContext, ContextAware {
-
-
-    override fun static(route: Name, data: Data<Binary>) {
-        data.meta[FileData.FILE_PATH_KEY]?.string?.let {
-            val file = try {
-                Path.of(it).toFile()
-            } catch (ex: Exception) {
-                //failure,
-                logger.error { "File $it could not be converted to java.io.File" }
-                return@let
-            }
-
-            val fileName = route.toWebPath()
-            ktorRoute.staticFiles(fileName, file)
-            //success, don't do anything else
-            return
-        }
-
-        if (data.type != typeOf<Binary>()) error("Can't directly serve file of type ${data.type}")
-        ktorRoute.get(route.toWebPath()) {
-            val binary = data.await()
-            val extension = data.meta[FileData.FILE_EXTENSION_KEY]?.string?.let { ".$it" } ?: ""
-            val contentType: ContentType = extension
-                .let(ContentType::fromFileExtension)
-                .firstOrNull()
-                ?: ContentType.Any
-            call.respondBytes(contentType = contentType) {
-                //TODO optimize using streaming
-                binary.toByteArray()
-            }
-        }
-    }
-
-
-    private class KtorPageContext(
-        override val site: KtorSiteContext,
-        override val host: Url,
-        override val pageRoute: Name,
-        override val pageMeta: Meta,
-    ) : PageContext {
-
-        override fun resolvePageRef(
-            pageName: Name,
-            targetSite: SiteContext,
-        ): String {
-            return if (pageName.endsWith(SiteContext.INDEX_PAGE_TOKEN)) {
-                resolveRef(pageName.cutLast().toWebPath(), targetSite)
-            } else {
-                resolveRef(pageName.toWebPath(), targetSite)
-            }
-        }
-    }
-
-    override fun page(route: Name, data: DataTree<*>?, pageMeta: Meta, content: HtmlPage) {
-        ktorRoute.get(route.toWebPath()) {
-            val request = call.request
-            //substitute host for url for backwards calls
-//            val url = URLBuilder(baseUrl).apply {
-//                protocol = URLProtocol.createOrDefault(request.origin.scheme)
-//                host = request.origin.serverHost
-//                port = request.origin.serverPort
-//            }
-
-            val hostUrl = URLBuilder().apply {
-                protocol = URLProtocol.createOrDefault(request.origin.scheme)
-                host = request.origin.serverHost
-                port = request.origin.serverPort
-            }
-
-            val modifiedPageMeta = pageMeta.copy {
-                "host" put hostUrl.buildString()
-                "path" put path.map { it.asValue() }.asValue()
-                "route" put (siteRoute + route).toString()
-            }
-
-            val pageContext = KtorPageContext(
-                site = this@KtorSiteContext,
-                host = hostUrl.build(),
-                pageRoute = siteRoute + route,
-                pageMeta = Laminate(modifiedPageMeta, siteMeta)
-            )
-            //render page in suspend environment
-            val html = HtmlPage.createHtmlString(pageContext, data, content)
-
-            call.respond(TextContent(html, ContentType.Text.Html.withCharset(Charsets.UTF_8), HttpStatusCode.OK))
-        }
-    }
-
-    override fun route(route: Name, data: DataTree<*>?, siteMeta: Meta, content: HtmlSite) {
-        val siteContext = SiteContextWithData(
-            KtorSiteContext(
-                context,
-                siteMeta = Laminate(siteMeta, this@KtorSiteContext.siteMeta),
-                path = path,
-                siteRoute = route,
-                parent,
-                ktorRoute = ktorRoute.createRouteFromPath(route.toWebPath())
-            ),
-            data ?: DataTree.EMPTY
-        )
-        with(content) {
-            with(siteContext) {
-                renderSite()
-            }
-        }
-    }
-
-    override fun site(route: Name, data: DataTree<*>?, siteMeta: Meta, content: HtmlSite) {
-        val siteContext = SiteContextWithData(
-            KtorSiteContext(
-                context,
-                siteMeta = Laminate(siteMeta, this@KtorSiteContext.siteMeta),
-                path = path + route.tokens.map { it.toStringUnescaped() },
-                siteRoute = Name.EMPTY,
-                this,
-                ktorRoute = ktorRoute.createRouteFromPath(route.toWebPath())
-            ),
-            data ?: DataTree.EMPTY
-        )
-        with(content) {
-            with(siteContext) {
-                renderSite()
-            }
-        }
-    }
-
-}
-
-public fun Route.site(
-    context: Context,
-    data: DataTree<*>?,
-    path: List<String> = emptyList(),
-    siteMeta: Meta = data?.meta ?: Meta.EMPTY,
-    content: HtmlSite,
-) {
-    val siteContext = SiteContextWithData(
-        KtorSiteContext(context, siteMeta, path = path, siteRoute = Name.EMPTY, null, this@Route),
-        data ?: DataTree.EMPTY
-    )
-    with(content) {
-        with(siteContext) {
-            renderSite()
-        }
-    }
-}
-//
-//public suspend fun Application.site(
-//    context: Context,
-//    data: DataSet<*>,
-//    baseUrl: String = "",
-//    siteMeta: Meta = data.meta,
-//    content: HtmlSite,
-//) {
-//    routing {}.site(context, data, baseUrl, siteMeta, content)
-//
-//}

snark-ktor/src/main/kotlin/space/kscience/snark/ktor/KtorSiteBuilder.kt

@@ -0,0 +1,170 @@
+package space.kscience.snark.ktor
+
+import io.ktor.http.URLBuilder
+import io.ktor.http.URLProtocol
+import io.ktor.server.application.Application
+import io.ktor.server.application.call
+import io.ktor.server.html.respondHtml
+import io.ktor.server.http.content.*
+import io.ktor.server.plugins.origin
+import io.ktor.server.routing.Route
+import io.ktor.server.routing.createRouteFromPath
+import io.ktor.server.routing.get
+import io.ktor.server.routing.routing
+import kotlinx.css.CssBuilder
+import kotlinx.html.CommonAttributeGroupFacade
+import kotlinx.html.HTML
+import kotlinx.html.style
+import space.kscience.dataforge.data.DataTree
+import space.kscience.dataforge.meta.Laminate
+import space.kscience.dataforge.meta.Meta
+import space.kscience.dataforge.meta.toMutableMeta
+import space.kscience.dataforge.names.Name
+import space.kscience.dataforge.names.cutLast
+import space.kscience.dataforge.names.endsWith
+import space.kscience.dataforge.names.plus
+import space.kscience.snark.SnarkEnvironment
+import space.kscience.snark.html.*
+import java.nio.file.Path
+import kotlin.contracts.InvocationKind
+import kotlin.contracts.contract
+import kotlin.io.path.isDirectory
+
+public fun CommonAttributeGroupFacade.css(block: CssBuilder.() -> Unit) {
+    style = CssBuilder().block().toString()
+}
+
+public class KtorSiteBuilder(
+    override val snark: SnarkHtmlPlugin,
+    override val data: DataTree<*>,
+    override val siteMeta: Meta,
+    private val baseUrl: String,
+    override val route: Name,
+    private val ktorRoute: Route,
+) : SiteBuilder {
+
+    override fun file(file: Path, remotePath: String) {
+        if (file.isDirectory()) {
+            ktorRoute.static(remotePath) {
+                //TODO check non-standard FS and convert
+                files(file.toFile())
+            }
+        } else if (remotePath.isBlank()) {
+            error("Can't mount file to an empty route")
+        } else {
+            ktorRoute.file(remotePath, file.toFile())
+        }
+    }
+
+    private fun resolveRef(baseUrl: String, ref: String) = if (baseUrl.isEmpty()) {
+        ref
+    } else if (ref.isEmpty()) {
+        baseUrl
+    } else {
+        "${baseUrl.removeSuffix("/")}/$ref"
+    }
+
+
+    private inner class KtorWebPage(
+        val pageBaseUrl: String,
+        override val pageMeta: Meta,
+    ) : WebPage {
+        override val snark: SnarkHtmlPlugin get() = this@KtorSiteBuilder.snark
+        override val data: DataTree<*> get() = this@KtorSiteBuilder.data
+
+        override fun resolveRef(ref: String): String = resolveRef(pageBaseUrl, ref)
+
+        override fun resolvePageRef(
+            pageName: Name,
+            relative: Boolean,
+        ): String {
+            val fullPageName = if(relative) route + pageName else pageName
+            return if (fullPageName.endsWith(SiteBuilder.INDEX_PAGE_TOKEN)) {
+                resolveRef(fullPageName.cutLast().toWebPath())
+            } else {
+                resolveRef(fullPageName.toWebPath())
+            }
+        }
+    }
+
+    override fun page(route: Name, pageMeta: Meta, content: context(WebPage, HTML)() -> Unit) {
+        ktorRoute.get(route.toWebPath()) {
+            call.respondHtml {
+                val request = call.request
+                //substitute host for url for backwards calls
+                val url = URLBuilder(baseUrl).apply {
+                    protocol = URLProtocol.createOrDefault(request.origin.scheme)
+                    host = request.origin.host
+                    port = request.origin.port
+                }
+
+                val modifiedPageMeta = pageMeta.toMutableMeta().apply {
+                    "name" put route.toString()
+                    "url" put url.buildString()
+                }
+
+                val pageBuilder = KtorWebPage(url.buildString(), Laminate(modifiedPageMeta, siteMeta))
+                content(pageBuilder, this)
+            }
+        }
+    }
+
+    override fun route(
+        routeName: Name,
+        dataOverride: DataTree<*>?,
+        routeMeta: Meta,
+    ): SiteBuilder = KtorSiteBuilder(
+        snark = snark,
+        data = dataOverride ?: data,
+        siteMeta = Laminate(routeMeta, siteMeta),
+        baseUrl = baseUrl,
+        route = this.route + routeName,
+        ktorRoute = ktorRoute.createRouteFromPath(routeName.toWebPath())
+    )
+
+    override fun site(
+        routeName: Name,
+        dataOverride: DataTree<*>?,
+        routeMeta: Meta,
+    ): SiteBuilder = KtorSiteBuilder(
+        snark = snark,
+        data = dataOverride ?: data,
+        siteMeta = Laminate(routeMeta, siteMeta),
+        baseUrl = resolveRef(baseUrl, routeName.toWebPath()),
+        route = Name.EMPTY,
+        ktorRoute = ktorRoute.createRouteFromPath(routeName.toWebPath())
+    )
+
+
+    override fun resourceFile(remotePath: String, resourcesPath: String) {
+        ktorRoute.resource(resourcesPath, resourcesPath)
+    }
+
+    override fun resourceDirectory(resourcesPath: String) {
+        ktorRoute.resources(resourcesPath)
+    }
+}
+
+context(Route, SnarkEnvironment)
+private fun siteInRoute(
+    baseUrl: String = "",
+    block: KtorSiteBuilder.() -> Unit,
+) {
+    contract {
+        callsInPlace(block, InvocationKind.EXACTLY_ONCE)
+    }
+    block(KtorSiteBuilder(buildHtmlPlugin(), data, meta, baseUrl, route = Name.EMPTY, this@Route))
+}
+
+context(Application)
+public fun SnarkEnvironment.site(
+    baseUrl: String = "",
+    block: KtorSiteBuilder.() -> Unit,
+) {
+    contract {
+        callsInPlace(block, InvocationKind.EXACTLY_ONCE)
+    }
+    routing {
+        siteInRoute(baseUrl, block)
+    }
+}

snark-ktor/src/main/kotlin/space/kscience/snark/ktor/WebInterface.kt

@@ -0,0 +1,131 @@
+package space.kscience.snark.ktor
+
+import io.ktor.http.*
+import io.ktor.server.application.*
+import io.ktor.server.response.*
+import io.ktor.server.routing.*
+import io.ktor.http.content.*
+import io.ktor.server.engine.*
+import io.ktor.server.netty.*
+import io.ktor.server.html.*
+import io.ktor.server.request.*
+import kotlinx.html.*
+import space.kscience.snark.storage.Directory
+import space.kscience.snark.storage.unzip.unzip
+import java.nio.file.Path
+import kotlin.io.createTempFile
+import kotlin.io.writeBytes
+import kotlin.io.path.Path
+
+public interface DataHolder {
+    public suspend fun init(relativePath: Path) : Directory
+
+    public suspend fun represent(relativePath: Path): String
+    public suspend fun toPdf(relativePath: Path) : Path
+}
+
+public class SNARKServer(private val dataHolder: DataHolder, private val port: Int): Runnable {
+    private var relativePath = Path("")
+
+    private suspend fun receivePath(call: ApplicationCall) {
+        val pathString = call.receiveParameters()["path"]?:""
+        relativePath = Path(pathString.dropWhile{it == '/'})
+        call.respondRedirect("/")
+    }
+    private suspend fun renderGet(call: ApplicationCall) {
+        call.respondText(dataHolder.represent(relativePath), ContentType.Text.Html)
+    }
+    private suspend fun renderFile(call: ApplicationCall) {
+        call.response.header(
+            HttpHeaders.ContentDisposition,
+            ContentDisposition.Attachment.withParameter(ContentDisposition.Parameters.FileName, "output.tex")
+                .toString()
+        )
+        call.respondFile(dataHolder.toPdf(relativePath).toFile())
+        call.respondRedirect("/")
+
+    }
+    private suspend fun renderUpload(call: ApplicationCall) {
+        val multipartData = call.receiveMultipart()
+        val tmp = createTempFile(suffix=".zip")
+        multipartData.forEachPart { part ->
+            when (part) {
+                is PartData.FileItem -> {
+                    val fileBytes = part.streamProvider().readBytes()
+                    tmp.writeBytes(fileBytes)
+                }
+                else -> {
+                }
+            }
+            part.dispose()
+        }
+        unzip(tmp.toPath().toString(), dataHolder.init(relativePath))
+        call.respondRedirect("/")
+    }
+    private suspend fun renderMainPage(call: ApplicationCall)  {
+        call.respondHtml(HttpStatusCode.OK) {
+            head {
+                title {
+                    +"SNARK"
+                }
+            }
+            body {
+                h1 {
+                    +"SNARK"
+                }
+                p {
+                    +("Path: /" + relativePath.toString())
+                }
+            }
+            body {
+                postForm(action = "/changePath") {
+                    label {
+                        + "Enter new path:"
+                    }
+                    input(name = "path", type = InputType.text) {}
+                    button {
+                        +"Change path"
+                    }
+                }
+                postForm (action = "/upload", encType = FormEncType.multipartFormData)  {
+                    label {
+                        +"Choose zip archive: "
+                    }
+                    input (name = "file", type = InputType.file) {}
+                    button {
+                        +"Upload file"
+                    }
+                }
+                a("/data") {
+                    +"Show data\n"
+                }
+                getForm (action = "/download") {
+                    button {
+                        +"Download latex\n"
+                    }
+                }
+            }
+        }
+    }
+    override fun run() {
+        embeddedServer(Netty, port) {
+            routing {
+                get("/") {
+                    renderMainPage(call)
+                }
+                post("/changePath") {
+                    receivePath(call)
+                }
+                post("/upload") {
+                    renderUpload(call)
+                }
+                get("/data") {
+                    renderGet(call)
+                }
+                get("/download") {
+                    renderFile(call)
+                }
+            }
+        }.start(wait = true)
+    }
+}

snark-ktor/src/main/kotlin/space/kscience/snark/ktor/extractData.kt

@@ -2,30 +2,56 @@ package space.kscience.snark.ktor
 
 import io.ktor.server.application.Application
 import io.ktor.server.application.log
-import io.ktor.server.config.tryGetString
+import space.kscience.dataforge.context.info
+import space.kscience.dataforge.context.logger
+import java.net.URI
+import java.nio.file.FileSystems
 import java.nio.file.Files
 import java.nio.file.Path
 import java.time.LocalDateTime
 import kotlin.io.path.*
 
 
+public fun KtorSiteBuilder.extractResources(uri: URI, targetPath: Path): Path {
+    if (Files.isDirectory(targetPath)) {
+        logger.info { "Using existing data directory at $targetPath." }
+    } else {
+        logger.info { "Copying data from $uri into $targetPath." }
+        targetPath.createDirectories()
+        //Copy everything into a temporary directory
+        FileSystems.newFileSystem(uri, emptyMap<String, Any>()).use { fs ->
+            val rootPath: Path = fs.provider().getPath(uri)
+            Files.walk(rootPath).forEach { source: Path ->
+                if (source.isRegularFile()) {
+                    val relative = source.relativeTo(rootPath).toString()
+                    val destination: Path = targetPath.resolve(relative)
+                    destination.parent.createDirectories()
+                    Files.copy(source, destination)
+                }
+            }
+        }
+    }
+    return targetPath
+}
+
+public fun KtorSiteBuilder.extractResources(resource: String, targetPath: Path): Path =
+    extractResources(javaClass.getResource(resource)!!.toURI(), targetPath)
+
 private const val DEPLOY_DATE_FILE = "deployDate"
 private const val BUILD_DATE_FILE = "/buildDate"
 
 /**
  * Prepare the data cache directory for snark. Clear data if it is outdated.
  * TODO make internal
- *
- * @return true if cache is valid and false if it is reset
  */
-fun Application.prepareSnarkDataCacheDirectory(dataPath: Path): Boolean {
+fun Application.prepareSnarkDataCacheDirectory(dataPath: Path) {
 
-    // Clear data directory if it is outdated
+// Clear data directory if it is outdated
     val deployDate = dataPath.resolve(DEPLOY_DATE_FILE).takeIf { it.exists() }
         ?.readText()?.let { LocalDateTime.parse(it) }
     val buildDate = javaClass.getResource(BUILD_DATE_FILE)?.readText()?.let { LocalDateTime.parse(it) }
 
-    val inProduction: Boolean = environment.config.tryGetString("ktor.environment.production") == "true"
+    val inProduction: Boolean = environment.config.propertyOrNull("ktor.environment.production") != null
 
     if (inProduction) {
         log.info("Production mode activated")
@@ -33,11 +59,7 @@ fun Application.prepareSnarkDataCacheDirectory(dataPath: Path): Boolean {
         log.info("Deploy date: $deployDate")
     }
 
-    if (!dataPath.exists()) {
-        dataPath.createDirectories()
-        dataPath.resolve(DEPLOY_DATE_FILE).writeText(LocalDateTime.now().toString())
-        return false
-    } else if (deployDate != null && buildDate != null && buildDate.isAfter(deployDate)) {
+    if (deployDate != null && buildDate != null && buildDate.isAfter(deployDate)) {
         log.info("Outdated data. Resetting data directory.")
 
         Files.walk(dataPath)
@@ -47,7 +69,6 @@ fun Application.prepareSnarkDataCacheDirectory(dataPath: Path): Boolean {
         //Writing deploy date file
         dataPath.createDirectories()
         dataPath.resolve(DEPLOY_DATE_FILE).writeText(LocalDateTime.now().toString())
-        return false
 
     } else if (inProduction && deployDate == null && buildDate != null) {
         val date = LocalDateTime.now().toString()
@@ -55,8 +76,5 @@ fun Application.prepareSnarkDataCacheDirectory(dataPath: Path): Boolean {
         //Writing deploy date in production mode if it does not exist
         dataPath.createDirectories()
         dataPath.resolve(DEPLOY_DATE_FILE).writeText(date)
-        return false
-    } else {
-        return true
     }
 }

snark-ktor/src/test/kotlin/space/kscience/snark/ktor/testServer.kt

@@ -0,0 +1,50 @@
+package space.kscience.snark.ktor
+
+import space.kscience.snark.storage.Directory
+import space.kscience.snark.storage.local.localStorage
+import java.nio.file.Path
+import kotlin.io.path.*
+import kotlin.io.path.createTempDirectory
+import kotlin.io.path.isDirectory
+import kotlin.io.path.listDirectoryEntries
+
+private class LocalDataHolder: DataHolder {
+    private var source: Path? = null
+    private var response: String = ""
+
+    private fun getPath(relativePath: Path) : Path {
+        return source!! / relativePath
+    }
+    override suspend fun init(relativePath: Path): Directory {
+        if (source == null) {
+            source = createTempDirectory()
+        }
+        val path = getPath(relativePath)
+        path.createDirectories()
+        path.toFile().deleteRecursively()
+        path.createDirectory()
+        return localStorage(path)
+    }
+    private fun buildResponse(from: Path, cur: Path) {
+        for (entry in cur.listDirectoryEntries()) {
+            if (entry.isDirectory()) {
+                buildResponse(from, entry)
+            } else {
+                response += from.relativize(entry).toString() + "<br>"
+            }
+        }
+    }
+    override suspend fun represent(relativePath: Path) : String =
+        if (source == null) {
+            "No data was loaded!"
+        } else {
+            response = "List of files:<br>"
+            val path = getPath(relativePath)
+            buildResponse(path, path)
+            response
+        }
+}
+
+fun main() {
+    SNARKServer(LocalDataHolder(), 9090).run()
+}

snark-main/build.gradle.kts

@@ -0,0 +1,27 @@
+plugins {
+    id("space.kscience.gradle.jvm")
+    `maven-publish`
+}
+
+val coroutinesVersion = space.kscience.gradle.KScienceVersions.coroutinesVersion
+
+
+dependencies {
+    implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:$coroutinesVersion")
+    implementation(project(":snark-ktor"))
+    implementation(project(":snark-storage-driver"))
+    implementation(project(":snark-document-builder"))
+
+    testImplementation(kotlin("test"))
+    testImplementation("org.junit.jupiter:junit-jupiter:5.8.1")
+}
+
+tasks.test {
+    useJUnitPlatform()
+}
+
+tasks.register<JavaExec>("run_server") {
+    classpath = sourceSets.main.get().runtimeClasspath
+    main = "space.kscience.snark.main.MainKt"
+    workingDir = File(workingDir.parent)
+}

snark-main/ci/install_java.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+JAVA_VERSION="$1"
+
+source ~/.sdkman/bin/sdkman-init.sh
+sdk install java "$JAVA_VERSION"

snark-main/ci/install_sdk.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+curl -s "https://get.sdkman.io" | bash

snark-main/src/main/kotlin/space/kscience/snark/main/Main.kt

@@ -0,0 +1,19 @@
+package space.kscience.snark.main
+
+import kotlinx.coroutines.coroutineScope
+import kotlinx.coroutines.launch
+import kotlinx.coroutines.runBlocking
+import space.kscience.snark.ktor.SNARKServer
+import space.kscience.snark.storage.local.localStorage
+import kotlin.io.path.Path
+
+// Entrypoint
+fun main(): Unit = runBlocking {
+    // Parse config, create driver
+    val port = 8080
+    val directory = localStorage(Path("./rundata"))
+    val server = SNARKServer(ServerDataHolder(directory), port)
+    launch {
+        server.run()
+    }
+}

snark-main/src/main/kotlin/space/kscience/snark/main/ServerDataHolder.kt

@@ -0,0 +1,20 @@
+package space.kscience.snark.main
+
+import space.kscience.snark.ktor.DataHolder
+import space.kscience.snark.storage.Directory
+import documentBuilder.*
+import java.nio.file.Path
+
+internal class ServerDataHolder(private val directory: Directory): DataHolder {
+
+    override suspend fun init(relativePath: Path): Directory = directory
+
+
+    override suspend fun represent(relativePath: Path): String {
+        return buildDocument(directory, relativePath)
+    }
+
+    override suspend fun toPdf(relativePath: Path) :  Path {
+        return buildLatex(directory, relativePath)
+    }
+}

snark-main/src/test/kotlin/space/kscience/snark/main/Test.kt

@@ -0,0 +1,13 @@
+package space.kscience.snark.main
+
+import kotlinx.coroutines.delay
+import kotlinx.coroutines.runBlocking
+import org.junit.jupiter.api.Test
+
+class Test {
+    @Test
+    fun justWorks() = runBlocking {
+        delay(5)
+        //main()
+    }
+}

snark-pandoc-plugin/.gitignore

@@ -0,0 +1,2 @@
+.pandoc/
+

snark-pandoc-plugin/build.gradle.kts

@@ -0,0 +1,25 @@
+plugins {
+    id("java")
+}
+
+repositories {
+    mavenCentral()
+}
+
+java.sourceCompatibility = JavaVersion.VERSION_11
+
+dependencies {
+    implementation("commons-io:commons-io:2.7")
+    implementation("org.slf4j:slf4j-simple:2.0.6")
+    implementation("org.slf4j:slf4j-api:2.0.6")
+    implementation("org.apache.commons:commons-exec:1.3")
+    implementation("org.apache.commons:commons-compress:1.2")
+    implementation("org.apache.ant:ant:1.10.13")
+    implementation("com.fasterxml.jackson.core:jackson-databind:2.14.2")
+    testImplementation("org.junit.jupiter:junit-jupiter-api:5.8.1")
+    testRuntimeOnly("org.junit.jupiter:junit-jupiter-engine:5.8.1")
+}
+
+tasks.getByName<Test>("test") {
+    useJUnitPlatform()
+}

snark-pandoc-plugin/src/main/java/space/kscience/snark/pandoc/Installer.java

@@ -0,0 +1,367 @@
+package space.kscience.snark.pandoc;
+
+import java.io.BufferedInputStream;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.net.HttpURLConnection;
+import java.net.URI;
+import java.net.URL;
+import java.net.http.HttpClient;
+import java.net.http.HttpRequest;
+import java.net.http.HttpResponse;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.time.Duration;
+import java.util.Enumeration;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Properties;
+
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.commons.compress.archivers.ArchiveEntry;
+import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
+import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
+import org.apache.commons.compress.archivers.zip.ZipFile;
+import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
+import org.apache.commons.exec.OS;
+import org.apache.commons.io.FileUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+public class Installer {
+
+    private static final Logger log
+            = LoggerFactory.getLogger(Installer.class);
+
+    public enum OSType {
+        WINDOWS("windows-x86_64.zip", "windows"),
+        MAC_OS_AMD("x86_64-macOS.zip", "mac.os.amd"),
+        MAC_OS_ARM("arm64-macOS.zip", "mac.os.arm"),
+        LINUX_ARM("linux-arm64", "linux.arm"),
+        LINUX_AMD("linux-amd64", "linux.amd");
+
+        private final String assetSuffix;
+        private final String propertySuffix;
+        OSType(String assetSuf, String propertySuf) {
+            assetSuffix = assetSuf;
+            propertySuffix = propertySuf;
+        }
+        public String getAssetSuffix() {
+            return assetSuffix;
+        }
+        public String getPropertySuffix() {
+            return propertySuffix;
+        }
+    }
+
+    private static class OSData {
+        private URL urlForInstalling;
+        private Path fileToInstall;
+        private Path pathToPandoc;
+        public OSData() {}
+        public void setUrlForInstalling(URL urlForInstalling) {
+            this.urlForInstalling = urlForInstalling;
+        }
+        public URL getUrlForInstalling() {
+            return urlForInstalling;
+        }
+        public void setFileToInstall(Path fileToInstall) {
+            this.fileToInstall = fileToInstall;
+        }
+        public Path getFileToInstall() {
+            return fileToInstall;
+        }
+        public Path getPathToPandoc() {
+            return pathToPandoc;
+        }
+        public void setPathToPandoc(Path pathToPandoc) {
+            this.pathToPandoc = pathToPandoc;
+        }
+    }
+
+    private static Map<OSType, OSData> dataForInstalling = new HashMap<>(OSType.values().length);
+    private static final Properties properties = new Properties();
+    private static Path pandocDir = Path.of("./pandoc").toAbsolutePath();
+    private static final int TIMEOUT_SECONDS = 2;
+    private static final int ATTEMPTS = 3;
+
+    Installer() throws IOException, InterruptedException {
+        try {
+            properties.load(new FileInputStream(
+                    Thread.currentThread().getContextClassLoader().getResource("installer.properties").getPath()
+            ));
+        } catch (Exception ex) {
+            log.error("Error during download properties, ex: {}", ex.getMessage(), ex);
+            throw ex;
+        }
+
+        initFiles();
+        var resp = getGithubUrls();
+        initUrls(resp);
+    }
+
+    private void initFiles() {
+        for (var os : OSType.values()) {
+            dataForInstalling.put(os, new OSData());
+            switch (os) {
+                case LINUX_AMD :
+                case LINUX_ARM :
+                        dataForInstalling.get(os)
+                                .setFileToInstall(Path.of(pandocDir.toString() + "/pandoc.tar.gz"));
+                        break;
+                default :
+                        dataForInstalling.get(os)
+                                .setFileToInstall(Path.of(pandocDir.toString() + "/pandoc.zip"));
+            }
+        }
+    }
+
+    private void initUrls(ResponseDto responseDto) throws IOException {
+
+        for (var os : OSType.values()) {
+            var asset = responseDto.getAssetByOsSuffix(os.getAssetSuffix());
+            var currUrl = asset.getBrowserDownloadUrl();
+
+            var currPath = properties.getProperty("path.to.pandoc." + os.getPropertySuffix()).replace("{version}",
+                    responseDto.getTagName());
+
+            dataForInstalling.get(os).setUrlForInstalling(URI.create(currUrl).toURL());
+            dataForInstalling.get(os).setPathToPandoc(Path.of(pandocDir.toString() + currPath));
+            log.info("Init {} url : {}, path to pandoc: {}", os, currUrl, dataForInstalling.get(os).getPathToPandoc());
+        }
+    }
+
+    /**
+     * Install last released pandoc from github
+     * @return path to executable pandoc
+     * @throws IOException in case incorrect github url or path of installation directory
+     */
+    Path installPandoc() throws IOException {
+        log.info("Start install");
+        Path res;
+        if (OS.isFamilyMac()) {
+            if (OS.isArch("aarch64")) {
+                res = installPandoc(OSType.MAC_OS_ARM);
+            } else {
+                res = installPandoc(OSType.MAC_OS_AMD);
+            }
+        } else if (OS.isFamilyUnix()) {
+            if (OS.isArch("aarch64")) {
+                res = installPandoc(OSType.LINUX_ARM);
+            } else {
+                res = installPandoc(OSType.LINUX_AMD);
+            }
+        } else if (OS.isFamilyWindows()) {
+            res = installPandoc(OSType.WINDOWS);
+        } else {
+            throw new RuntimeException("Got unexpected os, could not install pandoc");
+        }
+        return res;
+    }
+
+    private Path installPandoc(OSType os) throws IOException {
+        log.info(
+                "Start installing pandoc os: {}, url: {}, file: {}",
+                os,
+                dataForInstalling.get(os).getUrlForInstalling(),
+                dataForInstalling.get(os).getFileToInstall()
+        );
+
+        clearInstallingDirectory();
+
+        if (!handleSaving(os)) {
+            throw new RuntimeException("Could not save file from github");
+        }
+        if (!unarchive(os)) {
+            throw new RuntimeException("Could not unzip file");
+        }
+
+        if (!dataForInstalling.get(os).getPathToPandoc().toFile().setExecutable(true)) {
+            throw new RuntimeException("Could not make pandoc executable");
+        }
+
+        return dataForInstalling.get(os).getPathToPandoc();
+    }
+
+    /**
+     * Downloads from a (http/https) URL and saves to a file.
+     * @param file File to write. Parent directory will be created if necessary
+     * @param url  http/https url to connect
+     * @param secsConnectTimeout Seconds to wait for connection establishment
+     * @param secsReadTimeout Read timeout in seconds - trasmission will abort if it freezes more than this
+     * @return true if successfully save file and false if:
+     *     connection interrupted, timeout (but something was read)
+     *     server error (500...)
+     *     could not connect: connection timeout java.net.SocketTimeoutException
+     *     could not connect: java.net.ConnectException
+     *     could not resolve host (bad host, or no internet - no dns)
+     * @throws IOException Only if URL is malformed or if could not create the file
+     * @throws FileNotFoundException if did not find file for save
+     */
+    private boolean saveUrl(final Path file, final URL url,
+                              int secsConnectTimeout, int secsReadTimeout) throws IOException {
+        Files.createDirectories(file.getParent()); // make sure parent dir exists , this can throw exception
+        var conn = url.openConnection(); // can throw exception if bad url
+        if (secsConnectTimeout > 0) {
+            conn.setConnectTimeout(secsConnectTimeout * 1000);
+        }
+        if (secsReadTimeout > 0) {
+            conn.setReadTimeout(secsReadTimeout * 1000);
+        }
+        var ret = true;
+        boolean somethingRead = false;
+        try (var is = conn.getInputStream()) {
+            try (var in = new BufferedInputStream(is);
+                 var fout = Files.newOutputStream(file)) {
+                final byte data[] = new byte[8192];
+                int count;
+                while ((count = in.read(data)) > 0) {
+                    somethingRead = true;
+                    fout.write(data, 0, count);
+                }
+            }
+        } catch (java.io.IOException e) {
+            int httpcode = 999;
+            try {
+                httpcode = ((HttpURLConnection) conn).getResponseCode();
+            } catch (Exception ee) {}
+
+            if (e instanceof FileNotFoundException) {
+                throw new FileNotFoundException("Did not found file for install");
+            }
+
+            if (somethingRead && e instanceof java.net.SocketTimeoutException) {
+                log.error("Read something, but connection interrupted: {}", e.getMessage(), e);
+                ret = false;
+            } else if (httpcode >= 400 && httpcode < 600 ) {
+                log.error("Got server error, httpcode: {}", httpcode);
+                ret = false;
+            } else if (e instanceof java.net.SocketTimeoutException) {
+                log.error("Connection timeout: {}", e.getMessage(), e);
+                ret = false;
+            } else if (e instanceof java.net.ConnectException) {
+                log.error("Could not connect: {}", e.getMessage(), e);
+                ret = false;
+            } else if (e instanceof java.net.UnknownHostException ) {
+                log.error("Could not resolve host: {}", e.getMessage(), e);
+                ret = false;
+            } else {
+                throw e;
+            }
+        }
+        return ret;
+    }
+
+    private boolean handleSaving(OSType os) throws IOException {
+        var attempt = 0;
+        var saveFile = false;
+
+        while (attempt < ATTEMPTS && !saveFile) {
+            ++attempt;
+            saveFile = saveUrl(
+                    dataForInstalling.get(os).getFileToInstall(),
+                    dataForInstalling.get(os).getUrlForInstalling(),
+                    TIMEOUT_SECONDS,
+                    TIMEOUT_SECONDS);
+        }
+
+        return saveFile;
+    }
+
+    private boolean unarchive(OSType os) {
+        try {
+            switch (os) {
+                case LINUX_AMD:
+                case LINUX_ARM :
+                    unTarGz(dataForInstalling.get(os).getFileToInstall(), pandocDir);
+                    break;
+                default :
+                    unZip(dataForInstalling.get(os).getFileToInstall(), pandocDir);
+            }
+        } catch (IOException e) {
+            log.error("Could not perform unarchiving: {}", e.getMessage(), e);
+            return false;
+        }
+        return true;
+    }
+    private void unTarGz(Path pathInput, Path targetDir) throws IOException {
+        try (var tarIn =
+                new TarArchiveInputStream(
+                        new GzipCompressorInputStream(
+                                new BufferedInputStream(Files.newInputStream(pathInput))))) {
+            ArchiveEntry archiveEntry;
+            while ((archiveEntry = tarIn.getNextEntry()) != null) {
+                var pathEntryOutput = targetDir.resolve(archiveEntry.getName());
+                if (archiveEntry.isDirectory()) {
+                    Files.createDirectory(pathEntryOutput);
+                } else {
+                    Files.copy(tarIn, pathEntryOutput);
+                }
+            }
+        }
+    }
+
+    private void unZip(Path pathInput, Path targetDir) throws IOException {
+        ZipFile zipFile = new ZipFile(pathInput.toFile());
+        try {
+            Enumeration<ZipArchiveEntry> entries = zipFile.getEntries();
+            while (entries.hasMoreElements()) {
+                ZipArchiveEntry zipEntry = entries.nextElement();
+                var pathEntryOutput = targetDir.resolve(zipEntry.getName());
+                if (zipEntry.isDirectory()) {
+                    Files.createDirectories(pathEntryOutput);
+                } else {
+                    Files.createDirectories(pathEntryOutput.getParent());
+                    Files.copy(zipFile.getInputStream(zipEntry), pathEntryOutput);
+                }
+            }
+        } finally {
+            zipFile.close();
+        }
+    }
+
+
+    /**
+     * Clear installing directory
+     */
+    public static void clearInstallingDirectory() {
+        try {
+            FileUtils.cleanDirectory(pandocDir.toFile());
+        } catch (IOException e) {
+            log.error("Could not clean installing directory");
+        }
+    }
+
+    /**
+     * Set directory to install pandoc
+     * @param newDir
+     */
+    public void setInstallingDirectory(Path newDir) {
+        pandocDir = newDir.toAbsolutePath();
+    }
+
+    private ResponseDto getGithubUrls() throws IOException, InterruptedException {
+        var uri = URI.create(properties.getProperty("github.url"));
+        var client = HttpClient.newHttpClient();
+        var request = HttpRequest
+                .newBuilder()
+                .uri(uri)
+                .version(HttpClient.Version.HTTP_2)
+                .timeout(Duration.ofMinutes(1))
+                .header("Accept",  "application/vnd.github+json")
+                .GET()
+                .build();
+        var response = client.send(request, HttpResponse.BodyHandlers.ofString());
+        log.info("Got response from github, status: {}", response.statusCode());
+
+        var objectMapper = new ObjectMapper();
+        objectMapper.configure(DeserializationFeature.USE_JAVA_ARRAY_FOR_JSON_ARRAY, true);
+        objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+
+        return objectMapper.readValue(response.body(), ResponseDto.class);
+
+    }
+}

snark-pandoc-plugin/src/main/java/space/kscience/snark/pandoc/PandocWrapper.java

@@ -0,0 +1,142 @@
+package space.kscience.snark.pandoc;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Function;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+public class PandocWrapper {
+
+    private static final Logger log
+            = LoggerFactory.getLogger(PandocWrapper.class);
+
+    private static final Installer installer;
+
+    static {
+        try {
+            installer = new Installer();
+        } catch (IOException | InterruptedException e) {
+            throw new RuntimeException(e);
+        }
+    }
+    private static String pandocPath = "pandoc"; // got pandoc at PATH
+
+    public static String getPandocPath() {
+        return pandocPath;
+    }
+
+    /**
+     * Install pandoc if needed then perform block
+     * @param block
+     * @return block's return value
+     * @param <T>
+     */
+    public  <T> Object use(Function<PandocWrapper, T> block) {
+        if (!isPandocInstalled()) {
+            installPandoc();
+        }
+        return block.apply(this);
+    }
+
+    /**
+     * Check if pandoc is installed
+     * @return true if installed false otherwise
+     */
+    public static boolean isPandocInstalled() {
+        var pb = new PandocCommandBuilder().getVersion();
+        return execute(pb);
+    }
+    /**
+     * Call pandoc with options described by commandBuilder.
+     * @param commandBuilder
+     * @return true if successfully false otherwise
+     */
+    public static boolean execute(PandocCommandBuilder commandBuilder) {
+        return execute(commandBuilder, null, null);
+    }
+
+    /**
+     * Call pandoc with options described by commandBuilder and log output to outputFile
+     * @param commandBuilder
+     * @return true if successfully false otherwise
+     */
+    public static boolean execute(PandocCommandBuilder commandBuilder, Path outputFile) {
+        return execute(commandBuilder, outputFile, null);
+    }
+    /**
+     * Call pandoc with options described by commandBuilder and log output to outputFile and error to errorFile.
+     * In case errors write exit code to errorFile
+     * @param commandBuilder
+     * @return true if successfully false otherwise
+     */
+    public static boolean execute(PandocCommandBuilder commandBuilder, Path outputFile, Path errorFile) {
+        try {
+            Process pandoc = new ProcessBuilder(commandBuilder.build()).start();
+            pandoc.waitFor(1, TimeUnit.SECONDS);
+
+            BufferedReader inp = new BufferedReader(new InputStreamReader(pandoc.getInputStream()));
+            String currLine = inp.readLine();
+
+            log.info("log output from pandoc to: {}", outputFile);
+            do {
+                if (outputFile == null) {
+                    log.info(currLine);
+                } else {
+                    Files.writeString(outputFile, currLine + "\n", StandardOpenOption.CREATE, StandardOpenOption.APPEND);
+                }
+            } while ((currLine = inp.readLine()) != null);
+            inp.close();
+
+            if (pandoc.exitValue() == 0) {
+                log.info("Successfully execute");
+                return true;
+            } else {
+                log.error("Got problems with executing, pandoc exit error: {}", pandoc.exitValue());
+
+                BufferedReader input = new BufferedReader(new InputStreamReader(pandoc.getErrorStream()));
+                String line = input.readLine();
+                log.info("log error stream from pandoc to: {}", errorFile);
+
+                if (errorFile != null) {
+                    Files.writeString(errorFile, "exit code: " + pandoc.exitValue());
+                }
+                do {
+                    if (errorFile == null) {
+                        log.info(line);
+                    } else {
+                        Files.writeString(errorFile, currLine + "\n", StandardOpenOption.CREATE, StandardOpenOption.APPEND);
+                    }
+                } while ((line = input.readLine()) != null);
+                input.close();
+
+                return false;
+            }
+        } catch (Exception e) {
+            log.error("Got problems with executing: " + e.getMessage());
+            return false;
+        }
+    }
+
+    /**
+     * Install pandoc and set executable path.
+     * @return true if success false otherwise
+     */
+    public static boolean installPandoc() {
+        try {
+            pandocPath = installer.installPandoc().toString();
+            return true;
+        } catch (Exception e) {
+            log.error("Got error: {}", e.getMessage(), e);
+            return false;
+        }
+    }
+
+}

snark-pandoc-plugin/src/main/java/space/kscience/snark/pandoc/ResponseDto.java

@@ -0,0 +1,70 @@
+package space.kscience.snark.pandoc;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+/**
+ * Response from github/releases/latest
+ */
+public class ResponseDto {
+
+    public AssetDto[] getAssets() {
+        return assets;
+    }
+
+    public void setAssets(AssetDto[] assets) {
+        this.assets = assets;
+    }
+
+    /**
+     * @param osSuffix
+     * @return asset appropriate to os
+     */
+    public AssetDto getAssetByOsSuffix(String osSuffix) {
+        for (var asset : assets) {
+            if (asset.getName().contains(osSuffix)) {
+                return asset;
+            }
+        }
+        throw new IllegalArgumentException("Unexpected osSuffix");
+    }
+
+
+    public static class AssetDto {
+
+        @JsonProperty("browser_download_url")
+        private String browserDownloadUrl;
+        private String name;
+
+        public String getBrowserDownloadUrl() {
+            return browserDownloadUrl;
+        }
+
+        public void setBrowserDownloadUrl(String browserDownloadUrl) {
+            this.browserDownloadUrl = browserDownloadUrl;
+        }
+
+        public AssetDto() {}
+
+        public String getName() {
+            return name;
+        }
+
+        public void setName(String name) {
+            this.name = name;
+        }
+    }
+    private AssetDto[] assets;
+
+    public String getTagName() {
+        return tagName;
+    }
+
+    public void setTagName(String tagName) {
+        this.tagName = tagName;
+    }
+
+    @JsonProperty("tag_name")
+    private String tagName;
+
+    public ResponseDto() {}
+}