[WIP] refactor in progress

build.gradle.kts

@@ -6,7 +6,7 @@ plugins {
 
 allprojects {
     group = "space.kscience"
-    version = "0.1.0-dev-1"
+    version = "0.2.0-dev-1"
 
     repositories {
         mavenCentral()
@@ -14,7 +14,7 @@ allprojects {
     }
 }
 
-val dataforgeVersion by extra("0.6.2")
+val dataforgeVersion by extra("0.8.0-dev-1")
 
 ksciencePublish {
     pom("https://github.com/SciProgCentre/snark") {

gradle.properties

@@ -1,3 +1,3 @@
 kotlin.code.style=official
 
-toolsVersion=0.15.0-kotlin-1.9.20
+toolsVersion=0.15.2-kotlin-1.9.21

settings.gradle.kts

@@ -41,5 +41,6 @@ include(
     ":snark-gradle-plugin",
     ":snark-core",
     ":snark-html",
-    ":snark-ktor"
+    ":snark-ktor",
+    ":snark-pandoc"
 )

snark-core/src/commonMain/kotlin/space/kscience/snark/ReWrapAction.kt

@@ -0,0 +1,60 @@
+package space.kscience.snark
+
+import space.kscience.dataforge.actions.AbstractAction
+import space.kscience.dataforge.data.*
+import space.kscience.dataforge.meta.Meta
+import space.kscience.dataforge.meta.MutableMeta
+import space.kscience.dataforge.meta.copy
+import space.kscience.dataforge.names.Name
+import space.kscience.dataforge.names.NameToken
+import space.kscience.dataforge.names.replaceLast
+import kotlin.reflect.KType
+import kotlin.reflect.typeOf
+
+/**
+ * An action to change header (name and meta) without changing the data itself or its computation state
+ */
+public class ReWrapAction<R : Any>(
+    type: KType,
+    private val newMeta: MutableMeta.(name: Name) -> Unit = {},
+    private val newName: (name: Name, meta: Meta?) -> Name,
+) : AbstractAction<R, R>(type) {
+    override fun DataSetBuilder<R>.generate(data: DataSet<R>, meta: Meta) {
+        data.forEach { namedData ->
+            data(
+                newName(namedData.name, namedData.meta),
+                namedData.data.withMeta(namedData.meta.copy { newMeta(namedData.name) })
+            )
+        }
+    }
+
+    override fun DataSourceBuilder<R>.update(dataSet: DataSet<R>, meta: Meta, updateKey: Name) {
+        val datum = dataSet[updateKey]
+        data(
+            newName(updateKey, datum?.meta),
+            datum?.withMeta(datum.meta.copy { newMeta(updateKey) })
+        )
+    }
+
+    public companion object {
+        public inline fun <reified R : Any> removeExtensions(
+            vararg bypassExtensions: String,
+            noinline newMeta: MutableMeta.(name: Name) -> Unit = {},
+        ): ReWrapAction<R> = ReWrapAction(typeOf<R>(), newMeta = newMeta) { name, _ ->
+            name.replaceLast { token ->
+                val extension = token.body.substringAfterLast('.')
+                if (extension in bypassExtensions) {
+                    NameToken(token.body.removeSuffix(".$extension"))
+                } else {
+                    token
+                }
+            }
+        }
+    }
+}
+
+public inline fun <reified R : Any> ReWrapAction(
+    noinline newMeta: MutableMeta.(name: Name) -> Unit = {},
+    noinline newName: (Name, Meta?) -> Name
+): ReWrapAction<R> =    ReWrapAction(typeOf<R>(), newMeta, newName)
+

snark-core/src/commonMain/kotlin/space/kscience/snark/Snark.kt

@@ -0,0 +1,56 @@
+package space.kscience.snark
+
+import kotlinx.io.readByteArray
+import space.kscience.dataforge.context.Context
+import space.kscience.dataforge.context.PluginFactory
+import space.kscience.dataforge.context.PluginTag
+import space.kscience.dataforge.context.gather
+import space.kscience.dataforge.io.IOPlugin
+import space.kscience.dataforge.io.IOReader
+import space.kscience.dataforge.meta.Meta
+import space.kscience.dataforge.meta.get
+import space.kscience.dataforge.meta.string
+import space.kscience.dataforge.names.Name
+import space.kscience.dataforge.names.parseAsName
+import space.kscience.dataforge.workspace.WorkspacePlugin
+
+/**
+ * Represents a Snark workspace plugin.
+ */
+public class Snark : WorkspacePlugin() {
+    public val io: IOPlugin by require(IOPlugin)
+    override val tag: PluginTag get() = Companion.tag
+
+    public val readers: Map<Name, SnarkReader<Any>> by lazy {
+        context.gather<SnarkReader<Any>>(SnarkReader.DF_TYPE, inherit = true)
+    }
+
+    /**
+     * A lazy-initialized map of `TextProcessor` instances used for page-based text transformation.
+     *
+     * @property textProcessors The `TextProcessor` instances accessible by their names.
+     */
+    public val textProcessors: Map<Name, TextProcessor> by lazy {
+        context.gather(TextProcessor.DF_TYPE, true)
+    }
+
+    public fun preprocessor(transformationMeta: Meta): TextProcessor {
+        val transformationName = transformationMeta.string
+            ?: transformationMeta["name"].string ?: error("Transformation name not defined in $transformationMeta")
+        return textProcessors[transformationName.parseAsName()]
+            ?: error("Text transformation with name $transformationName not found in $this")
+    }
+
+    public companion object : PluginFactory<Snark> {
+        override val tag: PluginTag = PluginTag("snark")
+
+        override fun build(context: Context, meta: Meta): Snark = Snark()
+
+        private val byteArrayIOReader = IOReader {
+            readByteArray()
+        }
+
+        internal val byteArraySnarkParser = SnarkReader(byteArrayIOReader)
+
+    }
+}

snark-core/src/commonMain/kotlin/space/kscience/snark/SnarkParser.kt

@@ -1,56 +0,0 @@
-package space.kscience.snark
-
-import io.ktor.utils.io.core.Input
-import io.ktor.utils.io.core.readBytes
-import space.kscience.dataforge.context.Context
-import space.kscience.dataforge.io.IOReader
-import space.kscience.dataforge.io.asBinary
-import space.kscience.dataforge.io.readWith
-import space.kscience.dataforge.meta.Meta
-import space.kscience.dataforge.misc.Type
-import kotlin.reflect.KType
-import kotlin.reflect.typeOf
-
-/**
- * A parser of binary content including priority flag and file extensions
- */
-@Type(SnarkParser.TYPE)
-public interface SnarkParser<out R> {
-    public val type: KType
-
-    public val fileExtensions: Set<String>
-
-    public val priority: Int get() = DEFAULT_PRIORITY
-
-    //TODO use Binary instead of ByteArray
-    public fun parse(context: Context, meta: Meta, bytes: ByteArray): R
-
-    public fun asReader(context: Context, meta: Meta): IOReader<R> = object : IOReader<R> {
-        override val type: KType get() = this@SnarkParser.type
-
-        override fun readObject(input: Input): R = parse(context, meta, input.readBytes())
-    }
-
-    public companion object {
-        public const val TYPE: String = "snark.parser"
-        public const val DEFAULT_PRIORITY: Int = 10
-    }
-}
-
-@PublishedApi
-internal class SnarkParserWrapper<R : Any>(
-    val reader: IOReader<R>,
-    override val type: KType,
-    override val fileExtensions: Set<String>,
-) : SnarkParser<R> {
-    override fun parse(context: Context, meta: Meta, bytes: ByteArray): R = bytes.asBinary().readWith(reader)
-}
-
-/**
- * Create a generic parser from reader
- */
-@Suppress("FunctionName")
-public inline fun <reified R : Any> SnarkParser(
-    reader: IOReader<R>,
-    vararg fileExtensions: String,
-): SnarkParser<R> = SnarkParserWrapper(reader, typeOf<R>(), fileExtensions.toSet())

snark-core/src/commonMain/kotlin/space/kscience/snark/SnarkReader.kt

@@ -0,0 +1,43 @@
+package space.kscience.snark
+
+import space.kscience.dataforge.io.IOReader
+import space.kscience.dataforge.io.asBinary
+import space.kscience.dataforge.misc.DfType
+import space.kscience.snark.SnarkReader.Companion.DEFAULT_PRIORITY
+import space.kscience.snark.SnarkReader.Companion.DF_TYPE
+
+@DfType(DF_TYPE)
+public interface SnarkReader<out T> : IOReader<T> {
+    public val types: Set<String>
+    public val priority: Int get() = DEFAULT_PRIORITY
+    public fun readFrom(source: String): T
+
+    public companion object {
+        public const val DF_TYPE: String = "snark.reader"
+        public const val DEFAULT_PRIORITY: Int = 10
+    }
+}
+
+/**
+ * A wrapper class for IOReader that adds priority and MIME type handling.
+ *
+ * @param T The type of data to be read by the IOReader.
+ * @property reader The underlying IOReader instance used for reading data.
+ * @property types The set of supported types that can be read by the SnarkIOReader.
+ * @property priority The priority of the SnarkIOReader. Higher priority SnarkIOReader instances will be preferred over lower priority ones.
+ */
+
+private class SnarkReaderWrapper<out T>(
+    private val reader: IOReader<T>,
+    override val types: Set<String>,
+    override val priority: Int = DEFAULT_PRIORITY,
+) : IOReader<T> by reader, SnarkReader<T> {
+
+    override fun readFrom(source: String): T = readFrom(source.encodeToByteArray().asBinary())
+}
+
+public fun <T : Any> SnarkReader(
+    reader: IOReader<T>,
+    vararg types: String,
+    priority: Int = DEFAULT_PRIORITY,
+): SnarkReader<T> = SnarkReaderWrapper(reader, types.toSet(), priority)

snark-core/src/commonMain/kotlin/space/kscience/snark/TextProcessor.kt

@@ -0,0 +1,21 @@
+package space.kscience.snark
+
+import space.kscience.dataforge.misc.DfType
+import space.kscience.dataforge.names.NameToken
+
+/**
+ * An object that conducts page-based text transformation. Like using link replacement or templating.
+ */
+@DfType(TextProcessor.DF_TYPE)
+public fun interface TextProcessor {
+
+    public fun process(text: CharSequence): String
+
+    public companion object {
+        public const val DF_TYPE: String = "snark.textTransformation"
+        public val TEXT_TRANSFORMATION_KEY: NameToken = NameToken("transformation")
+        public val TEXT_PREPROCESSOR_KEY: NameToken = NameToken("preprocessor")
+    }
+}
+
+

snark-core/src/jvmMain/kotlin/space/kscience/snark/ImageIOReader.kt

@@ -0,0 +1,21 @@
+package space.kscience.snark
+
+import kotlinx.io.Source
+import kotlinx.io.asInputStream
+import space.kscience.dataforge.io.IOReader
+import java.awt.image.BufferedImage
+import javax.imageio.ImageIO
+import kotlin.reflect.KType
+import kotlin.reflect.typeOf
+
+/**
+ * The ImageIOReader class is an implementation of the IOReader interface specifically for reading images using the ImageIO library.
+ * It reads the image data from a given source and returns a BufferedImage object.
+ *
+ * @property type The KType of the data to be read by the ImageIOReader.
+ */
+public object ImageIOReader : IOReader<BufferedImage> {
+    override val type: KType get() = typeOf<BufferedImage>()
+
+    override fun readFrom(source: Source): BufferedImage = ImageIO.read(source.asInputStream())
+}

snark-core/src/jvmMain/kotlin/space/kscience/snark/extensions.kt

@@ -0,0 +1,16 @@
+package space.kscience.snark
+
+import space.kscience.dataforge.data.DataSet
+import space.kscience.dataforge.data.filterByType
+import space.kscience.dataforge.names.Name
+import space.kscience.dataforge.names.parseAsName
+import space.kscience.dataforge.names.startsWith
+import kotlin.reflect.typeOf
+
+public inline fun <reified R : Any> DataSet<*>.branch(
+    branchName: Name,
+): DataSet<R> = filterByType(typeOf<R>()) { name, _ -> name.startsWith(branchName) }
+
+public inline fun <reified R : Any> DataSet<*>.branch(
+    branchName: String,
+): DataSet<R> = filterByType(typeOf<R>()) { name, _ -> name.startsWith(branchName.parseAsName()) }

snark-core/src/jvmMain/kotlin/space/kscience/snark/snarkWorkspace.kt

@@ -0,0 +1,60 @@
+@file:OptIn(DFExperimental::class)
+
+package space.kscience.snark
+
+import space.kscience.dataforge.data.DataTree
+import space.kscience.dataforge.data.node
+import space.kscience.dataforge.io.Binary
+import space.kscience.dataforge.io.IOPlugin
+import space.kscience.dataforge.meta.*
+import space.kscience.dataforge.misc.DFExperimental
+import space.kscience.dataforge.workspace.Workspace
+import space.kscience.dataforge.workspace.WorkspaceBuilder
+import space.kscience.dataforge.workspace.readRawDirectory
+import kotlin.io.path.Path
+import kotlin.io.path.toPath
+
+
+/**
+ * Reads the specified resources and returns a [DataTree] containing the data.
+ *
+ * @param resources The names of the resources to read.
+ * @param classLoader The class loader to use for loading the resources. By default, it uses the current thread's context class loader.
+ * @return A DataTree containing the data read from the resources.
+ */
+private fun IOPlugin.readResources(
+    vararg resources: String,
+    classLoader: ClassLoader = Thread.currentThread().contextClassLoader,
+): DataTree<Binary> = DataTree {
+    //    require(resource.isNotBlank()) {"Can't mount root resource tree as data root"}
+    resources.forEach { resource ->
+        val path = classLoader.getResource(resource)?.toURI()?.toPath() ?: error(
+            "Resource with name $resource is not resolved"
+        )
+        node(resource, readRawDirectory(path))
+    }
+}
+
+public fun Snark.workspace(
+    meta: Meta,
+    workspaceBuilder: WorkspaceBuilder.() -> Unit = {},
+): Workspace = Workspace {
+
+
+    data {
+        meta.getIndexed("directory").forEach { (index, directoryMeta) ->
+            val dataDirectory = directoryMeta["path"].string ?: error("Directory path not defined")
+            val nodeName = directoryMeta["name"].string ?: directoryMeta.string ?: index ?: ""
+            val data = io.readRawDirectory(Path(dataDirectory))
+            node(nodeName, data)
+        }
+        meta.getIndexed("resource").forEach { (index, resourceMeta) ->
+            val resource = resourceMeta["path"]?.stringList ?: listOf("/")
+            val nodeName = resourceMeta["name"].string ?: resourceMeta.string ?: index ?: ""
+            val data: DataTree<Binary> = io.readResources(*resource.toTypedArray())
+            node(nodeName, data)
+        }
+    }
+
+    workspaceBuilder()
+}

snark-html/build.gradle.kts

@@ -1,5 +1,5 @@
 plugins {
-    id("space.kscience.gradle.jvm")
+    id("space.kscience.gradle.mpp")
     `maven-publish`
 }
 
@@ -7,21 +7,22 @@ val dataforgeVersion: String by rootProject.extra
 val ktorVersion = space.kscience.gradle.KScienceVersions.ktorVersion
 
 kscience{
+    jvm()
     useContextReceivers()
-}
-
-dependencies {
+    commonMain{
         api(projects.snarkCore)
 
-    api("org.jetbrains.kotlinx:kotlinx-html:0.8.0")
+        api(spclibs.kotlinx.html)
         api("org.jetbrains.kotlin-wrappers:kotlin-css")
 
-    api("io.ktor:ktor-utils:$ktorVersion")
-
+        api("io.ktor:ktor-http:$ktorVersion")
         api("space.kscience:dataforge-io-yaml:$dataforgeVersion")
-    api("org.jetbrains:markdown:0.4.0")
+        api("org.jetbrains:markdown:0.5.2")
     }
 
+}
+
+
 readme {
     maturity = space.kscience.gradle.Maturity.EXPERIMENTAL
     feature("data") { "Data-based processing. Instead of traditional layout-based" }

snark-html/src/jvmMain/kotlin/space/kscience/snark/html/HtmlPage.kt

@@ -0,0 +1,67 @@
+package space.kscience.snark.html
+
+import kotlinx.html.HTML
+import kotlinx.html.stream.createHTML
+import kotlinx.html.visitTagAndFinalize
+import space.kscience.dataforge.data.DataSet
+import space.kscience.dataforge.data.DataSetBuilder
+import space.kscience.dataforge.data.static
+import space.kscience.dataforge.meta.Meta
+import space.kscience.dataforge.names.Name
+
+public fun interface HtmlPage {
+
+    context(PageContextWithData, HTML)
+    public fun renderPage()
+
+    public companion object {
+        public fun createHtmlString(
+            pageContext: PageContext,
+            dataSet: DataSet<*>,
+            page: HtmlPage,
+        ): String = createHTML().run {
+            HTML(kotlinx.html.emptyMap, this, null).visitTagAndFinalize(this) {
+                with(PageContextWithData(pageContext, dataSet)) {
+                    with(page) {
+                        renderPage()
+                    }
+                }
+            }
+        }
+    }
+}
+
+
+// data builders
+
+public fun DataSetBuilder<Any>.page(
+    name: Name,
+    pageMeta: Meta = Meta.EMPTY,
+    block: context(PageContextWithData) HTML.() -> Unit,
+) {
+    val page = HtmlPage(block)
+    static<HtmlPage>(name, page, pageMeta)
+}
+
+
+//                if (data.type == typeOf<HtmlData>()) {
+//                    val languageMeta: Meta = Language.forName(name)
+//
+//                    val dataMeta: Meta = if (languageMeta.isEmpty()) {
+//                        data.meta
+//                    } else {
+//                        data.meta.toMutableMeta().apply {
+//                            "languages" put languageMeta
+//                        }
+//                    }
+//
+//                    page(name, dataMeta) { pageContext->
+//                        head {
+//                            title = dataMeta["title"].string ?: "Untitled page"
+//                        }
+//                        body {
+//                            @Suppress("UNCHECKED_CAST")
+//                            htmlData(pageContext, data as HtmlData)
+//                        }
+//                    }
+//                }

snark-html/src/jvmMain/kotlin/space/kscience/snark/html/HtmlSite.kt

@@ -0,0 +1,37 @@
+package space.kscience.snark.html
+
+import space.kscience.dataforge.data.DataSet
+import space.kscience.dataforge.data.DataSetBuilder
+import space.kscience.dataforge.data.static
+import space.kscience.dataforge.meta.Meta
+import space.kscience.dataforge.meta.getIndexed
+import space.kscience.dataforge.meta.string
+import space.kscience.dataforge.names.Name
+import space.kscience.dataforge.names.asName
+import space.kscience.dataforge.names.parseAsName
+
+public fun interface HtmlSite {
+    context(SiteContextWithData)
+    public fun renderSite()
+}
+
+public fun DataSetBuilder<Any>.site(
+    name: Name,
+    siteMeta: Meta,
+    block: (siteContext: SiteContext, data: DataSet<Any>) -> Unit,
+) {
+    static(name, HtmlSite { block(site, siteData) }, siteMeta)
+}
+
+//public fun DataSetBuilder<Any>.site(name: Name, block: DataSetBuilder<Any>.() -> Unit) {
+//    node(name, block)
+//}
+
+internal fun DataSetBuilder<Any>.assetsFrom(rootMeta: Meta) {
+    rootMeta.getIndexed("file".asName()).forEach { (_, meta) ->
+        val webName: String? by meta.string()
+        val name by meta.string { error("File path is not provided") }
+        val fileName = name.parseAsName()
+        static(fileName, webName?.parseAsName() ?: fileName)
+    }
+}

snark-html/src/jvmMain/kotlin/space/kscience/snark/html/Language.kt

@@ -0,0 +1,158 @@
+package space.kscience.snark.html
+
+import space.kscience.dataforge.data.DataSet
+import space.kscience.dataforge.data.branch
+import space.kscience.dataforge.meta.*
+import space.kscience.dataforge.names.Name
+import space.kscience.dataforge.names.asName
+import space.kscience.dataforge.names.parseAsName
+import space.kscience.dataforge.names.plus
+import space.kscience.snark.SnarkBuilder
+import space.kscience.snark.html.Language.Companion.SITE_LANGUAGES_KEY
+import space.kscience.snark.html.Language.Companion.SITE_LANGUAGE_KEY
+
+
+public class Language : Scheme() {
+
+    /**
+     * Language key override
+     */
+    public var key: String? by string()
+
+    /**
+     * Page name prefix
+     */
+    public var prefix: String? by string()
+
+    /**
+     * An override for data path. By default uses [prefix]
+     */
+    public var dataPath: String? by string()
+
+    /**
+     * Target page name with a given language key
+     */
+    public var target: Name?
+        get() = meta["target"].string?.parseAsName(false)
+        set(value) {
+            meta["target"] = value?.toString()?.asValue()
+        }
+
+    public companion object : SchemeSpec<Language>(::Language) {
+
+        public val LANGUAGE_KEY: Name = "language".asName()
+
+        public val LANGUAGE_MAP_KEY: Name = "languageMap".asName()
+
+        public val SITE_LANGUAGE_KEY: Name = SiteContext.SITE_META_KEY + LANGUAGE_KEY
+
+        public val SITE_LANGUAGES_KEY: Name = SiteContext.SITE_META_KEY + LANGUAGE_MAP_KEY
+
+        public const val DEFAULT_LANGUAGE: String = "en"
+
+//        /**
+//         * Automatically build a language map for a data piece with given [name] based on existence of appropriate data nodes.
+//         */
+//        context(PageContextWithData)
+//        public fun languageMapFor(name: Name): Meta = Meta {
+//            val currentLanguagePrefix = languages[language]?.get(Language::prefix.name)?.string ?: language
+//            val fullName = (site.route.removeFirstOrNull(currentLanguagePrefix.asName()) ?: site.route) + name
+//            languages.forEach { (key, meta) ->
+//                val languagePrefix: String = meta[Language::prefix.name].string ?: key
+//                val nameWithLanguage: Name = if (languagePrefix.isBlank()) {
+//                    fullName
+//                } else {
+//                    languagePrefix.asName() + fullName
+//                }
+//                if (data.resolveHtmlOrNull(name) != null) {
+//                    key put meta.asMutableMeta().apply {
+//                        Language::target.name put nameWithLanguage.toString()
+//                    }
+//                }
+//            }
+//        }
+    }
+}
+
+public fun Language(prefix: String): Language = Language { this.prefix = prefix }
+
+public val SiteContext.languages: Map<String, Meta>
+    get() = siteMeta[SITE_LANGUAGES_KEY]?.items?.mapKeys { it.key.toStringUnescaped() } ?: emptyMap()
+
+public val SiteContext.language: String
+    get() = siteMeta[SITE_LANGUAGE_KEY].string ?: Language.DEFAULT_LANGUAGE
+
+public val SiteContext.languagePrefix: Name
+    get() = languages[language]?.let { it[Language::prefix.name].string ?: language }?.parseAsName() ?: Name.EMPTY
+
+/**
+ * Create a multiple sites for different languages. All sites use the same [content], but rely on different data
+ *
+ * @param data a common data root for all sites
+ */
+@SnarkBuilder
+public fun SiteContext.multiLanguageSite(
+    data: DataSet<*>,
+    languageMap: Map<String, Language>,
+    content: HtmlSite,
+) {
+    languageMap.forEach { (languageKey, language) ->
+        val prefix = language.prefix ?: languageKey
+        val languageSiteMeta = Meta {
+            SITE_LANGUAGE_KEY put languageKey
+            SITE_LANGUAGES_KEY put Meta {
+                languageMap.forEach {
+                    it.key put it.value
+                }
+            }
+        }
+        site(
+            prefix.parseAsName(),
+            data.branch(language.dataPath ?: prefix),
+            siteMeta = Laminate(languageSiteMeta, siteMeta),
+            content
+        )
+    }
+}
+
+/**
+ * The language key of this page
+ */
+public val PageContext.language: String
+    get() = pageMeta[Language.LANGUAGE_KEY]?.string ?: pageMeta[SITE_LANGUAGE_KEY]?.string ?: Language.DEFAULT_LANGUAGE
+
+/**
+ * Mapping of language keys to other language versions of this page
+ */
+public fun PageContext.getLanguageMap(): Map<String, Meta> =
+    pageMeta[Language.LANGUAGE_MAP_KEY]?.items?.mapKeys { it.key.toStringUnescaped() } ?: emptyMap()
+
+public fun PageContext.localisedPageRef(pageName: Name, relative: Boolean = false): String {
+    val prefix = getLanguageMap()[language]?.get(Language::prefix.name)?.string?.parseAsName() ?: Name.EMPTY
+    return resolvePageRef(prefix + pageName, relative)
+}
+
+//
+///**
+// * Render all pages in a node with given name. Use localization prefix if appropriate data is available.
+// */
+//public fun SiteContext.localizedPages(
+//    dataPath: Name,
+//    remotePath: Name = dataPath,
+//    dataRenderer: DataRenderer = DataRenderer.DEFAULT,
+//) {
+//    val item = resolveData.getItem(languagePrefix + dataPath)
+//        ?: resolveData.getItem(dataPath)
+//        ?: error("No data found by name $dataPath")
+//    route(remotePath) {
+//        pages(item, dataRenderer)
+//    }
+//}
+//
+//public fun SiteContext.localizedPages(
+//    dataPath: String,
+//    remotePath: Name = dataPath.parseAsName(),
+//    dataRenderer: DataRenderer = DataRenderer.DEFAULT,
+//) {
+//    localizedPages(dataPath.parseAsName(), remotePath, dataRenderer = dataRenderer)
+//}

snark-html/src/jvmMain/kotlin/space/kscience/snark/html/MetaMaskData.kt

@@ -0,0 +1,20 @@
+package space.kscience.snark.html
+
+import space.kscience.dataforge.data.Data
+import space.kscience.dataforge.meta.Meta
+import space.kscience.dataforge.meta.MutableMeta
+import space.kscience.dataforge.meta.copy
+
+
+private class MetaMaskData<T>(val origin: Data<T>, override val meta: Meta) : Data<T> by origin
+
+/**
+ * A data with overriden meta. It reflects original data computed state.
+ */
+public fun <T> Data<T>.withMeta(newMeta: Meta): Data<T> = if (this is MetaMaskData) {
+    MetaMaskData(origin, newMeta)
+} else {
+    MetaMaskData(this, newMeta)
+}
+
+public inline fun <T> Data<T>.mapMeta(block: MutableMeta.() -> Unit): Data<T> = withMeta(meta.copy(block))

snark-html/src/jvmMain/kotlin/space/kscience/snark/html/PageContext.kt

@@ -0,0 +1,60 @@
+package space.kscience.snark.html
+
+import space.kscience.dataforge.data.DataSet
+import space.kscience.dataforge.meta.Meta
+import space.kscience.dataforge.meta.get
+import space.kscience.dataforge.meta.string
+import space.kscience.dataforge.names.Name
+import space.kscience.dataforge.names.asName
+import space.kscience.dataforge.names.hasIndex
+import space.kscience.dataforge.names.parseAsName
+import space.kscience.snark.SnarkBuilder
+import space.kscience.snark.SnarkContext
+
+public fun Name.toWebPath(): String = tokens.joinToString(separator = "/") {
+    if (it.hasIndex()) {
+        "${it.body}[${it.index}]"
+    } else {
+        it.body
+    }
+}
+
+/**
+ *  A context for building a single page
+ */
+@SnarkBuilder
+public interface PageContext :  SnarkContext {
+
+    public val site: SiteContext
+
+    /**
+     * A metadata for a page. It should include site metadata
+     */
+    public val pageMeta: Meta
+
+    /**
+     * Resolve absolute url for given [ref]
+     *
+     */
+    public fun resolveRef(ref: String): String
+
+    /**
+     * Resolve absolute url for a page with given [pageName].
+     *
+     * @param relative if true, add [SiteContext] route to the absolute page name
+     */
+    public fun resolvePageRef(pageName: Name, relative: Boolean = false): String
+}
+
+context(PageContext)
+public val page: PageContext
+    get() = this@PageContext
+
+public fun PageContext.resolvePageRef(pageName: String): String = resolvePageRef(pageName.parseAsName())
+
+public val PageContext.homeRef: String get() = resolvePageRef(SiteContext.INDEX_PAGE_TOKEN.asName())
+
+public val PageContext.name: Name? get() = pageMeta["name"].string?.parseAsName()
+
+
+public class PageContextWithData(private val pageContext: PageContext, public val data: DataSet<*>): PageContext by pageContext

snark-html/src/jvmMain/kotlin/space/kscience/snark/html/PageFragment.kt

@@ -0,0 +1,90 @@
+package space.kscience.snark.html
+
+import kotlinx.coroutines.runBlocking
+import kotlinx.html.FlowContent
+import space.kscience.dataforge.data.*
+import space.kscience.dataforge.meta.Meta
+import space.kscience.dataforge.meta.get
+import space.kscience.dataforge.meta.int
+import space.kscience.dataforge.meta.string
+import space.kscience.dataforge.names.Name
+import space.kscience.dataforge.names.parseAsName
+import space.kscience.dataforge.names.plus
+import space.kscience.dataforge.names.startsWith
+import space.kscience.snark.SnarkContext
+
+public fun interface PageFragment {
+
+    context(PageContextWithData)
+    public fun FlowContent.renderFragment()
+}
+
+context(PageContextWithData)
+public fun FlowContent.fragment(fragment: PageFragment): Unit{
+    with(fragment) {
+        renderFragment()
+    }
+}
+
+
+context(PageContextWithData)
+public fun FlowContent.fragment(data: Data<PageFragment>): Unit = runBlocking {
+    fragment(data.await())
+}
+
+
+context(SnarkContext)
+public val Data<*>.id: String
+    get() = meta["id"]?.string ?: "block[${hashCode()}]"
+
+context(SnarkContext)
+public val Data<*>.language: String?
+    get() = meta["language"].string?.lowercase()
+
+context(SnarkContext)
+public val Data<*>.order: Int?
+    get() = meta["order"]?.int
+
+context(SnarkContext)
+public val Data<*>.published: Boolean
+    get() = meta["published"].string != "false"
+
+
+/**
+ * Resolve a Html builder by its full name
+ */
+context(SnarkContext)
+public fun DataSet<*>.resolveHtmlOrNull(name: Name): Data<PageFragment>? {
+    val resolved = (getByType<PageFragment>(name) ?: getByType<PageFragment>(name + SiteContext.INDEX_PAGE_TOKEN))
+
+    return resolved?.takeIf {
+        it.published //TODO add language confirmation
+    }
+}
+
+context(SnarkContext)
+public fun DataSet<*>.resolveHtmlOrNull(name: String): Data<PageFragment>? = resolveHtmlOrNull(name.parseAsName())
+
+context(SnarkContext)
+public fun DataSet<*>.resolveHtml(name: String): Data<PageFragment> = resolveHtmlOrNull(name)
+    ?: error("Html fragment with name $name is not resolved")
+
+/**
+ * Find all Html blocks using given name/meta filter
+ */
+context(SnarkContext)
+public fun DataSet<*>.resolveAllHtml(
+    predicate: (name: Name, meta: Meta) -> Boolean,
+): Map<Name, Data<PageFragment>> = filterByType<PageFragment> { name, meta ->
+    predicate(name, meta)
+            && meta["published"].string != "false"
+    //TODO add language confirmation
+}.asSequence().associate { it.name to it.data }
+
+context(SnarkContext)
+public fun DataSet<*>.findHtmlByContentType(
+    contentType: String,
+    baseName: Name = Name.EMPTY,
+): Map<Name, Data<PageFragment>> = resolveAllHtml { name, meta ->
+    name.startsWith(baseName) && meta["content_type"].string == contentType
+}

snark-html/src/jvmMain/kotlin/space/kscience/snark/html/Postprocessor.kt

@@ -0,0 +1,85 @@
+package space.kscience.snark.html
+
+import kotlinx.html.*
+import space.kscience.dataforge.meta.string
+import space.kscience.dataforge.names.parseAsName
+import space.kscience.snark.TextProcessor
+
+public class WebPageTextProcessor(private val page: PageContext) : TextProcessor {
+    private val regex = """\$\{([\w.]*)(?>\("(.*)"\))?}""".toRegex()
+
+    /**
+     * A basic [TextProcessor] that replaces `${...}` expressions in text. The following expressions are recognised:
+     * * `homeRef` resolves to [homeRef]
+     * * `resolveRef("...")` -> [PageContext.resolveRef]
+     * * `resolvePageRef("...")` -> [PageContext.resolvePageRef]
+     * * `pageMeta.get("...") -> [PageContext.pageMeta] get string method
+     * Otherwise return unchanged string
+     */
+    override fun process(text: CharSequence): String = text.replace(regex) { match ->
+        when (match.groups[1]!!.value) {
+            "homeRef" -> page.homeRef
+            "resolveRef" -> {
+                val refString = match.groups[2]?.value ?: error("resolveRef requires a string (quoted) argument")
+                page.resolveRef(refString)
+            }
+
+            "resolvePageRef" -> {
+                val refString = match.groups[2]?.value
+                    ?: error("resolvePageRef requires a string (quoted) argument")
+                page.localisedPageRef(refString.parseAsName())
+            }
+
+            "pageMeta.get" -> {
+                val nameString = match.groups[2]?.value
+                    ?: error("resolvePageRef requires a string (quoted) argument")
+                page.pageMeta[nameString.parseAsName()].string ?: "@null"
+            }
+
+            else -> match.value
+        }
+    }
+
+}
+
+/**
+ * A tag consumer wrapper that wraps existing [TagConsumer] and adds postprocessing.
+ *
+ */
+public class Postprocessor<out R>(
+    public val page: PageContext,
+    private val consumer: TagConsumer<R>,
+    private val processor: TextProcessor = WebPageTextProcessor(page),
+) : TagConsumer<R> by consumer {
+
+    override fun onTagAttributeChange(tag: Tag, attribute: String, value: String?) {
+        if (tag is A && attribute == "href" && value != null) {
+            consumer.onTagAttributeChange(tag, attribute, processor.process(value))
+        } else {
+            consumer.onTagAttributeChange(tag, attribute, value)
+        }
+    }
+
+    override fun onTagContent(content: CharSequence) {
+        consumer.onTagContent(processor.process(content))
+    }
+
+    override fun onTagContentUnsafe(block: Unsafe.() -> Unit) {
+        val proxy = object :Unsafe{
+            override fun String.unaryPlus() {
+                consumer.onTagContentUnsafe {
+                    processor.process(this@unaryPlus).unaryPlus()
+                }
+            }
+        }
+        proxy.block()
+    }
+}
+
+context(PageContext)
+public inline fun FlowContent.postprocess(block: FlowContent.() -> Unit) {
+    val fc = object : FlowContent by this {
+        override val consumer: TagConsumer<*> = Postprocessor(page, this@postprocess.consumer)
+    }
+    fc.block()
+}

snark-html/src/jvmMain/kotlin/space/kscience/snark/html/SiteContext.kt

@@ -0,0 +1,233 @@
+package space.kscience.snark.html
+
+import space.kscience.dataforge.data.*
+import space.kscience.dataforge.io.Binary
+import space.kscience.dataforge.meta.Meta
+import space.kscience.dataforge.meta.get
+import space.kscience.dataforge.meta.string
+import space.kscience.dataforge.names.*
+import space.kscience.snark.SnarkBuilder
+import space.kscience.snark.SnarkContext
+import kotlin.reflect.full.isSubtypeOf
+import kotlin.reflect.typeOf
+
+
+/**
+ * An abstraction, which is used to render sites to the different rendering engines
+ */
+@SnarkBuilder
+public interface SiteContext : SnarkContext {
+
+    /**
+     * Route name of this [SiteContext] relative to the site root
+     */
+    public val route: Name
+
+    /**
+     * Site configuration
+     */
+    public val siteMeta: Meta
+
+    /**
+     * Renders a static file or resource for the given route and data.
+     *
+     * @param route The route name of the static file relative to the site root.
+     * @param data The data object containing the binary data for the static file.
+     */
+    public fun static(route: Name, data: Data<Binary>)
+
+
+    /**
+     * Create a single page at given [route]. If the route is empty, create an index page the current route.
+     *
+     * @param pageMeta additional page meta. [PageContext] will use both it and [siteMeta]
+     */
+    @SnarkBuilder
+    public fun page(
+        route: Name,
+        data: DataSet<*>,
+        pageMeta: Meta = Meta.EMPTY,
+        content: HtmlPage,
+    )
+
+    /**
+     * Create a route block with its own data. Does not change base url
+     */
+    @SnarkBuilder
+    public fun route(
+        route: Name,
+        data: DataSet<*>,
+        siteMeta: Meta = Meta.EMPTY,
+        content: HtmlSite,
+    )
+
+    /**
+     * Creates a sub-site and sets it as site base url
+     * @param route mount site at [rootName]
+     */
+    @SnarkBuilder
+    public fun site(
+        route: Name,
+        data: DataSet<*>,
+        siteMeta: Meta = Meta.EMPTY,
+        content: HtmlSite,
+    )
+
+
+    public companion object {
+        public val SITE_META_KEY: Name = "site".asName()
+        public val INDEX_PAGE_TOKEN: NameToken = NameToken("index")
+        public val UP_PAGE_TOKEN: NameToken = NameToken("..")
+    }
+}
+
+public fun SiteContext.static(dataSet: DataSet<Binary>, prefix: Name = Name.EMPTY) {
+    dataSet.forEach { (name, data) ->
+        static(prefix + name, data)
+    }
+}
+
+
+public fun SiteContext.static(dataSet: DataSet<*>, branch: String, prefix: String = branch) {
+    val branchName = branch.parseAsName()
+    val prefixName = prefix.parseAsName()
+    val binaryType = typeOf<Binary>()
+    dataSet.forEach { (name, data) ->
+        @Suppress("UNCHECKED_CAST")
+        if (name.startsWith(branchName) && data.type.isSubtypeOf(binaryType)) {
+            static(prefixName + name, data as Data<Binary>)
+        }
+    }
+}
+
+
+
+context(SiteContext)
+public val site: SiteContext
+    get() = this@SiteContext
+
+
+/**
+ * A wrapper for site context that allows convenient site building experience
+ */
+public class SiteContextWithData(private val site: SiteContext, public val siteData: DataSet<*>) : SiteContext by site
+
+
+@SnarkBuilder
+public fun SiteContextWithData.static(branch: String, prefix: String = branch): Unit = static(siteData, branch, prefix)
+
+
+@SnarkBuilder
+public fun SiteContextWithData.page(
+    route: Name = Name.EMPTY,
+    pageMeta: Meta = Meta.EMPTY,
+    content: HtmlPage,
+): Unit = page(route, siteData, pageMeta, content)
+
+@SnarkBuilder
+public fun SiteContextWithData.route(
+    route: String,
+    data: DataSet<*> = siteData,
+    siteMeta: Meta = Meta.EMPTY,
+    content: HtmlSite,
+): Unit = route(route.parseAsName(), data, siteMeta,content)
+
+@SnarkBuilder
+public fun SiteContextWithData.site(
+    route: String,
+    data: DataSet<*> = siteData,
+    siteMeta: Meta = Meta.EMPTY,
+    content: HtmlSite,
+): Unit = site(route.parseAsName(), data, siteMeta,content)
+
+/**
+ * Render all pages and sites found in the data
+ */
+public suspend fun SiteContext.renderPages(data: DataSet<*>): Unit {
+
+    // Render all sub-sites
+    data.filterByType<HtmlSite>().forEach { siteData: NamedData<HtmlSite> ->
+        // generate a sub-site context and render the data in sub-site context
+        val dataPrefix = siteData.meta["site.dataPath"].string?.asName() ?: Name.EMPTY
+        site(
+            route = siteData.meta["site.route"].string?.asName() ?: siteData.name,
+            data.branch(dataPrefix),
+            siteMeta = siteData.meta,
+            siteData.await()
+        )
+    }
+
+    // Render all stand-alone pages in default site
+    data.filterByType<HtmlPage>().forEach { pageData: NamedData<HtmlPage> ->
+        val dataPrefix = pageData.meta["page.dataPath"].string?.asName() ?: Name.EMPTY
+        page(
+            route = pageData.meta["page.route"].string?.asName() ?: pageData.name,
+            data.branch(dataPrefix),
+            pageMeta = pageData.meta,
+            pageData.await()
+        )
+    }
+}
+
+
+//
+///**
+// * Recursively renders the data items in [data]. If [LAYOUT_KEY] is defined in an item, use it to load
+// * layout from the context, otherwise render children nodes as name segments and individual data items using [dataRenderer].
+// */
+//public fun SiteContext.pages(
+//    dataRenderer: DataRenderer = DataRenderer.DEFAULT,
+//) {
+//    val layoutMeta = siteData().meta[LAYOUT_KEY]
+//    if (layoutMeta != null) {
+//        //use layout if it is defined
+//        snark.siteLayout(layoutMeta).render(siteData())
+//    } else {
+//        when (siteData()) {
+//            is DataTreeItem.Node -> {
+//                siteData().tree.items.forEach { (token, item) ->
+//                    //Don't apply index token
+//                    if (token == SiteLayout.INDEX_PAGE_TOKEN) {
+//                        pages(item, dataRenderer)
+//                    } else if (item is DataTreeItem.Leaf) {
+//                        dataRenderer(token.asName(), item.data)
+//                    } else {
+//                        route(token.asName()) {
+//                            pages(item, dataRenderer)
+//                        }
+//                    }
+//                }
+//            }
+//
+//            is DataTreeItem.Leaf -> {
+//                dataRenderer(Name.EMPTY, siteData().data)
+//            }
+//        }
+//        siteData().meta[SiteLayout.ASSETS_KEY]?.let {
+//            assetsFrom(it)
+//        }
+//    }
+//    //TODO watch for changes
+//}
+//
+///**
+// * Render all pages in a node with given name
+// */
+//public fun SiteContext.pages(
+//    dataPath: Name,
+//    remotePath: Name = dataPath,
+//    dataRenderer: DataRenderer = DataRenderer.DEFAULT,
+//) {
+//    val item = resolveData.getItem(dataPath) ?: error("No data found by name $dataPath")
+//    route(remotePath) {
+//        pages(item, dataRenderer)
+//    }
+//}
+//
+//public fun SiteContext.pages(
+//    dataPath: String,
+//    remotePath: Name = dataPath.parseAsName(),
+//    dataRenderer: DataRenderer = DataRenderer.DEFAULT,
+//) {
+//    pages(dataPath.parseAsName(), remotePath, dataRenderer = dataRenderer)
+//}

snark-html/src/jvmMain/kotlin/space/kscience/snark/html/SnarkHtml.kt

@@ -0,0 +1,150 @@
+@file:OptIn(DFExperimental::class)
+
+package space.kscience.snark.html
+
+import io.ktor.http.ContentType
+import kotlinx.coroutines.CoroutineScope
+import kotlinx.io.readByteArray
+import space.kscience.dataforge.actions.Action
+import space.kscience.dataforge.actions.mapping
+import space.kscience.dataforge.context.Context
+import space.kscience.dataforge.context.PluginFactory
+import space.kscience.dataforge.context.PluginTag
+import space.kscience.dataforge.data.*
+import space.kscience.dataforge.io.*
+import space.kscience.dataforge.io.yaml.YamlMetaFormat
+import space.kscience.dataforge.io.yaml.YamlPlugin
+import space.kscience.dataforge.meta.Meta
+import space.kscience.dataforge.meta.get
+import space.kscience.dataforge.meta.set
+import space.kscience.dataforge.meta.string
+import space.kscience.dataforge.misc.DFExperimental
+import space.kscience.dataforge.names.Name
+import space.kscience.dataforge.names.asName
+import space.kscience.dataforge.provider.dfType
+import space.kscience.dataforge.workspace.*
+import space.kscience.snark.ReWrapAction
+import space.kscience.snark.Snark
+import space.kscience.snark.SnarkReader
+import space.kscience.snark.TextProcessor
+import java.net.URLConnection
+import kotlin.io.path.Path
+import kotlin.io.path.extension
+
+
+public fun <T : Any, R : Any> DataSet<T>.transform(action: Action<T, R>, meta: Meta = Meta.EMPTY): DataSet<R> =
+    action.execute(this, meta)
+
+public fun <T : Any> DataSetBuilder<T>.fill(dataSet: DataSet<T>) {
+    node(Name.EMPTY, dataSet)
+}
+
+/**
+ * A plugin used for rendering a [DataTree] as HTML
+ */
+public class SnarkHtml : WorkspacePlugin() {
+    private val snark by require(Snark)
+    private val yaml by require(YamlPlugin)
+    public val io: IOPlugin get() = snark.io
+
+    override val tag: PluginTag get() = Companion.tag
+
+    override fun content(target: String): Map<Name, Any> = when (target) {
+        SnarkReader::class.dfType -> mapOf(
+            "html".asName() to HtmlReader,
+            "markdown".asName() to MarkdownReader,
+            "json".asName() to SnarkReader(JsonMetaFormat, ContentType.Application.Json.toString()),
+            "yaml".asName() to SnarkReader(YamlMetaFormat, "text/yaml", "yaml"),
+        )
+
+        else -> super.content(target)
+    }
+
+    private fun getContentType(name: Name, meta: Meta): String = meta[CONTENT_TYPE_KEY].string ?: run {
+        val filePath = meta[FileData.FILE_PATH_KEY]?.string ?: name.toString()
+        URLConnection.guessContentTypeFromName(filePath) ?: Path(filePath).extension
+    }
+
+
+    public val prepareHeaderAction: Action<Any, Any> = ReWrapAction.removeExtensions("html", "md") { name ->
+        val contentType = getContentType(name, this)
+        set(CONTENT_TYPE_KEY, contentType)
+    }
+
+    public val parseAction: Action<Any, Any> = Action.mapping {
+        val contentType = getContentType(name, meta)
+
+        val parser = snark.readers.values.filter { parser ->
+            contentType in parser.types
+        }.maxByOrNull {
+            it.priority
+        }
+        //pass data for which parser is not found
+        if (parser == null) {
+            result { it }
+        } else {
+
+            val preprocessor = meta[TextProcessor.TEXT_PREPROCESSOR_KEY]?.let { snark.preprocessor(it) }
+            result {
+
+                when (it) {
+                    is CharSequence -> {
+                        val string = it.toString()
+                        val preprocessed = preprocessor?.process(string) ?: string
+                        parser.readFrom(preprocessed)
+                    }
+
+                    is Binary -> {
+                        if (preprocessor == null) {
+                            parser.readFrom(it)
+                        } else {
+                            //TODO provide encoding
+                            val string = it.toByteArray().decodeToString()
+                            parser.readFrom(preprocessor.process(string))
+                        }
+                    }
+
+                    // bypass for non textual-data
+                    else -> it
+                }
+            }
+        }
+    }
+
+    public val parse: TaskReference<Any> by task<Any>({
+        description = "Parse all data for which reader is resolved"
+    }) {
+        fill(from(allData).transform(parseAction, taskMeta))
+    }
+
+
+    public companion object : PluginFactory<SnarkHtml> {
+        override val tag: PluginTag = PluginTag("snark.html")
+
+        public val CONTENT_TYPE_KEY: Name = "contentType".asName()
+
+        override fun build(context: Context, meta: Meta): SnarkHtml = SnarkHtml()
+
+        private val byteArrayIOReader = IOReader {
+            readByteArray()
+        }
+
+        internal val byteArraySnarkParser = SnarkReader(byteArrayIOReader)
+
+    }
+}
+
+
+public fun SnarkHtml.readSiteData(
+    binaries: DataSource<Binary>,
+    meta: Meta = Meta.EMPTY,
+): DataSet<Any> = binaries.transform(parseAction, meta)
+
+
+public fun SnarkHtml.readSiteData(
+    coroutineScope: CoroutineScope,
+    meta: Meta = Meta.EMPTY,
+    builder: context(IOPlugin)  DataSourceBuilder<Any>.() -> Unit,
+): DataSet<Any> = DataSource(coroutineScope) { builder(io, this) }
+    .transform(prepareHeaderAction, meta)
+    .transform(parseAction, meta)

snark-html/src/jvmMain/kotlin/space/kscience/snark/html/readers.kt

@@ -0,0 +1,51 @@
+package space.kscience.snark.html
+
+import kotlinx.html.div
+import kotlinx.html.unsafe
+import kotlinx.io.Source
+import kotlinx.io.readString
+import org.intellij.markdown.flavours.commonmark.CommonMarkFlavourDescriptor
+import org.intellij.markdown.html.HtmlGenerator
+import org.intellij.markdown.parser.MarkdownParser
+import space.kscience.snark.SnarkReader
+import kotlin.reflect.KType
+import kotlin.reflect.typeOf
+
+public object HtmlReader : SnarkReader<PageFragment> {
+    override val types: Set<String> = setOf("html")
+
+    override fun readFrom(source: String): PageFragment = PageFragment {
+        div {
+            unsafe { +source }
+        }
+    }
+
+    override fun readFrom(source: Source): PageFragment = readFrom(source.readString())
+    override val type: KType = typeOf<PageFragment>()
+}
+
+public object MarkdownReader : SnarkReader<PageFragment> {
+    override val type: KType = typeOf<PageFragment>()
+
+    override val types: Set<String> = setOf("text/markdown", "md", "markdown")
+
+    override fun readFrom(source: String): PageFragment = PageFragment {
+        val parsedTree = markdownParser.buildMarkdownTreeFromString(source)
+        val htmlString = HtmlGenerator(source, parsedTree, markdownFlavor).generateHtml()
+
+        div {
+            unsafe {
+                +htmlString
+            }
+        }
+    }
+
+    private val markdownFlavor = CommonMarkFlavourDescriptor()
+    private val markdownParser = MarkdownParser(markdownFlavor)
+
+    override fun readFrom(source: Source): PageFragment = readFrom(source.readString())
+
+    public val snarkReader: SnarkReader<PageFragment> = SnarkReader(this, "text/markdown")
+
+}
+

snark-html/src/jvmMain/kotlin/space/kscience/snark/html/static/StaticSiteContext.kt

@@ -0,0 +1,180 @@
+package space.kscience.snark.html.static
+
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.runBlocking
+import kotlinx.io.asSink
+import kotlinx.io.buffered
+import space.kscience.dataforge.data.*
+import space.kscience.dataforge.io.Binary
+import space.kscience.dataforge.io.writeBinary
+import space.kscience.dataforge.meta.*
+import space.kscience.dataforge.names.Name
+import space.kscience.dataforge.names.isEmpty
+import space.kscience.dataforge.names.plus
+import space.kscience.dataforge.workspace.FileData
+import space.kscience.snark.html.*
+import java.nio.file.Path
+import kotlin.io.path.*
+import kotlin.reflect.typeOf
+
+
+/**
+ * An implementation of [SiteContext] to render site as a static directory [outputPath]
+ */
+internal class StaticSiteContext(
+    override val siteMeta: Meta,
+    private val baseUrl: String,
+    override val route: Name,
+    private val outputPath: Path,
+) : SiteContext {
+
+
+//    @OptIn(ExperimentalPathApi::class)
+//    private suspend fun files(item: DataTreeItem<Any>, routeName: Name) {
+//        //try using direct file rendering
+//        item.meta[FileData.FILE_PATH_KEY]?.string?.let {
+//            val file = Path.of(it)
+//            val targetPath = outputPath.resolve(routeName.toWebPath())
+//            targetPath.parent.createDirectories()
+//            file.copyToRecursively(targetPath, followLinks = false)
+//            //success, don't do anything else
+//            return@files
+//        }
+//
+//        when (item) {
+//            is DataTreeItem.Leaf -> {
+//                val datum = item.data
+//                if (datum.type != typeOf<Binary>()) error("Can't directly serve file of type ${item.data.type}")
+//                val targetPath = outputPath.resolve(routeName.toWebPath())
+//                val binary = datum.await() as Binary
+//                targetPath.outputStream().asSink().buffered().use {
+//                    it.writeBinary(binary)
+//                }
+//            }
+//
+//            is DataTreeItem.Node -> {
+//                item.tree.items.forEach { (token, childItem) ->
+//                    files(childItem, routeName + token)
+//                }
+//            }
+//        }
+//    }
+
+    @OptIn(ExperimentalPathApi::class)
+    override fun static(route: Name, data: Data<Binary>) {
+        //if data is a file, copy it
+        data.meta[FileData.FILE_PATH_KEY]?.string?.let {
+            val file = Path.of(it)
+            val targetPath = outputPath.resolve(route.toWebPath())
+            targetPath.parent.createDirectories()
+            file.copyToRecursively(targetPath, followLinks = false)
+            //success, don't do anything else
+            return
+        }
+
+        if (data.type != typeOf<Binary>()) error("Can't directly serve file of type ${data.type}")
+        val targetPath = outputPath.resolve(route.toWebPath())
+        runBlocking(Dispatchers.IO) {
+            val binary = data.await()
+            targetPath.outputStream().asSink().buffered().use {
+                it.writeBinary(binary)
+            }
+        }
+    }
+
+    private fun resolveRef(baseUrl: String, ref: String) = if (baseUrl.isEmpty()) {
+        ref
+    } else if (ref.isEmpty()) {
+        baseUrl
+    } else {
+        "${baseUrl.removeSuffix("/")}/$ref"
+    }
+
+    class StaticPageContext(override val site: StaticSiteContext, override val pageMeta: Meta) : PageContext {
+
+        override fun resolveRef(ref: String): String =
+            site.resolveRef(site.baseUrl, ref)
+
+        override fun resolvePageRef(pageName: Name, relative: Boolean): String = resolveRef(
+            (if (relative) site.route + pageName else pageName).toWebPath() + ".html"
+        )
+    }
+
+    override fun page(route: Name, data: DataSet<Any>, pageMeta: Meta, content: HtmlPage) {
+
+
+        val modifiedPageMeta = pageMeta.toMutableMeta().apply {
+            "name" put route.toString()
+        }
+
+        val newPath = if (route.isEmpty()) {
+            outputPath.resolve("index.html")
+        } else {
+            outputPath.resolve(route.toWebPath() + ".html")
+        }
+
+        newPath.parent.createDirectories()
+
+        val pageContext = StaticPageContext(this, Laminate(modifiedPageMeta, siteMeta))
+        newPath.writeText(HtmlPage.createHtmlString(pageContext, data, content))
+    }
+
+    override fun route(route: Name, data: DataSet<*>, siteMeta: Meta, content: HtmlSite) {
+        val siteContextWithData = SiteContextWithData(
+            StaticSiteContext(
+                siteMeta = Laminate(siteMeta, this@StaticSiteContext.siteMeta),
+                baseUrl = baseUrl,
+                route = route,
+                outputPath = outputPath.resolve(route.toWebPath())
+            ),
+            data
+        )
+        with(content) {
+            with(siteContextWithData) {
+                renderSite()
+            }
+        }
+    }
+
+    override fun site(route: Name, data: DataSet<Any>, siteMeta: Meta, content: HtmlSite) {
+        val siteContextWithData = SiteContextWithData(
+            StaticSiteContext(
+                siteMeta = Laminate(siteMeta, this@StaticSiteContext.siteMeta),
+                baseUrl = if (baseUrl == "") "" else resolveRef(baseUrl, route.toWebPath()),
+                route = Name.EMPTY,
+                outputPath = outputPath.resolve(route.toWebPath())
+            ),
+            data
+        )
+        with(content) {
+            with(siteContextWithData) {
+                renderSite()
+            }
+        }
+    }
+
+}
+
+/**
+ * Create a static site using given [SnarkEnvironment] in provided [outputPath].
+ * Use [siteUrl] as a base for all resolved URLs. By default, use [outputPath] absolute path as a base.
+ *
+ */
+@Suppress("UnusedReceiverParameter")
+public suspend fun SnarkHtml.staticSite(
+    data: DataSet<*>,
+    outputPath: Path,
+    siteUrl: String = outputPath.absolutePathString().replace("\\", "/"),
+    siteMeta: Meta = data.meta,
+    content: HtmlSite,
+) {
+    val siteContextWithData = SiteContextWithData(
+        StaticSiteContext(siteMeta, siteUrl, Name.EMPTY, outputPath),
+        data
+    )
+    with(content){
+        with(siteContextWithData) {
+            renderSite()
+        }
+    }
+}

snark-html/src/main/kotlin/space/kscience/snark/html/DataRenderer.kt

@@ -1,48 +0,0 @@
-package space.kscience.snark.html
-
-import kotlinx.html.body
-import kotlinx.html.head
-import kotlinx.html.title
-import space.kscience.dataforge.data.Data
-import space.kscience.dataforge.meta.*
-import space.kscience.dataforge.names.Name
-import kotlin.reflect.typeOf
-
-/**
- * Render (or don't) given data piece
- */
-public interface DataRenderer {
-
-    context(SiteBuilder)
-    public operator fun invoke(name: Name, data: Data<Any>)
-
-    public companion object {
-        public val DEFAULT: DataRenderer = object : DataRenderer {
-
-            context(SiteBuilder)
-            override fun invoke(name: Name, data: Data<Any>) {
-                if (data.type == typeOf<HtmlData>()) {
-                    val languageMeta: Meta = Language.forName(name)
-
-                    val dataMeta: Meta = if (languageMeta.isEmpty()) {
-                        data.meta
-                    } else {
-                        data.meta.toMutableMeta().apply {
-                            "languages" put languageMeta
-                        }
-                    }
-
-                    page(name, dataMeta) {
-                        head {
-                            title = dataMeta["title"].string ?: "Untitled page"
-                        }
-                        body {
-                            @Suppress("UNCHECKED_CAST")
-                            htmlData(data as HtmlData)
-                        }
-                    }
-                }
-            }
-        }
-    }
-}

snark-html/src/main/kotlin/space/kscience/snark/html/HtmlData.kt

@@ -1,48 +0,0 @@
-package space.kscience.snark.html
-
-import kotlinx.coroutines.Dispatchers
-import kotlinx.coroutines.runBlocking
-import kotlinx.html.FlowContent
-import kotlinx.html.TagConsumer
-import space.kscience.dataforge.data.Data
-import space.kscience.dataforge.data.await
-import space.kscience.dataforge.meta.get
-import space.kscience.dataforge.meta.int
-import space.kscience.dataforge.meta.string
-import space.kscience.snark.SnarkContext
-
-
-//TODO replace by VisionForge type
-//typealias HtmlFragment = context(PageBuilder, TagConsumer<*>) () -> Unit
-
-public fun interface HtmlFragment {
-    public fun TagConsumer<*>.renderFragment(page: WebPage)
-    //TODO move pageBuilder to a context receiver after KT-52967 is fixed
-}
-
-public typealias HtmlData = Data<HtmlFragment>
-
-//fun HtmlData(meta: Meta, content: context(PageBuilder) TagConsumer<*>.() -> Unit): HtmlData =
-//    Data(HtmlFragment(content), meta)
-
-
-context(WebPage)
-public fun FlowContent.htmlData(data: HtmlData): Unit = runBlocking(Dispatchers.IO) {
-    with(data.await()) { consumer.renderFragment(page) }
-}
-
-context(SnarkContext)
-public val Data<*>.id: String
-    get() = meta["id"]?.string ?: "block[${hashCode()}]"
-
-context(SnarkContext)
-public val Data<*>.language: String?
-    get() = meta["language"].string?.lowercase()
-
-context(SnarkContext)
-public val Data<*>.order: Int?
-    get() = meta["order"]?.int
-
-context(SnarkContext)
-public val Data<*>.published: Boolean
-    get() = meta["published"].string != "false"

snark-html/src/main/kotlin/space/kscience/snark/html/ImageIOReader.kt

@@ -1,15 +0,0 @@
-package space.kscience.snark.html
-
-import io.ktor.util.asStream
-import io.ktor.utils.io.core.Input
-import space.kscience.dataforge.io.IOReader
-import java.awt.image.BufferedImage
-import javax.imageio.ImageIO
-import kotlin.reflect.KType
-import kotlin.reflect.typeOf
-
-internal object ImageIOReader : IOReader<BufferedImage> {
-    override val type: KType get() = typeOf<BufferedImage>()
-
-    override fun readObject(input: Input): BufferedImage = ImageIO.read(input.asStream())
-}

snark-html/src/main/kotlin/space/kscience/snark/html/Language.kt

@@ -1,149 +0,0 @@
-package space.kscience.snark.html
-
-import space.kscience.dataforge.data.getItem
-import space.kscience.dataforge.meta.*
-import space.kscience.dataforge.names.*
-import space.kscience.snark.SnarkBuilder
-import space.kscience.snark.html.Language.Companion.SITE_LANGUAGES_KEY
-import space.kscience.snark.html.Language.Companion.SITE_LANGUAGE_KEY
-
-
-
-public class Language : Scheme() {
-    /**
-     * Language key override
-     */
-    public var key: String? by string()
-
-    /**
-     * Page name prefix
-     */
-    public var prefix: String? by string()
-
-    /**
-     * Target page name with a given language key
-     */
-    public var target: Name?
-        get() = meta["target"].string?.parseAsName(false)
-        set(value) {
-            meta["target"] = value?.toString()?.asValue()
-        }
-
-    public companion object : SchemeSpec<Language>(::Language) {
-
-        public val LANGUAGE_KEY: Name = "language".asName()
-
-        public val LANGUAGES_KEY: Name = "languages".asName()
-
-        public val SITE_LANGUAGE_KEY: Name = SiteBuilder.SITE_META_KEY + LANGUAGE_KEY
-
-        public val SITE_LANGUAGES_KEY: Name = SiteBuilder.SITE_META_KEY + LANGUAGES_KEY
-
-        public const val DEFAULT_LANGUAGE: String = "en"
-
-        /**
-         * Automatically build a language map for a data piece with given [name] based on existence of appropriate data nodes.
-         */
-        context(SiteBuilder)
-        public fun forName(name: Name): Meta = Meta {
-            val currentLanguagePrefix = languages[language]?.get(Language::prefix.name)?.string ?: language
-            val fullName = (route.removeFirstOrNull(currentLanguagePrefix.asName()) ?: route) + name
-            languages.forEach { (key, meta) ->
-                val languagePrefix: String = meta[Language::prefix.name].string ?: key
-                val nameWithLanguage: Name = if (languagePrefix.isBlank()) {
-                    fullName
-                } else {
-                    languagePrefix.asName() + fullName
-                }
-                if (data.getItem(name) != null) {
-                    key put meta.asMutableMeta().apply {
-                        Language::target.name put nameWithLanguage.toString()
-                    }
-                }
-            }
-        }
-    }
-}
-
-public val SiteBuilder.languages: Map<String, Meta>
-    get() = siteMeta[SITE_LANGUAGES_KEY]?.items?.mapKeys { it.key.toStringUnescaped() } ?: emptyMap()
-
-public val SiteBuilder.language: String
-    get() = siteMeta[SITE_LANGUAGE_KEY].string ?: Language.DEFAULT_LANGUAGE
-
-public val SiteBuilder.languagePrefix: Name
-    get() = languages[language]?.let { it[Language::prefix.name].string ?: language }?.parseAsName() ?: Name.EMPTY
-
-public fun SiteBuilder.withLanguages(languageMap: Map<String, Meta>, block: SiteBuilder.(language: String) -> Unit) {
-    languageMap.forEach { (languageKey, languageMeta) ->
-        val prefix = languageMeta[Language::prefix.name].string ?: languageKey
-        val routeMeta = Meta {
-            SITE_LANGUAGE_KEY put languageKey
-            SITE_LANGUAGES_KEY put Meta {
-                languageMap.forEach {
-                    it.key put it.value
-                }
-            }
-        }
-        route(prefix, routeMeta = routeMeta) {
-            block(languageKey)
-        }
-    }
-}
-
-@SnarkBuilder
-public fun SiteBuilder.withLanguages(
-    vararg language: Pair<String, String>,
-    block: SiteBuilder.(language: String) -> Unit,
-) {
-    val languageMap = language.associate {
-        it.first to Meta {
-            Language::prefix.name put it.second
-        }
-    }
-    withLanguages(languageMap, block)
-}
-
-
-
-/**
- * The language key of this page
- */
-public val WebPage.language: String
-    get() = pageMeta[Language.LANGUAGE_KEY]?.string ?: pageMeta[SITE_LANGUAGE_KEY]?.string ?: Language.DEFAULT_LANGUAGE
-
-/**
- * Mapping of language keys to other language versions of this page
- */
-public val WebPage.languages: Map<String, Meta>
-    get() = pageMeta[Language.LANGUAGES_KEY]?.items?.mapKeys { it.key.toStringUnescaped() } ?: emptyMap()
-
-public fun WebPage.localisedPageRef(pageName: Name, relative: Boolean = false): String {
-    val prefix = languages[language]?.get(Language::prefix.name)?.string?.parseAsName() ?: Name.EMPTY
-    return resolvePageRef(prefix + pageName, relative)
-}
-
-
-/**
- * Render all pages in a node with given name. Use localization prefix if appropriate data is available.
- */
-public fun SiteBuilder.localizedPages(
-    dataPath: Name,
-    remotePath: Name = dataPath,
-    dataRenderer: DataRenderer = DataRenderer.DEFAULT,
-) {
-    val item = data.getItem(languagePrefix + dataPath)
-        ?: data.getItem(dataPath)
-        ?: error("No data found by name $dataPath")
-    route(remotePath) {
-        pages(item, dataRenderer)
-    }
-}
-
-public fun SiteBuilder.localizedPages(
-    dataPath: String,
-    remotePath: Name = dataPath.parseAsName(),
-    dataRenderer: DataRenderer = DataRenderer.DEFAULT,
-) {
-    localizedPages(dataPath.parseAsName(), remotePath, dataRenderer = dataRenderer)
-}

snark-html/src/main/kotlin/space/kscience/snark/html/SiteBuilder.kt

@@ -1,248 +0,0 @@
-package space.kscience.snark.html
-
-import kotlinx.html.HTML
-import space.kscience.dataforge.context.Context
-import space.kscience.dataforge.context.ContextAware
-import space.kscience.dataforge.data.DataTree
-import space.kscience.dataforge.data.DataTreeItem
-import space.kscience.dataforge.data.branch
-import space.kscience.dataforge.data.getItem
-import space.kscience.dataforge.meta.Meta
-import space.kscience.dataforge.meta.get
-import space.kscience.dataforge.meta.getIndexed
-import space.kscience.dataforge.meta.string
-import space.kscience.dataforge.names.Name
-import space.kscience.dataforge.names.NameToken
-import space.kscience.dataforge.names.asName
-import space.kscience.dataforge.names.parseAsName
-import space.kscience.snark.SnarkBuilder
-import space.kscience.snark.SnarkContext
-import space.kscience.snark.html.SiteLayout.Companion.LAYOUT_KEY
-
-
-/**
- * An abstraction, which is used to render sites to the different rendering engines
- */
-@SnarkBuilder
-public interface SiteBuilder : ContextAware, SnarkContext {
-
-    /**
-     * Route name of this [SiteBuilder] relative to the site root
-     */
-    public val route: Name
-
-    /**
-     * Data used for site construction. The type of the data is not limited
-     */
-    public val data: DataTree<*>
-
-    /**
-     * Snark plugin and context used for layout resolution, preprocessors, etc
-     */
-    public val snark: SnarkHtmlPlugin
-
-    override val context: Context get() = snark.context
-
-    /**
-     * Site configuration
-     */
-    public val siteMeta: Meta
-
-    /**
-     * Serve a static data as a file from [data] with given [dataName] at given [routeName].
-     */
-    public fun static(dataName: Name, routeName: Name = dataName)
-
-
-    /**
-     * Create a single page at given [route]. If route is empty, create an index page at current route.
-     *
-     * @param pageMeta additional page meta. [WebPage] will use both it and [siteMeta]
-     */
-    @SnarkBuilder
-    public fun page(route: Name = Name.EMPTY, pageMeta: Meta = Meta.EMPTY, content: context(HTML, WebPage) () -> Unit)
-
-    /**
-     * Create a route with optional data tree override. For example one could use a subtree of the initial tree.
-     * By default, the same data tree is used for route.
-     */
-    public fun route(
-        routeName: Name,
-        dataOverride: DataTree<*>? = null,
-        routeMeta: Meta = Meta.EMPTY,
-    ): SiteBuilder
-
-    /**
-     * Creates a route and sets it as site base url
-     */
-    public fun site(
-        routeName: Name,
-        dataOverride: DataTree<*>? = null,
-        routeMeta: Meta = Meta.EMPTY,
-    ): SiteBuilder
-
-
-    public companion object {
-        public val SITE_META_KEY: Name = "site".asName()
-        public val INDEX_PAGE_TOKEN: NameToken = NameToken("index")
-        public val UP_PAGE_TOKEN: NameToken = NameToken("..")
-    }
-}
-
-context(SiteBuilder)
-public val site: SiteBuilder
-    get() = this@SiteBuilder
-
-@SnarkBuilder
-public inline fun SiteBuilder.route(
-    route: Name,
-    dataOverride: DataTree<*>? = null,
-    routeMeta: Meta = Meta.EMPTY,
-    block: SiteBuilder.() -> Unit,
-) {
-    route(route, dataOverride, routeMeta).apply(block)
-}
-
-@SnarkBuilder
-public inline fun SiteBuilder.route(
-    route: String,
-    dataOverride: DataTree<*>? = null,
-    routeMeta: Meta = Meta.EMPTY,
-    block: SiteBuilder.() -> Unit,
-) {
-    route(route.parseAsName(), dataOverride, routeMeta).apply(block)
-}
-
-@SnarkBuilder
-public inline fun SiteBuilder.site(
-    route: Name,
-    dataOverride: DataTree<*>? = null,
-    routeMeta: Meta = Meta.EMPTY,
-    block: SiteBuilder.() -> Unit,
-) {
-    site(route, dataOverride, routeMeta).apply(block)
-}
-
-@SnarkBuilder
-public inline fun SiteBuilder.site(
-    route: String,
-    dataOverride: DataTree<*>? = null,
-    routeMeta: Meta = Meta.EMPTY,
-    block: SiteBuilder.() -> Unit,
-) {
-    site(route.parseAsName(), dataOverride, routeMeta).apply(block)
-}
-
-public inline fun SiteBuilder.withData(
-    data: DataTree<*>,
-    block: SiteBuilder.() -> Unit
-){
-    route(Name.EMPTY, data).apply(block)
-}
-
-public inline fun SiteBuilder.withDataBranch(
-    name: Name,
-    block: SiteBuilder.() -> Unit
-){
-    route(Name.EMPTY, data.branch(name)).apply(block)
-}
-
-public inline fun SiteBuilder.withDataBranch(
-    name: String,
-    block: SiteBuilder.() -> Unit
-){
-    route(Name.EMPTY, data.branch(name)).apply(block)
-}
-
-///**
-// * Create a stand-alone site at a given node
-// */
-//public fun SiteBuilder.site(route: Name, dataRoot: DataTree<*>, block: SiteBuilder.() -> Unit) {
-//    val mountedData = data.copy(
-//        data = dataRoot,
-//        baseUrlPath = data.resolveRef(route.tokens.joinToString(separator = "/")),
-//        meta = Laminate(dataRoot.meta, data.meta) //layering dataRoot meta over existing data
-//    )
-//    route(route) {
-//        withData(mountedData).block()
-//    }
-//}
-
-public fun SiteBuilder.static(dataName: String): Unit = static(dataName.parseAsName())
-
-public fun SiteBuilder.static(dataName: String, routeName: String): Unit = static(
-    dataName.parseAsName(),
-    routeName.parseAsName()
-)
-
-internal fun SiteBuilder.assetsFrom(rootMeta: Meta) {
-    rootMeta.getIndexed("file".asName()).forEach { (_, meta) ->
-        val webName: String? by meta.string()
-        val name by meta.string { error("File path is not provided") }
-        val fileName = name.parseAsName()
-        static(fileName, webName?.parseAsName() ?: fileName)
-    }
-}
-
-
-/**
- * Recursively renders the data items in [data]. If [LAYOUT_KEY] is defined in an item, use it to load
- * layout from the context, otherwise render children nodes as name segments and individual data items using [dataRenderer].
- */
-public fun SiteBuilder.pages(
-    data: DataTreeItem<*>,
-    dataRenderer: DataRenderer = DataRenderer.DEFAULT,
-) {
-    val layoutMeta = data.meta[LAYOUT_KEY]
-    if (layoutMeta != null) {
-        //use layout if it is defined
-        snark.siteLayout(layoutMeta).render(data)
-    } else {
-        when (data) {
-            is DataTreeItem.Node -> {
-                data.tree.items.forEach { (token, item) ->
-                    //Don't apply index token
-                    if (token == SiteLayout.INDEX_PAGE_TOKEN) {
-                        pages(item, dataRenderer)
-                    } else if (item is DataTreeItem.Leaf) {
-                        dataRenderer(token.asName(), item.data)
-                    } else {
-                        route(token.asName()) {
-                            pages(item, dataRenderer)
-                        }
-                    }
-                }
-            }
-
-            is DataTreeItem.Leaf -> {
-                dataRenderer(Name.EMPTY, data.data)
-            }
-        }
-        data.meta[SiteLayout.ASSETS_KEY]?.let {
-            assetsFrom(it)
-        }
-    }
-    //TODO watch for changes
-}
-
-/**
- * Render all pages in a node with given name
- */
-public fun SiteBuilder.pages(
-    dataPath: Name,
-    remotePath: Name = dataPath,
-    dataRenderer: DataRenderer = DataRenderer.DEFAULT,
-) {
-    val item = data.getItem(dataPath) ?: error("No data found by name $dataPath")
-    route(remotePath) {
-        pages(item, dataRenderer)
-    }
-}
-
-public fun SiteBuilder.pages(
-    dataPath: String,
-    remotePath: Name = dataPath.parseAsName(),
-    dataRenderer: DataRenderer = DataRenderer.DEFAULT,
-) {
-    pages(dataPath.parseAsName(), remotePath, dataRenderer = dataRenderer)
-}

snark-html/src/main/kotlin/space/kscience/snark/html/SiteLayout.kt

@@ -1,32 +0,0 @@
-package space.kscience.snark.html
-
-import space.kscience.dataforge.data.DataTreeItem
-import space.kscience.dataforge.misc.Type
-import space.kscience.dataforge.names.NameToken
-
-/**
- * An abstraction to render singular data or a data tree.
- */
-@Type(SiteLayout.TYPE)
-public fun interface SiteLayout {
-
-    context(SiteBuilder)
-    public fun render(item: DataTreeItem<*>)
-
-    public companion object {
-        public const val TYPE: String = "snark.layout"
-        public const val LAYOUT_KEY: String = "layout"
-        public const val ASSETS_KEY: String = "assets"
-        public val INDEX_PAGE_TOKEN: NameToken = NameToken("index")
-    }
-}
-
-
-/**
- * The default [SiteLayout]. It renders all [HtmlData] pages with simple headers via [SiteLayout.defaultDataRenderer]
- */
-public object DefaultSiteLayout : SiteLayout {
-    context(SiteBuilder) override fun render(item: DataTreeItem<*>) {
-        pages(item)
-    }
-}

snark-html/src/main/kotlin/space/kscience/snark/html/SnarkHtmlPlugin.kt

@@ -1,124 +0,0 @@
-package space.kscience.snark.html
-
-import io.ktor.utils.io.core.readBytes
-import space.kscience.dataforge.context.*
-import space.kscience.dataforge.data.DataTree
-import space.kscience.dataforge.data.node
-import space.kscience.dataforge.io.IOPlugin
-import space.kscience.dataforge.io.IOReader
-import space.kscience.dataforge.io.JsonMetaFormat
-import space.kscience.dataforge.io.yaml.YamlMetaFormat
-import space.kscience.dataforge.io.yaml.YamlPlugin
-import space.kscience.dataforge.meta.Meta
-import space.kscience.dataforge.meta.get
-import space.kscience.dataforge.meta.string
-import space.kscience.dataforge.misc.DFExperimental
-import space.kscience.dataforge.names.Name
-import space.kscience.dataforge.names.asName
-import space.kscience.dataforge.names.parseAsName
-import space.kscience.dataforge.workspace.FileData
-import space.kscience.dataforge.workspace.readDataDirectory
-import space.kscience.snark.SnarkParser
-import java.nio.file.Path
-import kotlin.io.path.extension
-import kotlin.io.path.toPath
-
-/**
- * A plugin used for rendering a [DataTree] as HTML
- */
-public class SnarkHtmlPlugin : AbstractPlugin() {
-    private val yaml by require(YamlPlugin)
-    public val io: IOPlugin get() = yaml.io
-
-    override val tag: PluginTag get() = Companion.tag
-
-    internal val parsers: Map<Name, SnarkParser<Any>> by lazy {
-        context.gather(SnarkParser.TYPE, true)
-    }
-
-    private val siteLayouts: Map<Name, SiteLayout> by lazy {
-        context.gather(SiteLayout.TYPE, true)
-    }
-
-    private val textProcessors: Map<Name, TextProcessor> by lazy {
-        context.gather(TextProcessor.TYPE, true)
-    }
-
-    internal fun siteLayout(layoutMeta: Meta): SiteLayout {
-        val layoutName = layoutMeta.string
-            ?: layoutMeta["name"].string ?: error("Layout name not defined in $layoutMeta")
-        return siteLayouts[layoutName.parseAsName()] ?: error("Layout with name $layoutName not found in $this")
-    }
-
-    internal fun textProcessor(transformationMeta: Meta): TextProcessor {
-        val transformationName = transformationMeta.string
-            ?: transformationMeta["name"].string ?: error("Transformation name not defined in $transformationMeta")
-        return textProcessors[transformationName.parseAsName()]
-            ?: error("Text transformation with name $transformationName not found in $this")
-    }
-
-    override fun content(target: String): Map<Name, Any> = when (target) {
-        SnarkParser.TYPE -> mapOf(
-            "html".asName() to SnarkHtmlParser,
-            "markdown".asName() to SnarkMarkdownParser,
-            "json".asName() to SnarkParser(JsonMetaFormat, "json"),
-            "yaml".asName() to SnarkParser(YamlMetaFormat, "yaml", "yml"),
-            "png".asName() to SnarkParser(ImageIOReader, "png"),
-            "jpg".asName() to SnarkParser(ImageIOReader, "jpg", "jpeg"),
-            "gif".asName() to SnarkParser(ImageIOReader, "gif"),
-            "svg".asName() to SnarkParser(IOReader.binary, "svg"),
-            "raw".asName() to SnarkParser(IOReader.binary, "css", "js", "scss", "woff", "woff2", "ttf", "eot")
-        )
-
-        TextProcessor.TYPE -> mapOf(
-            "basic".asName() to BasicTextProcessor
-        )
-
-        else -> super.content(target)
-    }
-
-    public companion object : PluginFactory<SnarkHtmlPlugin> {
-        override val tag: PluginTag = PluginTag("snark")
-
-        override fun build(context: Context, meta: Meta): SnarkHtmlPlugin = SnarkHtmlPlugin()
-
-        private val byteArrayIOReader = IOReader {
-            readBytes()
-        }
-
-        internal val byteArraySnarkParser = SnarkParser(byteArrayIOReader)
-    }
-}
-
-@OptIn(DFExperimental::class)
-public fun SnarkHtmlPlugin.readDirectory(path: Path): DataTree<Any> = io.readDataDirectory(
-    path,
-    setOf("md", "html", "yaml", "json")
-) { dataPath, meta ->
-    val fileExtension = meta[FileData.FILE_EXTENSION_KEY].string ?: dataPath.extension
-    val parser: SnarkParser<Any> = parsers.values.filter { parser ->
-        fileExtension in parser.fileExtensions
-    }.maxByOrNull {
-        it.priority
-    } ?: run {
-        logger.debug { "The parser is not found for file $dataPath with meta $meta" }
-        SnarkHtmlPlugin.byteArraySnarkParser
-    }
-
-    parser.asReader(context, meta)
-}
-
-public fun SnarkHtmlPlugin.readResources(
-    vararg resources: String,
-    classLoader: ClassLoader = Thread.currentThread().contextClassLoader,
-): DataTree<Any> {
-//    require(resource.isNotBlank()) {"Can't mount root resource tree as data root"}
-    return DataTree {
-        resources.forEach { resource ->
-            val path = classLoader.getResource(resource)?.toURI()?.toPath() ?: error(
-                "Resource with name $resource is not resolved"
-            )
-            node(resource, readDirectory(path))
-        }
-    }
-}

snark-html/src/main/kotlin/space/kscience/snark/html/SnarkTextParser.kt

@@ -1,58 +0,0 @@
-package space.kscience.snark.html
-
-import kotlinx.html.div
-import kotlinx.html.unsafe
-import org.intellij.markdown.flavours.commonmark.CommonMarkFlavourDescriptor
-import org.intellij.markdown.html.HtmlGenerator
-import org.intellij.markdown.parser.MarkdownParser
-import space.kscience.dataforge.context.Context
-import space.kscience.dataforge.meta.Meta
-import space.kscience.dataforge.meta.get
-import space.kscience.snark.SnarkParser
-import kotlin.reflect.KType
-import kotlin.reflect.typeOf
-
-public abstract class SnarkTextParser<R> : SnarkParser<R> {
-    public abstract fun parseText(text: String, meta: Meta): R
-
-    override fun parse(context: Context, meta: Meta, bytes: ByteArray): R =
-        parseText(bytes.decodeToString(), meta)
-
-    public fun transformText(text: String, meta: Meta, page: WebPage): String =
-        meta[TextProcessor.TEXT_TRANSFORMATION_KEY]?.let {
-            with(page) { page.snark.textProcessor(it).process(text) }
-        } ?: text
-}
-
-
-internal object SnarkHtmlParser : SnarkTextParser<HtmlFragment>() {
-    override val fileExtensions: Set<String> = setOf("html")
-    override val type: KType = typeOf<HtmlFragment>()
-
-    override fun parseText(text: String, meta: Meta): HtmlFragment = HtmlFragment { page ->
-        div {
-            unsafe { +transformText(text, meta, page) }
-        }
-    }
-}
-
-internal object SnarkMarkdownParser : SnarkTextParser<HtmlFragment>() {
-    override val fileExtensions: Set<String> = setOf("markdown", "mdown", "mkdn", "mkd", "md")
-    override val type: KType = typeOf<HtmlFragment>()
-
-    private val markdownFlavor = CommonMarkFlavourDescriptor()
-    private val markdownParser = MarkdownParser(markdownFlavor)
-
-    override fun parseText(text: String, meta: Meta): HtmlFragment = HtmlFragment { page ->
-        val transformedText = SnarkHtmlParser.transformText(text, meta, page)
-        val parsedTree = markdownParser.buildMarkdownTreeFromString(transformedText)
-        val htmlString = HtmlGenerator(transformedText, parsedTree, markdownFlavor).generateHtml()
-
-        div {
-            unsafe {
-                +htmlString
-            }
-        }
-    }
-}
-

snark-html/src/main/kotlin/space/kscience/snark/html/StaticSiteBuilder.kt

@@ -1,201 +0,0 @@
-package space.kscience.snark.html
-
-import io.ktor.utils.io.streams.asInput
-import io.ktor.utils.io.streams.asOutput
-import kotlinx.coroutines.runBlocking
-import kotlinx.html.HTML
-import kotlinx.html.html
-import kotlinx.html.stream.createHTML
-import space.kscience.dataforge.data.DataTree
-import space.kscience.dataforge.data.DataTreeItem
-import space.kscience.dataforge.data.await
-import space.kscience.dataforge.data.getItem
-import space.kscience.dataforge.io.Binary
-import space.kscience.dataforge.io.toByteArray
-import space.kscience.dataforge.io.writeBinary
-import space.kscience.dataforge.meta.*
-import space.kscience.dataforge.names.Name
-import space.kscience.dataforge.names.isEmpty
-import space.kscience.dataforge.names.plus
-import space.kscience.dataforge.workspace.FileData
-import java.nio.file.Path
-import kotlin.contracts.InvocationKind
-import kotlin.contracts.contract
-import kotlin.io.path.*
-import kotlin.reflect.typeOf
-
-
-/**
- * An implementation of [SiteBuilder] to render site as a static directory [outputPath]
- */
-internal class StaticSiteBuilder(
-    override val snark: SnarkHtmlPlugin,
-    override val data: DataTree<*>,
-    override val siteMeta: Meta,
-    private val baseUrl: String,
-    override val route: Name,
-    private val outputPath: Path,
-) : SiteBuilder {
-
-
-//    private fun Path.copyRecursively(target: Path) {
-//        Files.walk(this).forEach { source: Path ->
-//            val destination: Path = target.resolve(source.relativeTo(this))
-//            if (!destination.isDirectory()) {
-//                //avoid re-creating directories
-//                source.copyTo(destination, true)
-//            }
-//        }
-//    }
-
-    @OptIn(ExperimentalPathApi::class)
-    private suspend fun files(item: DataTreeItem<Any>, routeName: Name) {
-        //try using direct file rendering
-        item.meta[FileData.FILE_PATH_KEY]?.string?.let {
-            val file = Path.of(it)
-            val targetPath = outputPath.resolve(routeName.toWebPath())
-            targetPath.parent.createDirectories()
-            file.copyToRecursively(targetPath, followLinks = false)
-            //success, don't do anything else
-            return@files
-        }
-
-        when (item) {
-            is DataTreeItem.Leaf -> {
-                val datum = item.data
-                if (datum.type != typeOf<Binary>()) error("Can't directly serve file of type ${item.data.type}")
-                val targetPath = outputPath.resolve(routeName.toWebPath())
-                val binary = datum.await() as Binary
-                targetPath.outputStream().asOutput().use{
-                    it.writeBinary(binary)
-                }
-            }
-
-            is DataTreeItem.Node -> {
-                item.tree.items.forEach { (token, childItem) ->
-                    files(childItem, routeName + token)
-                }
-            }
-        }
-    }
-
-    override fun static(dataName: Name, routeName: Name) {
-        val item: DataTreeItem<Any> = data.getItem(dataName) ?: error("Data with name $dataName is not resolved")
-        runBlocking {
-            files(item, routeName)
-        }
-    }
-
-//
-//    override fun file(file: Path, webPath: String) {
-//        val targetPath = outputPath.resolve(webPath)
-//        if (file.isDirectory()) {
-//            targetPath.parent.createDirectories()
-//            file.copyRecursively(targetPath)
-//        } else if (webPath.isBlank()) {
-//            error("Can't mount file to an empty route")
-//        } else {
-//            targetPath.parent.createDirectories()
-//            file.copyTo(targetPath, true)
-//        }
-//    }
-//
-//    override fun resourceFile(resourcesPath: String, webPath: String) {
-//        val targetPath = outputPath.resolve(webPath)
-//        targetPath.parent.createDirectories()
-//        javaClass.getResource(resourcesPath)?.let { Path.of(it.toURI()) }?.copyTo(targetPath, true)
-//    }
-//
-//    override fun resourceDirectory(resourcesPath: String) {
-//        outputPath.parent.createDirectories()
-//        javaClass.getResource(resourcesPath)?.let { Path.of(it.toURI()) }?.copyRecursively(outputPath)
-//    }
-
-    private fun resolveRef(baseUrl: String, ref: String) = if (baseUrl.isEmpty()) {
-        ref
-    } else if (ref.isEmpty()) {
-        baseUrl
-    } else {
-        "${baseUrl.removeSuffix("/")}/$ref"
-    }
-
-    inner class StaticWebPage(override val pageMeta: Meta) : WebPage {
-        override val data: DataTree<*> get() = this@StaticSiteBuilder.data
-
-        override val snark: SnarkHtmlPlugin get() = this@StaticSiteBuilder.snark
-
-
-        override fun resolveRef(ref: String): String =
-            this@StaticSiteBuilder.resolveRef(this@StaticSiteBuilder.baseUrl, ref)
-
-        override fun resolvePageRef(pageName: Name, relative: Boolean): String = resolveRef(
-            (if (relative) this@StaticSiteBuilder.route + pageName else pageName).toWebPath() + ".html"
-        )
-    }
-
-
-    override fun page(route: Name, pageMeta: Meta, content: context(HTML) WebPage.() -> Unit) {
-        val htmlBuilder = createHTML()
-
-        val modifiedPageMeta = pageMeta.toMutableMeta().apply {
-            "name" put route.toString()
-        }
-
-        htmlBuilder.html {
-            content(this, StaticWebPage(Laminate(modifiedPageMeta, siteMeta)))
-        }
-
-        val newPath = if (route.isEmpty()) {
-            outputPath.resolve("index.html")
-        } else {
-            outputPath.resolve(route.toWebPath() + ".html")
-        }
-
-        newPath.parent.createDirectories()
-        newPath.writeText(htmlBuilder.finalize())
-    }
-
-    override fun route(
-        routeName: Name,
-        dataOverride: DataTree<*>?,
-        routeMeta: Meta,
-    ): SiteBuilder = StaticSiteBuilder(
-        snark = snark,
-        data = dataOverride ?: data,
-        siteMeta = Laminate(routeMeta, siteMeta),
-        baseUrl = baseUrl,
-        route = route + routeName,
-        outputPath = outputPath.resolve(routeName.toWebPath())
-    )
-
-    override fun site(
-        routeName: Name,
-        dataOverride: DataTree<*>?,
-        routeMeta: Meta,
-    ): SiteBuilder = StaticSiteBuilder(
-        snark = snark,
-        data = dataOverride ?: data,
-        siteMeta = Laminate(routeMeta, siteMeta),
-        baseUrl = if (baseUrl == "") "" else resolveRef(baseUrl, routeName.toWebPath()),
-        route = Name.EMPTY,
-        outputPath = outputPath.resolve(routeName.toWebPath())
-    )
-}
-
-/**
- * Create a static site using given [SnarkEnvironment] in provided [outputPath].
- * Use [siteUrl] as a base for all resolved URLs. By default, use [outputPath] absolute path as a base.
- *
- */
-public fun SnarkHtmlPlugin.static(
-    data: DataTree<*>,
-    outputPath: Path,
-    siteUrl: String = outputPath.absolutePathString().replace("\\", "/"),
-    siteMeta: Meta = data.meta,
-    block: SiteBuilder.() -> Unit,
-) {
-    contract {
-        callsInPlace(block, InvocationKind.EXACTLY_ONCE)
-    }
-    StaticSiteBuilder(this, data, siteMeta, siteUrl, Name.EMPTY, outputPath).block()
-}

snark-html/src/main/kotlin/space/kscience/snark/html/TextProcessor.kt

@@ -1,61 +0,0 @@
-package space.kscience.snark.html
-
-import space.kscience.dataforge.meta.get
-import space.kscience.dataforge.meta.string
-import space.kscience.dataforge.misc.Type
-import space.kscience.dataforge.names.NameToken
-import space.kscience.dataforge.names.parseAsName
-
-/**
- * An object that conducts page-based text transformation. Like using link replacement or templating.
- */
-@Type(TextProcessor.TYPE)
-public fun interface TextProcessor {
-    context(WebPage)
-    public fun process(text: String): String
-
-    public companion object {
-        public const val TYPE: String = "snark.textTransformation"
-        public val TEXT_TRANSFORMATION_KEY: NameToken = NameToken("transformation")
-    }
-}
-
-/**
- * A basic [TextProcessor] that replaces `${...}` expressions in text. The following expressions are recognised:
- * * `homeRef` resolves to [homeRef]
- * * `resolveRef("...")` -> [WebPage.resolveRef]
- * * `resolvePageRef("...")` -> [WebPage.resolvePageRef]
- * * `pageMeta.get("...") -> [WebPage.pageMeta] get string method
- * Otherwise return unchanged string
- */
-public object BasicTextProcessor : TextProcessor {
-
-    private val regex = """\$\{([\w.]*)(?>\("(.*)"\))?}""".toRegex()
-
-    context(WebPage)
-    override fun process(text: String): String = text.replace(regex) { match ->
-        when (match.groups[1]!!.value) {
-            "homeRef" -> homeRef
-            "resolveRef" -> {
-                val refString = match.groups[2]?.value ?: error("resolveRef requires a string (quoted) argument")
-                resolveRef(refString)
-            }
-
-            "resolvePageRef" -> {
-                val refString = match.groups[2]?.value
-                    ?: error("resolvePageRef requires a string (quoted) argument")
-                localisedPageRef(refString.parseAsName())
-            }
-
-            "pageMeta.get" -> {
-                val nameString = match.groups[2]?.value
-                    ?: error("resolvePageRef requires a string (quoted) argument")
-                pageMeta[nameString.parseAsName()].string ?: "@null"
-            }
-
-            else -> match.value
-        }
-    }
-}
-
-

snark-html/src/main/kotlin/space/kscience/snark/html/WebPage.kt

@@ -1,100 +0,0 @@
-package space.kscience.snark.html
-
-import space.kscience.dataforge.context.Context
-import space.kscience.dataforge.context.ContextAware
-import space.kscience.dataforge.data.*
-import space.kscience.dataforge.meta.Meta
-import space.kscience.dataforge.meta.get
-import space.kscience.dataforge.meta.string
-import space.kscience.dataforge.names.*
-import space.kscience.snark.SnarkBuilder
-import space.kscience.snark.SnarkContext
-
-context(SnarkContext)
-public fun Name.toWebPath(): String = tokens.joinToString(separator = "/") {
-    if (it.hasIndex()) {
-        "${it.body}[${it.index}]"
-    } else {
-        it.body
-    }
-}
-
-/**
- *  A context for building a single page
- */
-@SnarkBuilder
-public interface WebPage : ContextAware, SnarkContext {
-
-    public val snark: SnarkHtmlPlugin
-
-    override val context: Context get() = snark.context
-
-    public val data: DataTree<*>
-
-    /**
-     * A metadata for a page. It should include site metadata
-     */
-    public val pageMeta: Meta
-
-    /**
-     * Resolve absolute url for given [ref]
-     *
-     */
-    public fun resolveRef(ref: String): String
-
-    /**
-     * Resolve absolute url for a page with given [pageName].
-     *
-     * @param relative if true, add [SiteBuilder] route to the absolute page name
-     */
-    public fun resolvePageRef(pageName: Name, relative: Boolean = false): String
-}
-
-context(WebPage)
-public val page: WebPage
-    get() = this@WebPage
-
-public fun WebPage.resolvePageRef(pageName: String): String = resolvePageRef(pageName.parseAsName())
-
-public val WebPage.homeRef: String get() = resolvePageRef(SiteBuilder.INDEX_PAGE_TOKEN.asName())
-
-public val WebPage.name: Name? get() = pageMeta["name"].string?.parseAsName()
-
-/**
- * Resolve a Html builder by its full name
- */
-context(SnarkContext)
-public fun DataTree<*>.resolveHtmlOrNull(name: Name): HtmlData? {
-    val resolved = (getByType<HtmlFragment>(name) ?: getByType<HtmlFragment>(name + SiteBuilder.INDEX_PAGE_TOKEN))
-
-    return resolved?.takeIf {
-        it.published //TODO add language confirmation
-    }
-}
-
-context(SnarkContext)
-public fun DataTree<*>.resolveHtmlOrNull(name: String): HtmlData? = resolveHtmlOrNull(name.parseAsName())
-
-context(SnarkContext)
-public fun DataTree<*>.resolveHtml(name: String): HtmlData = resolveHtmlOrNull(name)
-    ?: error("Html fragment with name $name is not resolved")
-
-/**
- * Find all Html blocks using given name/meta filter
- */
-context(SnarkContext)
-public fun DataTree<*>.resolveAllHtml(predicate: (name: Name, meta: Meta) -> Boolean): Map<Name, HtmlData> =
-    filterByType<HtmlFragment> { name, meta ->
-        predicate(name, meta)
-                && meta["published"].string != "false"
-        //TODO add language confirmation
-    }.asSequence().associate { it.name to it.data }
-
-
-context(SnarkContext)
-public fun DataTree<*>.findByContentType(
-    contentType: String,
-    baseName: Name = Name.EMPTY,
-): Map<Name, Data<HtmlFragment>> = resolveAllHtml { name, meta ->
-    name.startsWith(baseName) && meta["content_type"].string == contentType
-}

snark-html/src/main/kotlin/space/kscience/snark/html/htmlEnvironment.kt

@@ -1,37 +0,0 @@
-package space.kscience.snark.html
-
-import space.kscience.dataforge.data.DataTreeItem
-import space.kscience.dataforge.names.Name
-import space.kscience.dataforge.names.parseAsName
-
-public class SnarkHtmlEnvironmentBuilder {
-    public val layouts: HashMap<Name, SiteLayout> = HashMap()
-
-    public fun layout(name: String, body: context(SiteBuilder) (DataTreeItem<*>) -> Unit) {
-        layouts[name.parseAsName()] = object : SiteLayout {
-            context(SiteBuilder) override fun render(item: DataTreeItem<*>) = body(site, item)
-        }
-    }
-}
-
-
-//public fun SnarkEnvironment.html(block: SnarkHtmlEnvironmentBuilder.() -> Unit) {
-//    contract {
-//        callsInPlace(block, InvocationKind.EXACTLY_ONCE)
-//    }
-//
-//    val envBuilder = SnarkHtmlEnvironmentBuilder().apply(block)
-//
-//    val plugin = object : AbstractPlugin() {
-//        val snark by require(SnarkHtmlPlugin)
-//
-//        override val tag: PluginTag = PluginTag("@extension[${hashCode()}]")
-//
-//
-//        override fun content(target: String): Map<Name, Any> = when (target) {
-//            SiteLayout.TYPE -> envBuilder.layouts
-//            else -> super.content(target)
-//        }
-//    }
-//    registerPlugin(plugin)
-//}

snark-ktor/build.gradle.kts

@@ -1,5 +1,5 @@
 plugins {
-    id("space.kscience.gradle.jvm")
+    id("space.kscience.gradle.mpp")
     `maven-publish`
 }
 
@@ -7,15 +7,17 @@ val dataforgeVersion: String by rootProject.extra
 val ktorVersion = space.kscience.gradle.KScienceVersions.ktorVersion
 
 kscience{
+    jvm()
     useContextReceivers()
-}
 
-dependencies {
+    jvmMain{
         api(projects.snarkHtml)
 
         api("io.ktor:ktor-server-core:$ktorVersion")
         api("io.ktor:ktor-server-html-builder:$ktorVersion")
         api("io.ktor:ktor-server-host-common:$ktorVersion")
-
-    testApi("io.ktor:ktor-server-tests:$ktorVersion")
+    }
+    jvmTest{
+        api("io.ktor:ktor-server-tests:$ktorVersion")
+    }
 }

snark-ktor/src/jvmMain/kotlin/space/kscience/snark/ktor/KtorSiteContext.kt

@@ -0,0 +1,197 @@
+package space.kscience.snark.ktor
+
+import io.ktor.http.*
+import io.ktor.http.content.TextContent
+import io.ktor.server.application.call
+import io.ktor.server.http.content.staticFiles
+import io.ktor.server.plugins.origin
+import io.ktor.server.response.respond
+import io.ktor.server.response.respondBytes
+import io.ktor.server.routing.Route
+import io.ktor.server.routing.createRouteFromPath
+import io.ktor.server.routing.get
+import space.kscience.dataforge.context.Context
+import space.kscience.dataforge.context.ContextAware
+import space.kscience.dataforge.context.error
+import space.kscience.dataforge.context.logger
+import space.kscience.dataforge.data.Data
+import space.kscience.dataforge.data.DataSet
+import space.kscience.dataforge.data.await
+import space.kscience.dataforge.io.Binary
+import space.kscience.dataforge.io.toByteArray
+import space.kscience.dataforge.meta.Laminate
+import space.kscience.dataforge.meta.Meta
+import space.kscience.dataforge.meta.string
+import space.kscience.dataforge.meta.toMutableMeta
+import space.kscience.dataforge.names.Name
+import space.kscience.dataforge.names.cutLast
+import space.kscience.dataforge.names.endsWith
+import space.kscience.dataforge.names.plus
+import space.kscience.dataforge.workspace.FileData
+import space.kscience.snark.html.*
+import java.nio.file.Path
+import kotlin.reflect.typeOf
+
+//public fun CommonAttributeGroupFacade.css(block: CssBuilder.() -> Unit) {
+//    style = CssBuilder().block().toString()
+//}
+
+public class KtorSiteContext(
+    override val context: Context,
+    override val siteMeta: Meta,
+    private val baseUrl: String,
+    override val route: Name,
+    private val ktorRoute: Route,
+) : SiteContext, ContextAware {
+
+
+    override fun static(route: Name, data: Data<Binary>) {
+        data.meta[FileData.FILE_PATH_KEY]?.string?.let {
+            val file = try {
+                Path.of(it).toFile()
+            } catch (ex: Exception) {
+                //failure,
+                logger.error { "File $it could not be converted to java.io.File" }
+                return@let
+            }
+
+            val fileName = route.toWebPath()
+            ktorRoute.staticFiles(fileName, file)
+            //success, don't do anything else
+            return
+        }
+
+        if (data.type != typeOf<Binary>()) error("Can't directly serve file of type ${data.type}")
+        ktorRoute.get(route.toWebPath()) {
+            val binary = data.await()
+            val extension = data.meta[FileData.FILE_EXTENSION_KEY]?.string?.let { ".$it" } ?: ""
+            val contentType: ContentType = extension
+                .let(ContentType::fromFileExtension)
+                .firstOrNull()
+                ?: ContentType.Any
+            call.respondBytes(contentType = contentType) {
+                //TODO optimize using streaming
+                binary.toByteArray()
+            }
+        }
+    }
+
+    private fun resolveRef(baseUrl: String, ref: String) = if (baseUrl.isEmpty()) {
+        ref
+    } else if (ref.isEmpty()) {
+        baseUrl
+    } else {
+        "${baseUrl.removeSuffix("/")}/$ref"
+    }
+
+
+    private class KtorPageContext(
+        override val site: KtorSiteContext,
+        val pageBaseUrl: String,
+        override val pageMeta: Meta,
+    ) : PageContext {
+
+        override fun resolveRef(ref: String): String = site.resolveRef(pageBaseUrl, ref)
+
+        override fun resolvePageRef(
+            pageName: Name,
+            relative: Boolean,
+        ): String {
+            val fullPageName = if (relative) site.route + pageName else pageName
+            return if (fullPageName.endsWith(SiteContext.INDEX_PAGE_TOKEN)) {
+                resolveRef(fullPageName.cutLast().toWebPath())
+            } else {
+                resolveRef(fullPageName.toWebPath())
+            }
+        }
+    }
+
+    override fun page(route: Name, data: DataSet<*>, pageMeta: Meta, content: HtmlPage) {
+        ktorRoute.get(route.toWebPath()) {
+            val request = call.request
+            //substitute host for url for backwards calls
+            val url = URLBuilder(baseUrl).apply {
+                protocol = URLProtocol.createOrDefault(request.origin.scheme)
+                host = request.origin.serverHost
+                port = request.origin.serverPort
+            }
+
+            val modifiedPageMeta = pageMeta.toMutableMeta().apply {
+                "name" put route.toString()
+                "url" put url.buildString()
+            }
+            val pageContext =
+                KtorPageContext(this@KtorSiteContext, url.buildString(), Laminate(modifiedPageMeta, siteMeta))
+            //render page in suspend environment
+            val html = HtmlPage.createHtmlString(pageContext, data, content)
+
+            call.respond(TextContent(html, ContentType.Text.Html.withCharset(Charsets.UTF_8), HttpStatusCode.OK))
+        }
+    }
+
+    override fun route(route: Name, data: DataSet<*>, siteMeta: Meta, content: HtmlSite) {
+        val siteContext = SiteContextWithData(
+            KtorSiteContext(
+                context,
+                siteMeta = Laminate(siteMeta, this@KtorSiteContext.siteMeta),
+                baseUrl = baseUrl,
+                route = route,
+                ktorRoute = ktorRoute.createRouteFromPath(route.toWebPath())
+            ),
+            data
+        )
+        with(content) {
+            with(siteContext) {
+                renderSite()
+            }
+        }
+    }
+
+    override fun site(route: Name, data: DataSet<*>, siteMeta: Meta, content: HtmlSite) {
+        val siteContext = SiteContextWithData(
+            KtorSiteContext(
+                context,
+                siteMeta = Laminate(siteMeta, this@KtorSiteContext.siteMeta),
+                baseUrl = resolveRef(baseUrl, route.toWebPath()),
+                route = Name.EMPTY,
+                ktorRoute = ktorRoute.createRouteFromPath(route.toWebPath())
+            ),
+            data
+        )
+        with(content) {
+            with(siteContext) {
+                renderSite()
+            }
+        }
+    }
+
+}
+
+public fun Route.site(
+    context: Context,
+    data: DataSet<*>,
+    baseUrl: String = "",
+    siteMeta: Meta = data.meta,
+    content: HtmlSite,
+) {
+    val siteContext = SiteContextWithData(
+        KtorSiteContext(context, siteMeta, baseUrl, route = Name.EMPTY, this@Route),
+        data
+    )
+    with(content) {
+        with(siteContext) {
+            renderSite()
+        }
+    }
+}
+//
+//public suspend fun Application.site(
+//    context: Context,
+//    data: DataSet<*>,
+//    baseUrl: String = "",
+//    siteMeta: Meta = data.meta,
+//    content: HtmlSite,
+//) {
+//    routing {}.site(context, data, baseUrl, siteMeta, content)
+//
+//}

snark-ktor/src/jvmMain/kotlin/space/kscience/snark/ktor/extractData.kt

@@ -9,31 +9,6 @@ import java.time.LocalDateTime
 import kotlin.io.path.*
 
 
-//public fun KtorSiteBuilder.extractResources(uri: URI, targetPath: Path): Path {
-//    if (Files.isDirectory(targetPath)) {
-//        logger.info { "Using existing data directory at $targetPath." }
-//    } else {
-//        logger.info { "Copying data from $uri into $targetPath." }
-//        targetPath.createDirectories()
-//        //Copy everything into a temporary directory
-//        FileSystems.newFileSystem(uri, emptyMap<String, Any>()).use { fs ->
-//            val rootPath: Path = fs.provider().getPath(uri)
-//            Files.walk(rootPath).forEach { source: Path ->
-//                if (source.isRegularFile()) {
-//                    val relative = source.relativeTo(rootPath).toString()
-//                    val destination: Path = targetPath.resolve(relative)
-//                    destination.parent.createDirectories()
-//                    Files.copy(source, destination)
-//                }
-//            }
-//        }
-//    }
-//    return targetPath
-//}
-//
-//public fun KtorSiteBuilder.extractResources(resource: String, targetPath: Path): Path =
-//    extractResources(javaClass.getResource(resource)!!.toURI(), targetPath)
-
 private const val DEPLOY_DATE_FILE = "deployDate"
 private const val BUILD_DATE_FILE = "/buildDate"
 

snark-ktor/src/main/kotlin/space/kscience/snark/ktor/KtorSiteBuilder.kt

@@ -1,239 +0,0 @@
-package space.kscience.snark.ktor
-
-import io.ktor.http.ContentType
-import io.ktor.http.URLBuilder
-import io.ktor.http.URLProtocol
-import io.ktor.http.fromFileExtension
-import io.ktor.server.application.Application
-import io.ktor.server.application.call
-import io.ktor.server.html.respondHtml
-import io.ktor.server.http.content.staticFiles
-import io.ktor.server.plugins.origin
-import io.ktor.server.response.respondBytes
-import io.ktor.server.routing.Route
-import io.ktor.server.routing.createRouteFromPath
-import io.ktor.server.routing.get
-import io.ktor.server.routing.routing
-import kotlinx.css.CssBuilder
-import kotlinx.html.CommonAttributeGroupFacade
-import kotlinx.html.HTML
-import kotlinx.html.style
-import space.kscience.dataforge.context.error
-import space.kscience.dataforge.context.logger
-import space.kscience.dataforge.data.DataTree
-import space.kscience.dataforge.data.DataTreeItem
-import space.kscience.dataforge.data.await
-import space.kscience.dataforge.data.getItem
-import space.kscience.dataforge.io.Binary
-import space.kscience.dataforge.io.toByteArray
-import space.kscience.dataforge.meta.*
-import space.kscience.dataforge.names.Name
-import space.kscience.dataforge.names.cutLast
-import space.kscience.dataforge.names.endsWith
-import space.kscience.dataforge.names.plus
-import space.kscience.dataforge.workspace.FileData
-import space.kscience.snark.html.SiteBuilder
-import space.kscience.snark.html.SnarkHtmlPlugin
-import space.kscience.snark.html.WebPage
-import space.kscience.snark.html.toWebPath
-import java.nio.file.Path
-import kotlin.contracts.InvocationKind
-import kotlin.contracts.contract
-import kotlin.reflect.typeOf
-
-public fun CommonAttributeGroupFacade.css(block: CssBuilder.() -> Unit) {
-    style = CssBuilder().block().toString()
-}
-
-public class KtorSiteBuilder(
-    override val snark: SnarkHtmlPlugin,
-    override val data: DataTree<*>,
-    override val siteMeta: Meta,
-    private val baseUrl: String,
-    override val route: Name,
-    private val ktorRoute: Route,
-) : SiteBuilder {
-
-    private fun files(item: DataTreeItem<Any>, routeName: Name) {
-        //try using direct file rendering
-        item.meta[FileData.FILE_PATH_KEY]?.string?.let {
-            val file = try {
-                Path.of(it).toFile()
-            } catch (ex: Exception) {
-                //failure,
-                logger.error { "File $it could not be converted to java.io.File" }
-                return@let
-            }
-
-            val fileName = routeName.toWebPath()
-            ktorRoute.staticFiles(fileName, file)
-            //success, don't do anything else
-            return@files
-        }
-        when (item) {
-            is DataTreeItem.Leaf -> {
-                val datum = item.data
-                if (datum.type != typeOf<Binary>()) error("Can't directly serve file of type ${item.data.type}")
-                ktorRoute.get(routeName.toWebPath()) {
-                    val binary = datum.await() as Binary
-                    val extension = item.meta[FileData.FILE_EXTENSION_KEY]?.string?.let { ".$it" } ?: ""
-                    val contentType: ContentType = extension
-                        .let(ContentType::fromFileExtension)
-                        .firstOrNull()
-                        ?: ContentType.Any
-                    call.respondBytes(contentType = contentType) {
-                        //TODO optimize using streaming
-                        binary.toByteArray()
-                    }
-                }
-            }
-
-            is DataTreeItem.Node -> {
-                item.tree.items.forEach { (token, childItem) ->
-                    files(childItem, routeName + token)
-                }
-            }
-        }
-    }
-
-    override fun static(dataName: Name, routeName: Name) {
-        val item: DataTreeItem<Any> = data.getItem(dataName) ?: error("Data with name $dataName is not resolved")
-        files(item, routeName)
-    }
-//
-//    override fun file(file: Path, webPath: String) {
-//        if (file.isDirectory()) {
-//            ktorRoute.static(webPath) {
-//                //TODO check non-standard FS and convert
-//                files(file.toFile())
-//            }
-//        } else if (webPath.isBlank()) {
-//            error("Can't mount file to an empty route")
-//        } else {
-//            ktorRoute.file(webPath, file.toFile())
-//        }
-//    }
-
-//    override fun file(dataName: Name, webPath: String) {
-//        val fileData = data[dataName]
-//        if(fileData is FileData){
-//            ktorRoute.file(webPath)
-//        }
-//    }
-
-    private fun resolveRef(baseUrl: String, ref: String) = if (baseUrl.isEmpty()) {
-        ref
-    } else if (ref.isEmpty()) {
-        baseUrl
-    } else {
-        "${baseUrl.removeSuffix("/")}/$ref"
-    }
-
-
-    private inner class KtorWebPage(
-        val pageBaseUrl: String,
-        override val pageMeta: Meta,
-    ) : WebPage {
-        override val snark: SnarkHtmlPlugin get() = this@KtorSiteBuilder.snark
-        override val data: DataTree<*> get() = this@KtorSiteBuilder.data
-
-        override fun resolveRef(ref: String): String = this@KtorSiteBuilder.resolveRef(pageBaseUrl, ref)
-
-        override fun resolvePageRef(
-            pageName: Name,
-            relative: Boolean,
-        ): String {
-            val fullPageName = if (relative) this@KtorSiteBuilder.route + pageName else pageName
-            return if (fullPageName.endsWith(SiteBuilder.INDEX_PAGE_TOKEN)) {
-                resolveRef(fullPageName.cutLast().toWebPath())
-            } else {
-                resolveRef(fullPageName.toWebPath())
-            }
-        }
-    }
-
-    override fun page(route: Name, pageMeta: Meta, content: context(HTML, WebPage) () -> Unit) {
-        ktorRoute.get(route.toWebPath()) {
-            call.respondHtml {
-                val request = call.request
-                //substitute host for url for backwards calls
-                val url = URLBuilder(baseUrl).apply {
-                    protocol = URLProtocol.createOrDefault(request.origin.scheme)
-                    host = request.origin.serverHost
-                    port = request.origin.serverPort
-                }
-
-                val modifiedPageMeta = pageMeta.toMutableMeta().apply {
-                    "name" put route.toString()
-                    "url" put url.buildString()
-                }
-
-                val pageBuilder = KtorWebPage(url.buildString(), Laminate(modifiedPageMeta, siteMeta))
-                content(this, pageBuilder)
-            }
-        }
-    }
-
-    override fun route(
-        routeName: Name,
-        dataOverride: DataTree<*>?,
-        routeMeta: Meta,
-    ): SiteBuilder = KtorSiteBuilder(
-        snark = snark,
-        data = dataOverride ?: data,
-        siteMeta = Laminate(routeMeta, siteMeta),
-        baseUrl = baseUrl,
-        route = this.route + routeName,
-        ktorRoute = ktorRoute.createRouteFromPath(routeName.toWebPath())
-    )
-
-    override fun site(
-        routeName: Name,
-        dataOverride: DataTree<*>?,
-        routeMeta: Meta,
-    ): SiteBuilder = KtorSiteBuilder(
-        snark = snark,
-        data = dataOverride ?: data,
-        siteMeta = Laminate(routeMeta, siteMeta),
-        baseUrl = resolveRef(baseUrl, routeName.toWebPath()),
-        route = Name.EMPTY,
-        ktorRoute = ktorRoute.createRouteFromPath(routeName.toWebPath())
-    )
-
-//
-//    override fun resourceFile(resourcesPath: String, webPath: String) {
-//        ktorRoute.resource(resourcesPath, resourcesPath)
-//    }
-
-//    override fun resourceDirectory(resourcesPath: String) {
-//        ktorRoute.resources(resourcesPath)
-//    }
-}
-
-private fun Route.site(
-    snarkHtmlPlugin: SnarkHtmlPlugin,
-    data: DataTree<*>,
-    baseUrl: String = "",
-    siteMeta: Meta = data.meta,
-    block: KtorSiteBuilder.() -> Unit,
-) {
-    contract {
-        callsInPlace(block, InvocationKind.EXACTLY_ONCE)
-    }
-    block(KtorSiteBuilder(snarkHtmlPlugin, data, siteMeta, baseUrl, route = Name.EMPTY, this@Route))
-}
-
-public fun Application.site(
-    snark: SnarkHtmlPlugin,
-    data: DataTree<*>,
-    baseUrl: String = "",
-    siteMeta: Meta = data.meta,
-    block: SiteBuilder.() -> Unit,
-) {
-    contract {
-        callsInPlace(block, InvocationKind.EXACTLY_ONCE)
-    }
-    routing {
-        site(snark, data, baseUrl, siteMeta, block)
-    }
-}

snark-pandoc/README.md

@@ -0,0 +1,66 @@
+## Examples
+### Simple converting
+Convert from INPUT_FILE to OUTPUT_FILE:
+```java
+PandocWrapper wrapper = new PandocWrapper();
+wrapper.use(p -> {
+        var command = new PandocCommandBuilder(List.of(INPUT_FILE), OUTPUT_FILE);
+        PandocWrapper.execute(command);
+        });
+```
+Equal to:
+```
+pandoc --output=OUTPUT_FILE INPUT_FILE
+```
+### Convert and set formats
+Convert from INPUT_FILE to OUTPUT_FILE and set INPUT_FORMAT and OUTPUT_FORMAT:
+```java
+PandocWrapper wrapper = new PandocWrapper();
+wrapper.use(p -> {
+        var command = new PandocCommandBuilder(List.of(INPUT_FILE), OUTPUT_FILE);
+        command.formatForm(INPUT_FORMAT);
+        command.formatTo(OUTPUT_FORMAT);
+        PandocWrapper.execute(command);
+        });
+```
+Equal to:
+```
+pandoc --output=OUTPUT_FILE --from=INPUT_FORMAT --to=OUTPUT_FORMAT INPUT_FILE
+```
+### Converting with options
+Convert from INPUT_FILE to standalone OUTPUT_FILE and set variable KEY to VALUE :
+```java
+PandocWrapper wrapper = new PandocWrapper();
+wrapper.use(p -> {
+        var command = new PandocCommandBuilder(List.of(INPUT_FILE), OUTPUT_FILE);
+        command.standalone();
+        command.setVariable(KEY, VALUE);
+        PandocWrapper.execute(command);
+        });
+```
+Equal to:
+```
+pandoc --output=OUTPUT_FILE --standalone --variable=KEY:VALUE INPUT_FILE
+```
+
+### Write output from pandoc to file
+Receive possible input formats in OUTPUT_FILE:
+```java
+PandocWrapper wrapper = new PandocWrapper();
+wrapper.use(p -> {
+        var command = new PandocCommandBuilder();
+        command.getInputFormats();
+        PandocWrapper.execute(command, OUTPUT_FILE);
+        });
+```
+Then in OUTPUT_FILE will be a list supported input formats, one per line.
+
+### Write errors from pandoc to file
+Receive all from error stream and exit code in ERROR_FILE and output in OUTPUT_FILE:
+```java
+PandocWrapper wrapper = new PandocWrapper();
+wrapper.use(p -> {
+        var command = new PandocCommandBuilder(List.of(INPUT_FILE), OUTPUT_FILE);
+        PandocWrapper.execute(command, OUTPUT_FILE, ERROR_FILE);
+        });
+```

snark-pandoc/build.gradle.kts

@@ -0,0 +1,20 @@
+plugins {
+    id("space.kscience.gradle.mpp")
+}
+
+
+kscience {
+    useSerialization {
+        json()
+    }
+    jvm()
+    jvmMain {
+        api(spclibs.slf4j)
+        implementation("org.apache.commons:commons-exec:1.3")
+        implementation("org.apache.commons:commons-compress:1.2")
+    }
+    jvmTest{
+        implementation(spclibs.logback.classic)
+    }
+
+}

snark-pandoc/src/jvmMain/kotlin/space/kscience/snark/pandoc/Pandoc.kt

@@ -0,0 +1,56 @@
+package space.kscience.snark.pandoc
+
+import org.slf4j.Logger
+import org.slf4j.LoggerFactory
+import java.io.IOException
+import java.nio.file.Files
+import java.nio.file.Path
+import java.util.concurrent.TimeUnit
+import kotlin.io.path.Path
+
+public object Pandoc {
+    private val logger: Logger = LoggerFactory.getLogger(Pandoc::class.java)
+
+    private fun getOrInstallPandoc(pandocExecutablePath: Path): String = try {
+        ProcessBuilder("pandoc", "--version").start().waitFor()
+        "pandoc"
+    } catch (ex: IOException) {
+        if (Files.exists(pandocExecutablePath)) {
+            pandocExecutablePath.toAbsolutePath().toString()
+        } else {
+            logger.info("Pandoc not found in the system. Installing it from GitHub")
+            PandocInstaller.installPandoc(pandocExecutablePath).toAbsolutePath().toString()
+        }
+    }
+
+    /**
+     * Call pandoc with options described by commandBuilder.
+     * @param commandBuilder
+     * @return true if successfully false otherwise
+     */
+    public fun execute(
+        redirectOutput: Path? = null,
+        redirectError: Path? = null,
+        pandocExecutablePath: Path = Path("./pandoc").toAbsolutePath(),
+        commandBuilder: PandocCommandBuilder.() -> Unit,
+    ) {
+
+        val path = getOrInstallPandoc(pandocExecutablePath)
+
+        val commandLine = PandocCommandBuilder().apply(commandBuilder).build(path)
+        logger.info("Running pandoc: ${commandLine.joinToString(separator = " ")}")
+        val pandoc = ProcessBuilder(commandLine).apply {
+            if (redirectOutput != null) {
+                redirectOutput(redirectOutput.toFile())
+            }
+            if (redirectError != null) {
+                redirectError(redirectError.toFile())
+            }
+
+        }.start()
+        pandoc.waitFor()
+
+        if (pandoc.exitValue() != 0)
+            error("Non-zero process return for pandoc.")
+    }
+}

snark-pandoc/src/jvmMain/kotlin/space/kscience/snark/pandoc/PandocInstaller.kt

@@ -0,0 +1,266 @@
+package space.kscience.snark.pandoc
+
+import kotlinx.serialization.json.Json
+import org.apache.commons.compress.archivers.ArchiveEntry
+import org.apache.commons.compress.archivers.tar.TarArchiveInputStream
+import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream
+import org.apache.commons.exec.OS
+import org.slf4j.Logger
+import org.slf4j.LoggerFactory
+import java.io.BufferedInputStream
+import java.io.FileNotFoundException
+import java.io.IOException
+import java.net.*
+import java.net.http.HttpClient
+import java.net.http.HttpRequest
+import java.net.http.HttpResponse
+import java.nio.file.Files
+import java.nio.file.Path
+import java.nio.file.attribute.PosixFilePermission
+import java.time.Duration
+import java.util.*
+import java.util.zip.ZipInputStream
+import kotlin.io.path.Path
+import kotlin.io.path.inputStream
+
+internal object PandocInstaller {
+
+    private val log: Logger = LoggerFactory.getLogger(PandocInstaller::class.java)
+
+
+    private const val TIMEOUT_SECONDS = 2
+    private const val ATTEMPTS = 3
+
+    private enum class OSType(val assetSuffix: String, val propertySuffix: String) {
+        WINDOWS("windows-x86_64.zip", "windows"),
+        MAC_OS_AMD("x86_64-macOS.zip", "mac.os.amd"),
+        MAC_OS_ARM("arm64-macOS.zip", "mac.os.arm"),
+        LINUX_ARM("linux-arm64", "linux.arm"),
+        LINUX_AMD("linux-amd64", "linux.amd")
+    }
+
+    private val properties = Properties().apply {
+        load(PandocInstaller.javaClass.getResourceAsStream("/installer.properties")!!)
+    }
+
+    /**
+     * Install last released pandoc from github
+     * @return path to executable pandoc
+     * @throws IOException in case incorrect github url or path of installation directory
+     */
+    public fun installPandoc(targetPath: Path): Path {
+        log.info("Start install")
+        return if (OS.isFamilyMac()) {
+            if (OS.isArch("aarch64")) {
+                installPandoc(OSType.MAC_OS_ARM, targetPath)
+            } else {
+                installPandoc(OSType.MAC_OS_AMD, targetPath)
+            }
+        } else if (OS.isFamilyUnix()) {
+            if (OS.isArch("aarch64")) {
+                installPandoc(OSType.LINUX_ARM, targetPath)
+            } else {
+                installPandoc(OSType.LINUX_AMD, targetPath)
+            }
+        } else if (OS.isFamilyWindows()) {
+            installPandoc(OSType.WINDOWS, targetPath)
+        } else {
+            error("Got unexpected os, could not install pandoc")
+        }
+    }
+
+
+    private fun installPandoc(os: OSType, targetPath: Path): Path {
+
+        val githubResponse = getGithubUrls()
+        val asset = githubResponse.getAssetByOsSuffix(os.assetSuffix)
+        val currUrl = asset.browserDownloadUrl
+
+        val pandocUrl: URL = URI.create(currUrl).toURL()
+        val fileToInstall: Path = when (os) {
+            OSType.LINUX_AMD, OSType.LINUX_ARM -> Path("$targetPath/pandoc.tar.gz")
+            else -> Path("$targetPath/pandoc.zip")
+        }
+
+        log.info(
+            "Start installing pandoc os: {}, url: {}, file: {}",
+            os,
+            pandocUrl,
+            fileToInstall
+        )
+
+        val archivePath = downloadWithRetry(pandocUrl) ?: error("Could not save file from github")
+        val installPath = unPack(archivePath, targetPath, os) ?: error("Could not unzip file")
+
+
+        val pandocExecutablePath = installPath.resolve(
+            properties.getProperty("path.to.pandoc." + os.propertySuffix).replace(
+                "{version}",
+                githubResponse.tagName
+            )
+        )
+
+        if (os == OSType.LINUX_AMD || os == OSType.LINUX_ARM) {
+            Files.setPosixFilePermissions(pandocExecutablePath, setOf(PosixFilePermission.GROUP_EXECUTE))
+        }
+
+        return pandocExecutablePath
+    }
+
+    /**
+     * Downloads from a (http/https) URL and saves to a file.
+     * @param target File to write. Parent directory will be created if necessary
+     * @param url  http/https url to connect
+     * @param secsConnectTimeout Seconds to wait for connection establishment
+     * @param secsReadTimeout Read timeout in seconds - trasmission will abort if it freezes more than this
+     * @return true if successfully save file and false if:
+     * connection interrupted, timeout (but something was read)
+     * server error (500...)
+     * could not connect: connection timeout java.net.SocketTimeoutException
+     * could not connect: java.net.ConnectException
+     * could not resolve host (bad host, or no internet - no dns)
+     * @throws IOException Only if URL is malformed or if could not create the file
+     * @throws FileNotFoundException if did not find file for save
+     */
+    @Throws(IOException::class)
+    private fun downloadUrl(
+        target: Path,
+        url: URL,
+        secsConnectTimeout: Int,
+        secsReadTimeout: Int,
+    ): Path? {
+        Files.createDirectories(target.parent) // make sure parent dir exists , this can throw exception
+        val conn = url.openConnection() // can throw exception if bad url
+        if (secsConnectTimeout > 0) {
+            conn.connectTimeout = secsConnectTimeout * 1000
+        }
+        if (secsReadTimeout > 0) {
+            conn.readTimeout = secsReadTimeout * 1000
+        }
+        var ret = true
+        var somethingRead = false
+        try {
+            conn.getInputStream().use { `is` ->
+                BufferedInputStream(`is`).use { `in` ->
+                    Files.newOutputStream(target).use { fout ->
+                        val data = ByteArray(8192)
+                        var count: Int
+                        while ((`in`.read(data).also { count = it }) > 0) {
+                            somethingRead = true
+                            fout.write(data, 0, count)
+                        }
+                    }
+                }
+            }
+            return target
+        } catch (e: IOException) {
+            var httpcode = 999
+            try {
+                httpcode = (conn as HttpURLConnection).responseCode
+            } catch (ee: Exception) {
+            }
+
+            if (e is FileNotFoundException) {
+                throw FileNotFoundException("Did not found file for install")
+            }
+
+            if (somethingRead && e is SocketTimeoutException) {
+                log.error("Read something, but connection interrupted: {}", e.message, e)
+            } else if (httpcode >= 400 && httpcode < 600) {
+                log.error("Got server error, httpcode: {}", httpcode)
+            } else if (e is SocketTimeoutException) {
+                log.error("Connection timeout: {}", e.message, e)
+            } else if (e is ConnectException) {
+                log.error("Could not connect: {}", e.message, e)
+            } else if (e is UnknownHostException) {
+                log.error("Could not resolve host: {}", e.message, e)
+            } else {
+                throw e
+            }
+            return null
+        }
+    }
+
+    private fun downloadWithRetry(url: URL): Path? {
+        val targetPath = Files.createTempFile("pandoc", ".tmp")
+        log.info("Downloading pandoc to $targetPath")
+
+        repeat(ATTEMPTS) {
+            return downloadUrl(
+                targetPath,
+                url,
+                TIMEOUT_SECONDS,
+                TIMEOUT_SECONDS
+            )
+        }
+
+        return null
+    }
+
+    private fun unPack(sourcePath: Path, targetPath: Path, os: OSType): Path? {
+        try {
+            when (os) {
+                OSType.LINUX_AMD, OSType.LINUX_ARM -> unTarGz(sourcePath, targetPath)
+
+                else -> unZip(sourcePath, targetPath)
+            }
+        } catch (e: IOException) {
+            log.error("Could not perform unpacking: {}", e.message, e)
+            return null
+        }
+        return targetPath
+    }
+
+    private fun unTarGz(sourcePath: Path, targetDir: Path) {
+        TarArchiveInputStream(
+            GzipCompressorInputStream(
+                BufferedInputStream(Files.newInputStream(sourcePath))
+            )
+        ).use { tarIn ->
+            var archiveEntry: ArchiveEntry
+            while ((tarIn.nextEntry.also { archiveEntry = it }) != null) {
+                val pathEntryOutput = targetDir.resolve(archiveEntry.name)
+                if (archiveEntry.isDirectory) {
+                    Files.createDirectory(pathEntryOutput)
+                } else {
+                    Files.copy(tarIn, pathEntryOutput)
+                }
+            }
+        }
+    }
+
+    private fun unZip(sourcePath: Path, targetDir: Path) {
+        ZipInputStream(sourcePath.inputStream()).use { zis ->
+            do {
+                val entry = zis.nextEntry
+                if (entry == null) continue
+                val pathEntryOutput = targetDir.resolve(entry.name)
+                if (entry.isDirectory) {
+                    Files.createDirectories(pathEntryOutput)
+                } else {
+                    Files.createDirectories(pathEntryOutput.parent)
+                    Files.copy(zis, pathEntryOutput)
+                }
+                zis.closeEntry()
+            } while (entry != null)
+        }
+    }
+
+    private fun getGithubUrls(): ResponseDto {
+        val uri = URI.create(properties.getProperty("github.url"))
+        val client = HttpClient.newHttpClient()
+        val request = HttpRequest
+            .newBuilder()
+            .uri(uri)
+            .version(HttpClient.Version.HTTP_2)
+            .timeout(Duration.ofMinutes(1))
+            .header("Accept", "application/vnd.github+json")
+            .GET()
+            .build()
+
+        val response = client.send(request, HttpResponse.BodyHandlers.ofString())
+        log.info("Got response from github, status: {}", response.statusCode())
+
+        return Json { ignoreUnknownKeys = true }.decodeFromString(ResponseDto.serializer(), response.body())
+    }
+}

snark-pandoc/src/jvmMain/kotlin/space/kscience/snark/pandoc/ResponseDto.kt

@@ -0,0 +1,34 @@
+package space.kscience.snark.pandoc
+
+import kotlinx.serialization.SerialName
+import kotlinx.serialization.Serializable
+
+/**
+ * Response from github/releases/latest
+ */
+@Serializable
+internal class ResponseDto(
+    val assets: Array<AssetDto>,
+    @SerialName("tag_name") val tagName: String,
+) {
+    /**
+     * @param osSuffix
+     * @return asset appropriate to os
+     */
+    fun getAssetByOsSuffix(osSuffix: String?): AssetDto {
+        for (asset in assets) {
+            if (asset.name.contains(osSuffix!!)) {
+                return asset
+            }
+        }
+        throw IllegalArgumentException("Unexpected osSuffix")
+    }
+
+
+    @Serializable
+    public class AssetDto(
+        @SerialName("browser_download_url") val browserDownloadUrl: String,
+        val name: String
+    )
+
+}

snark-pandoc/src/jvmMain/resources/installer.properties

@@ -0,0 +1,8 @@
+path.to.pandoc.mac.os.arm=/pandoc-{version}-arm64/bin/pandoc
+path.to.pandoc.mac.os.amd=/pandoc-{version}-x86_64/bin/pandoc
+path.to.pandoc.windows=/pandoc-{version}/pandoc.exe
+path.to.pandoc.linux.amd=/pandoc-{version}/bin/pandoc
+path.to.pandoc.linux.arm=/pandoc-{version}/bin/pandoc
+
+github.url=https://api.github.com/repos/jgm/pandoc/releases/latest
+

snark-pandoc/src/jvmTest/kotlin/PandocTest.kt

@@ -0,0 +1,105 @@
+import org.junit.jupiter.api.Assertions.assertFalse
+import org.junit.jupiter.api.Assertions.assertTrue
+import org.junit.jupiter.api.Test
+import space.kscience.snark.pandoc.Pandoc
+import java.nio.file.Files
+import java.nio.file.Path
+import kotlin.io.path.exists
+import kotlin.io.path.isDirectory
+import kotlin.io.path.readText
+import kotlin.io.path.writeBytes
+import kotlin.test.assertContains
+import kotlin.test.assertFails
+
+class PandocTest {
+    @Test
+    fun when_gotPandocAndCorrectArgs_doConverting() {
+
+        val inputFile = Files.createTempFile("snark-pandoc", "first_test.md")
+        inputFile.writeBytes(javaClass.getResourceAsStream("/first_test.md")!!.readAllBytes())
+        val outputFile = Files.createTempFile("snark-pandoc", "output1.tex")
+
+        Pandoc.execute {
+            addInputFile(inputFile)
+            outputFile(outputFile)
+        }
+
+        assertTrue(outputFile.exists())
+
+        val result = outputFile.readText()
+
+        assertContains(result, "Some simple text")
+        assertContains(result, "\\subsection{Copy elision}")
+        assertContains(result, "return")
+    }
+
+    @Test
+    fun when_gotPandocAndNotExistsFromFile_then_error() {
+
+        val outputFile = Files.createTempFile("snark-pandoc", "output2.tex")
+        val notExistsFile = Path.of("./src/test/testing_directory/non_exists_test.md")
+        assertFalse(notExistsFile.exists())
+        assertFails {
+            Pandoc.execute {
+                addInputFile(notExistsFile)
+                outputFile(outputFile)
+            }
+        }
+    }
+
+    @Test
+    fun when_gotPandocAndPassDirectory_then_error() {
+        val tempDir = Files.createTempDirectory("snark-pandoc")
+        assertTrue(tempDir.isDirectory())
+
+        val outputFile = Files.createTempFile("snark-pandoc", "output3.tex")
+
+        assertFails {
+            Pandoc.execute {
+                addInputFile(tempDir)
+                outputFile(outputFile)
+            }
+        }
+
+    }
+
+    @Test
+    fun when_askVersionToFile_then_Ok() {
+        val outputFile = Files.createTempFile("snark-pandoc", "output4.tex")
+
+        val res = Pandoc.execute(redirectOutput = outputFile) {
+            getVersion()
+        }
+
+        val fileContent = outputFile.readText()
+        assertContains(fileContent, "pandoc")
+        assertContains(fileContent, "This is free software")
+    }
+
+    @Test
+    fun when_error_then_writeToErrorStream() {
+        val inputFile = Files.createTempFile("snark-pandoc", "simple.txt")
+        inputFile.writeBytes(javaClass.getResourceAsStream("/simple.txt")!!.readAllBytes())
+        val outputFile = Files.createTempFile("snark-pandoc", "output.txt")
+        val errorFile = Files.createTempFile("snark-pandoc", "error.txt")
+
+        assertFails {
+            Pandoc.execute(redirectError = errorFile) {
+                addInputFile(inputFile)
+                outputFile(outputFile)
+                formatFrom("txt")
+            }
+        }
+
+        assertContains(errorFile.readText(), "input format")
+    }
+
+
+//    @Test
+//    fun when_installPandoc_thenFindIt() {
+//        PandocInstaller.clearInstallingDirectory()
+//        assertTrue(Pandoc.installPandoc())
+//        assertTrue(Pandoc.isPandocInstalled())
+//    }
+
+}

snark-pandoc/src/jvmTest/resources/first_test.md

@@ -0,0 +1,15 @@
+
+
+## Copy elision
+### RVO/NRVO
+Some simple text
+```c++
+A f() {
+    return {5};
+}
+
+A g() {
+    A a(5);
+    return a;
+}
+```

snark-pandoc/src/jvmTest/resources/simple.txt

@@ -0,0 +1 @@
+hello