Initial commit

.gitignore

@@ -0,0 +1,6 @@
+.gradle/
+build/
+.idea/
+/logs/
+
+!gradle/wrapper/gradle-wrapper.jar

build.gradle.kts

@@ -0,0 +1,18 @@
+plugins {
+    id("ru.mipt.npm.gradle.project")
+}
+
+allprojects {
+    group = "space.kscience"
+    version = "0.1.0-dev-1"
+
+    if(name!="snark-gradle-plugin") {
+        tasks.withType<org.jetbrains.kotlin.gradle.tasks.KotlinCompile> {
+            kotlinOptions {
+                freeCompilerArgs = freeCompilerArgs + "-Xcontext-receivers"
+            }
+        }
+    }
+}
+
+val dataforgeVersion by extra("0.6.0-dev-9")

gradle.properties

@@ -0,0 +1,3 @@
+kotlin.code.style=official
+
+toolsVersion=0.11.7-kotlin-1.7.0

gradle/wrapper/gradle-wrapper.properties

@@ -0,0 +1,5 @@
+distributionBase=GRADLE_USER_HOME
+distributionPath=wrapper/dists
+distributionUrl=https\://services.gradle.org/distributions/gradle-7.4.2-bin.zip
+zipStoreBase=GRADLE_USER_HOME
+zipStorePath=wrapper/dists

gradlew

@@ -0,0 +1,234 @@
+#!/bin/sh
+
+#
+# Copyright © 2015-2021 the original authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+##############################################################################
+#
+#   Gradle start up script for POSIX generated by Gradle.
+#
+#   Important for running:
+#
+#   (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is
+#       noncompliant, but you have some other compliant shell such as ksh or
+#       bash, then to run this script, type that shell name before the whole
+#       command line, like:
+#
+#           ksh Gradle
+#
+#       Busybox and similar reduced shells will NOT work, because this script
+#       requires all of these POSIX shell features:
+#         * functions;
+#         * expansions «$var», «${var}», «${var:-default}», «${var+SET}»,
+#           «${var#prefix}», «${var%suffix}», and «$( cmd )»;
+#         * compound commands having a testable exit status, especially «case»;
+#         * various built-in commands including «command», «set», and «ulimit».
+#
+#   Important for patching:
+#
+#   (2) This script targets any POSIX shell, so it avoids extensions provided
+#       by Bash, Ksh, etc; in particular arrays are avoided.
+#
+#       The "traditional" practice of packing multiple parameters into a
+#       space-separated string is a well documented source of bugs and security
+#       problems, so this is (mostly) avoided, by progressively accumulating
+#       options in "$@", and eventually passing that to Java.
+#
+#       Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS,
+#       and GRADLE_OPTS) rely on word-splitting, this is performed explicitly;
+#       see the in-line comments for details.
+#
+#       There are tweaks for specific operating systems such as AIX, CygWin,
+#       Darwin, MinGW, and NonStop.
+#
+#   (3) This script is generated from the Groovy template
+#       https://github.com/gradle/gradle/blob/master/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt
+#       within the Gradle project.
+#
+#       You can find Gradle at https://github.com/gradle/gradle/.
+#
+##############################################################################
+
+# Attempt to set APP_HOME
+
+# Resolve links: $0 may be a link
+app_path=$0
+
+# Need this for daisy-chained symlinks.
+while
+    APP_HOME=${app_path%"${app_path##*/}"}  # leaves a trailing /; empty if no leading path
+    [ -h "$app_path" ]
+do
+    ls=$( ls -ld "$app_path" )
+    link=${ls#*' -> '}
+    case $link in             #(
+      /*)   app_path=$link ;; #(
+      *)    app_path=$APP_HOME$link ;;
+    esac
+done
+
+APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit
+
+APP_NAME="Gradle"
+APP_BASE_NAME=${0##*/}
+
+# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
+DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'
+
+# Use the maximum available, or set MAX_FD != -1 to use that value.
+MAX_FD=maximum
+
+warn () {
+    echo "$*"
+} >&2
+
+die () {
+    echo
+    echo "$*"
+    echo
+    exit 1
+} >&2
+
+# OS specific support (must be 'true' or 'false').
+cygwin=false
+msys=false
+darwin=false
+nonstop=false
+case "$( uname )" in                #(
+  CYGWIN* )         cygwin=true  ;; #(
+  Darwin* )         darwin=true  ;; #(
+  MSYS* | MINGW* )  msys=true    ;; #(
+  NONSTOP* )        nonstop=true ;;
+esac
+
+CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
+
+
+# Determine the Java command to use to start the JVM.
+if [ -n "$JAVA_HOME" ] ; then
+    if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
+        # IBM's JDK on AIX uses strange locations for the executables
+        JAVACMD=$JAVA_HOME/jre/sh/java
+    else
+        JAVACMD=$JAVA_HOME/bin/java
+    fi
+    if [ ! -x "$JAVACMD" ] ; then
+        die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME
+
+Please set the JAVA_HOME variable in your environment to match the
+location of your Java installation."
+    fi
+else
+    JAVACMD=java
+    which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
+
+Please set the JAVA_HOME variable in your environment to match the
+location of your Java installation."
+fi
+
+# Increase the maximum file descriptors if we can.
+if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then
+    case $MAX_FD in #(
+      max*)
+        MAX_FD=$( ulimit -H -n ) ||
+            warn "Could not query maximum file descriptor limit"
+    esac
+    case $MAX_FD in  #(
+      '' | soft) :;; #(
+      *)
+        ulimit -n "$MAX_FD" ||
+            warn "Could not set maximum file descriptor limit to $MAX_FD"
+    esac
+fi
+
+# Collect all arguments for the java command, stacking in reverse order:
+#   * args from the command line
+#   * the main class name
+#   * -classpath
+#   * -D...appname settings
+#   * --module-path (only if needed)
+#   * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables.
+
+# For Cygwin or MSYS, switch paths to Windows format before running java
+if "$cygwin" || "$msys" ; then
+    APP_HOME=$( cygpath --path --mixed "$APP_HOME" )
+    CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" )
+
+    JAVACMD=$( cygpath --unix "$JAVACMD" )
+
+    # Now convert the arguments - kludge to limit ourselves to /bin/sh
+    for arg do
+        if
+            case $arg in                                #(
+              -*)   false ;;                            # don't mess with options #(
+              /?*)  t=${arg#/} t=/${t%%/*}              # looks like a POSIX filepath
+                    [ -e "$t" ] ;;                      #(
+              *)    false ;;
+            esac
+        then
+            arg=$( cygpath --path --ignore --mixed "$arg" )
+        fi
+        # Roll the args list around exactly as many times as the number of
+        # args, so each arg winds up back in the position where it started, but
+        # possibly modified.
+        #
+        # NB: a `for` loop captures its iteration list before it begins, so
+        # changing the positional parameters here affects neither the number of
+        # iterations, nor the values presented in `arg`.
+        shift                   # remove old arg
+        set -- "$@" "$arg"      # push replacement arg
+    done
+fi
+
+# Collect all arguments for the java command;
+#   * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of
+#     shell script including quotes and variable substitutions, so put them in
+#     double quotes to make sure that they get re-expanded; and
+#   * put everything else in single quotes, so that it's not re-expanded.
+
+set -- \
+        "-Dorg.gradle.appname=$APP_BASE_NAME" \
+        -classpath "$CLASSPATH" \
+        org.gradle.wrapper.GradleWrapperMain \
+        "$@"
+
+# Use "xargs" to parse quoted args.
+#
+# With -n1 it outputs one arg per line, with the quotes and backslashes removed.
+#
+# In Bash we could simply go:
+#
+#   readarray ARGS < <( xargs -n1 <<<"$var" ) &&
+#   set -- "${ARGS[@]}" "$@"
+#
+# but POSIX shell has neither arrays nor command substitution, so instead we
+# post-process each arg (as a line of input to sed) to backslash-escape any
+# character that might be a shell metacharacter, then use eval to reverse
+# that process (while maintaining the separation between arguments), and wrap
+# the whole thing up as a single "set" statement.
+#
+# This will of course break if any of these variables contains a newline or
+# an unmatched quote.
+#
+
+eval "set -- $(
+        printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" |
+        xargs -n1 |
+        sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' |
+        tr '\n' ' '
+    )" '"$@"'
+
+exec "$JAVACMD" "$@"

gradlew.bat

@@ -0,0 +1,89 @@
+@rem
+@rem Copyright 2015 the original author or authors.
+@rem
+@rem Licensed under the Apache License, Version 2.0 (the "License");
+@rem you may not use this file except in compliance with the License.
+@rem You may obtain a copy of the License at
+@rem
+@rem      https://www.apache.org/licenses/LICENSE-2.0
+@rem
+@rem Unless required by applicable law or agreed to in writing, software
+@rem distributed under the License is distributed on an "AS IS" BASIS,
+@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+@rem See the License for the specific language governing permissions and
+@rem limitations under the License.
+@rem
+
+@if "%DEBUG%" == "" @echo off
+@rem ##########################################################################
+@rem
+@rem  Gradle startup script for Windows
+@rem
+@rem ##########################################################################
+
+@rem Set local scope for the variables with windows NT shell
+if "%OS%"=="Windows_NT" setlocal
+
+set DIRNAME=%~dp0
+if "%DIRNAME%" == "" set DIRNAME=.
+set APP_BASE_NAME=%~n0
+set APP_HOME=%DIRNAME%
+
+@rem Resolve any "." and ".." in APP_HOME to make it shorter.
+for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi
+
+@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
+set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m"
+
+@rem Find java.exe
+if defined JAVA_HOME goto findJavaFromJavaHome
+
+set JAVA_EXE=java.exe
+%JAVA_EXE% -version >NUL 2>&1
+if "%ERRORLEVEL%" == "0" goto execute
+
+echo.
+echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
+echo.
+echo Please set the JAVA_HOME variable in your environment to match the
+echo location of your Java installation.
+
+goto fail
+
+:findJavaFromJavaHome
+set JAVA_HOME=%JAVA_HOME:"=%
+set JAVA_EXE=%JAVA_HOME%/bin/java.exe
+
+if exist "%JAVA_EXE%" goto execute
+
+echo.
+echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
+echo.
+echo Please set the JAVA_HOME variable in your environment to match the
+echo location of your Java installation.
+
+goto fail
+
+:execute
+@rem Setup the command line
+
+set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
+
+
+@rem Execute Gradle
+"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %*
+
+:end
+@rem End local scope for the variables with windows NT shell
+if "%ERRORLEVEL%"=="0" goto mainEnd
+
+:fail
+rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
+rem the _cmd.exe /c_ return code!
+if  not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
+exit /b 1
+
+:mainEnd
+if "%OS%"=="Windows_NT" endlocal
+
+:omega

settings.gradle.kts

@@ -0,0 +1,45 @@
+rootProject.name = "snark"
+
+enableFeaturePreview("TYPESAFE_PROJECT_ACCESSORS")
+enableFeaturePreview("VERSION_CATALOGS")
+
+pluginManagement {
+
+    val toolsVersion: String by extra
+
+    repositories {
+        maven("https://repo.kotlin.link")
+        mavenCentral()
+        gradlePluginPortal()
+    }
+
+    plugins {
+        id("ru.mipt.npm.gradle.project") version toolsVersion
+        id("ru.mipt.npm.gradle.mpp") version toolsVersion
+        id("ru.mipt.npm.gradle.jvm") version toolsVersion
+        id("ru.mipt.npm.gradle.js") version toolsVersion
+    }
+}
+
+dependencyResolutionManagement {
+
+    val toolsVersion: String by extra
+
+    repositories {
+        maven("https://repo.kotlin.link")
+        mavenCentral()
+    }
+
+    versionCatalogs {
+        create("npmlibs") {
+            from("ru.mipt.npm:version-catalog:$toolsVersion")
+        }
+    }
+}
+
+include(
+    ":snark-gradle-plugin",
+    ":snark-core",
+    ":snark-html",
+    ":snark-ktor"
+)

snark-core/build.gradle.kts

@@ -0,0 +1,16 @@
+plugins{
+    id("ru.mipt.npm.gradle.mpp")
+    `maven-publish`
+}
+
+val dataforgeVersion: String by rootProject.extra
+
+kotlin{
+    sourceSets{
+        commonMain{
+            dependencies{
+                api("space.kscience:dataforge-workspace:$dataforgeVersion")
+            }
+        }
+    }
+}

snark-core/src/commonMain/kotlin/space/kscience/snark/SnarkContext.kt

@@ -0,0 +1,6 @@
+package space.kscience.snark
+
+/**
+ * A marker interface for Snark Page and Site builders
+ */
+public interface SnarkContext

snark-core/src/commonMain/kotlin/space/kscience/snark/SnarkParser.kt

@@ -0,0 +1,34 @@
+package space.kscience.snark
+
+import io.ktor.utils.io.core.Input
+import io.ktor.utils.io.core.readBytes
+import space.kscience.dataforge.context.Context
+import space.kscience.dataforge.io.IOReader
+import space.kscience.dataforge.meta.Meta
+import space.kscience.dataforge.misc.Type
+import kotlin.reflect.KType
+
+/**
+ * A parser of binary content including priority flag and file extensions
+ */
+@Type(SnarkParser.TYPE)
+public interface SnarkParser<out R> {
+    public val type: KType
+
+    public val fileExtensions: Set<String>
+
+    public val priority: Int get() = DEFAULT_PRIORITY
+
+    public fun parse(context: Context, meta: Meta, bytes: ByteArray): R
+
+    public fun reader(context: Context, meta: Meta): IOReader<R> = object : IOReader<R> {
+        override val type: KType get() = this@SnarkParser.type
+
+        override fun readObject(input: Input): R = parse(context, meta, input.readBytes())
+    }
+
+    public companion object {
+        public const val TYPE: String = "snark.parser"
+        public const val DEFAULT_PRIORITY: Int = 10
+    }
+}

snark-gradle-plugin/build.gradle.kts

@@ -0,0 +1,25 @@
+plugins{
+    id("ru.mipt.npm.gradle.jvm")
+    `kotlin-dsl`
+    `java-gradle-plugin`
+    `maven-publish`
+}
+
+repositories{
+    gradlePluginPortal()
+}
+
+dependencies{
+    implementation(npmlibs.kotlin.gradle)
+    implementation("com.github.mwiede:jsch:0.2.1")
+}
+
+gradlePlugin{
+    plugins {
+        create("snark-gradle") {
+            id = "space.kscience.snark"
+            description = "A plugin for snark-based websites"
+            implementationClass = "space.kscience.snark.plugin.SnarkGradlePlugin"
+        }
+    }
+}

snark-gradle-plugin/src/main/kotlin/SnarkGradlePlugin.kt

@@ -0,0 +1,53 @@
+package space.kscience.snark.plugin
+
+import org.gradle.api.Plugin
+import org.gradle.api.Project
+import org.gradle.kotlin.dsl.configure
+import org.jetbrains.kotlin.gradle.dsl.KotlinJvmProjectExtension
+import java.io.File
+import java.time.LocalDateTime
+
+public class SnarkExtension(private val project: Project) {
+
+    private var _dataDirectory: File? = null
+    public var dataDirectory: File
+        get() = _dataDirectory ?: project.rootDir.resolve(DEFAULT_DATA_PATH)
+        set(value) {
+            _dataDirectory = value
+        }
+
+    public companion object {
+        public const val DEFAULT_DATA_PATH: String = "data"
+    }
+}
+
+
+public class SnarkGradlePlugin : Plugin<Project> {
+    override fun apply(target: Project): Unit = with(target) {
+        val snarkExtension = SnarkExtension(this)
+        extensions.add("snark", snarkExtension)
+
+        plugins.withId("org.jetbrains.kotlin.jvm") {
+            val writeBuildDate = tasks.register("writeBuildDate") {
+                val outputFile = File(project.buildDir, "resources/main/buildDate")
+                doLast {
+                    val deployDate = LocalDateTime.now()
+                    outputFile.parentFile.mkdirs()
+                    outputFile.writeText(deployDate.toString())
+                }
+                outputs.file(outputFile)
+                outputs.upToDateWhen { false }
+            }
+
+            tasks.getByName("processResources").dependsOn(writeBuildDate)
+
+            configure<KotlinJvmProjectExtension> {
+                sourceSets.apply {
+                    getByName("main") {
+                        resources.srcDir(project.rootDir.resolve("data"))
+                    }
+                }
+            }
+        }
+    }
+}

snark-gradle-plugin/src/main/kotlin/uploads.kt

@@ -0,0 +1,100 @@
+package space.kscience.snark.plugin
+
+import com.jcraft.jsch.*
+import java.io.File
+import java.io.FileInputStream
+import java.util.*
+
+/**
+ * https://kodehelp.com/java-program-uploading-folder-content-recursively-from-local-to-sftp-server/
+ */
+private fun ChannelSftp.recursiveFolderUpload(sourceFile: File, destinationPath: String) {
+    if (sourceFile.isFile) {
+        // copy if it is a file
+        cd(destinationPath)
+        if (!sourceFile.name.startsWith(".")) put(
+            FileInputStream(sourceFile),
+            sourceFile.getName(),
+            ChannelSftp.OVERWRITE
+        )
+    } else {
+        val files = sourceFile.listFiles()
+        if (files != null && !sourceFile.getName().startsWith(".")) {
+            cd(destinationPath)
+            var attrs: SftpATTRS? = null
+            // check if the directory is already existing
+            val directoryPath = destinationPath + "/" + sourceFile.getName()
+            try {
+                attrs = stat(directoryPath)
+            } catch (e: Exception) {
+                println("$directoryPath does not exist")
+            }
+
+            // else create a directory
+            if (attrs != null) {
+                println("Directory $directoryPath exists IsDir=${attrs.isDir()}")
+            } else {
+                println("Creating directory $directoryPath")
+                mkdir(sourceFile.getName())
+            }
+            for (f in files) {
+                recursiveFolderUpload(f, destinationPath + "/" + sourceFile.getName())
+            }
+        }
+    }
+}
+
+public fun Session.uploadDirectory(
+    file: File,
+    targetDirectory: String,
+) {
+    var channel: ChannelSftp? = null
+    try {
+        val config = Properties()
+        config["StrictHostKeyChecking"] = "no"
+        channel = openChannel("sftp") as ChannelSftp // Open SFTP Channel
+        channel.connect()
+        channel.cd(targetDirectory) // Change Directory on SFTP Server
+        channel.recursiveFolderUpload(file, targetDirectory)
+    } finally {
+        channel?.disconnect()
+    }
+}
+
+public fun Session.execute(
+    command: String,
+): String {
+    var channel: ChannelExec? = null
+    try {
+        channel = openChannel("exec") as ChannelExec
+        channel.setCommand(command)
+        channel.inputStream = null
+        channel.setErrStream(System.err)
+        val input = channel.inputStream
+        channel.connect()
+        return input.use { it.readAllBytes().decodeToString() }
+    } finally {
+        channel?.disconnect()
+    }
+}
+
+public inline fun JSch.useSession(
+    host: String,
+    user: String,
+    port: Int = 22,
+    block: Session.() -> Unit,
+) {
+    var session: Session? = null
+    try {
+        session = getSession(user, host, port)
+        val config = Properties()
+        config["StrictHostKeyChecking"] = "no"
+        session.setConfig(config)
+        session.connect()
+        session.block()
+    } finally {
+        session?.disconnect()
+    }
+}
+
+public fun JSch(configuration: JSch.() -> Unit): JSch = JSch().apply(configuration)

snark-html/build.gradle.kts

@@ -0,0 +1,30 @@
+plugins {
+    id("ru.mipt.npm.gradle.jvm")
+    `maven-publish`
+}
+
+val dataforgeVersion: String by rootProject.extra
+val ktorVersion = ru.mipt.npm.gradle.KScienceVersions.ktorVersion
+
+dependencies {
+    api(projects.snarkCore)
+
+    api("org.jetbrains.kotlinx:kotlinx-html:0.7.5")
+    api("org.jetbrains.kotlin-wrappers:kotlin-css")
+
+    api("io.ktor:ktor-utils:$ktorVersion")
+
+    api("space.kscience:dataforge-io-yaml:$dataforgeVersion")
+    api("org.jetbrains:markdown:0.3.1")
+}
+
+readme {
+    maturity = ru.mipt.npm.gradle.Maturity.EXPERIMENTAL
+    feature("data") { "Data-based processing. Instead of traditional layout-based" }
+    feature("layouts") { "Use custom layouts to represent a data tree" }
+    feature("parsers") { "Add custom file formats and parsers using DataForge dependency injection" }
+    feature("preprocessor") { "Preprocessing text files using templates" }
+    feature("metadata") { "Trademark DataForge metadata layering and transformations" }
+    feature("dynamic") { "Generating dynamic site using KTor server" }
+    feature("static") { "Generating static site" }
+}

snark-html/src/main/kotlin/space/kscience/snark/html/HtmlData.kt

@@ -0,0 +1,38 @@
+package space.kscience.snark.html
+
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.runBlocking
+import kotlinx.html.FlowContent
+import kotlinx.html.TagConsumer
+import space.kscience.dataforge.data.Data
+import space.kscience.dataforge.data.await
+import space.kscience.dataforge.meta.get
+import space.kscience.dataforge.meta.int
+import space.kscience.dataforge.meta.string
+import space.kscience.snark.SnarkContext
+
+
+//TODO replace by VisionForge type
+//typealias HtmlFragment = context(PageBuilder, TagConsumer<*>) () -> Unit
+
+public fun interface HtmlFragment {
+    public fun TagConsumer<*>.renderFragment(page: Page)
+    //TODO move pageBuilder to a context receiver after KT-52967 is fixed
+}
+
+public typealias HtmlData = Data<HtmlFragment>
+
+//fun HtmlData(meta: Meta, content: context(PageBuilder) TagConsumer<*>.() -> Unit): HtmlData =
+//    Data(HtmlFragment(content), meta)
+
+
+context(Page) public fun FlowContent.htmlData(data: HtmlData): Unit = runBlocking(Dispatchers.IO) {
+    with(data.await()) { consumer.renderFragment(page) }
+}
+
+context(SnarkContext) public val Data<*>.id: String get() = meta["id"]?.string ?: "block[${hashCode()}]"
+context(SnarkContext) public val Data<*>.language: String? get() = meta["language"].string?.lowercase()
+
+context(SnarkContext) public val Data<*>.order: Int? get() = meta["order"]?.int
+
+context(SnarkContext) public val Data<*>.published: Boolean get() = meta["published"].string != "false"

snark-html/src/main/kotlin/space/kscience/snark/html/Page.kt

@@ -0,0 +1,68 @@
+package space.kscience.snark.html
+
+import space.kscience.dataforge.context.Context
+import space.kscience.dataforge.context.ContextAware
+import space.kscience.dataforge.data.*
+import space.kscience.dataforge.meta.Meta
+import space.kscience.dataforge.meta.get
+import space.kscience.dataforge.meta.string
+import space.kscience.dataforge.names.*
+import space.kscience.snark.SnarkContext
+
+context(SnarkContext) public fun Name.toWebPath(): String = tokens.joinToString(separator = "/") {
+    if (it.hasIndex()) {
+        "${it.body}[${it.index}]"
+    } else {
+        it.body
+    }
+}
+
+public interface Page : ContextAware, SnarkContext {
+
+    public val snark: SnarkPlugin
+
+    override val context: Context get() = snark.context
+
+    public val data: DataTree<*>
+
+    public val pageMeta: Meta
+
+    public fun resolveRef(ref: String): String
+
+    public fun resolvePageRef(pageName: Name): String
+}
+
+context(Page) public val page: Page get() = this@Page
+
+public fun Page.resolvePageRef(pageName: String): String = resolvePageRef(pageName.parseAsName())
+
+public val Page.homeRef: String get() = resolvePageRef(SiteBuilder.INDEX_PAGE_TOKEN.asName())
+
+/**
+ * Resolve a Html builder by its full name
+ */
+context(SnarkContext) public fun DataTree<*>.resolveHtml(name: Name): HtmlData? {
+    val resolved = (getByType<HtmlFragment>(name) ?: getByType<HtmlFragment>(name + SiteBuilder.INDEX_PAGE_TOKEN))
+
+    return resolved?.takeIf {
+        it.published //TODO add language confirmation
+    }
+}
+
+/**
+ * Find all Html blocks using given name/meta filter
+ */
+context(SnarkContext) public fun DataTree<*>.resolveAllHtml(predicate: (name: Name, meta: Meta) -> Boolean): Map<Name, HtmlData> =
+    filterByType<HtmlFragment> { name, meta ->
+        predicate(name, meta)
+                && meta["published"].string != "false"
+        //TODO add language confirmation
+    }.asSequence().associate { it.name to it.data }
+
+
+context(SnarkContext) public fun DataTree<*>.findByContentType(
+    contentType: String,
+    baseName: Name = Name.EMPTY,
+): Map<Name, Data<HtmlFragment>> = resolveAllHtml { name, meta ->
+    name.startsWith(baseName) && meta["content_type"].string == contentType
+}

snark-html/src/main/kotlin/space/kscience/snark/html/SiteBuilder.kt

@@ -0,0 +1,100 @@
+package space.kscience.snark.html
+
+import kotlinx.html.HTML
+import space.kscience.dataforge.context.Context
+import space.kscience.dataforge.context.ContextAware
+import space.kscience.dataforge.data.DataTree
+import space.kscience.dataforge.data.DataTreeItem
+import space.kscience.dataforge.meta.Meta
+import space.kscience.dataforge.names.Name
+import space.kscience.dataforge.names.NameToken
+import space.kscience.dataforge.names.parseAsName
+import space.kscience.snark.SnarkContext
+import java.nio.file.Path
+import kotlin.reflect.KType
+import kotlin.reflect.typeOf
+
+
+/**
+ * An abstraction, which is used to render sites to the different rendering engines
+ */
+public interface SiteBuilder : ContextAware, SnarkContext {
+
+    public val data: DataTree<*>
+
+    public val snark: SnarkPlugin
+
+    override val context: Context get() = snark.context
+
+    public val siteMeta: Meta
+
+    public fun assetFile(remotePath: String, file: Path)
+
+    public fun assetDirectory(remotePath: String, directory: Path)
+
+    public fun assetResourceFile(remotePath: String, resourcesPath: String)
+
+    public fun assetResourceDirectory(resourcesPath: String)
+
+    public fun page(route: Name = Name.EMPTY, content: context(Page, HTML) () -> Unit)
+
+    /**
+     * Create a route with optional data tree override. For example one could use a subtree of the initial tree.
+     * By default, the same data tree is used for route
+     */
+    public fun route(
+        routeName: Name,
+        dataOverride: DataTree<*>? = null,
+        metaOverride: Meta? = null,
+        setAsRoot: Boolean = false,
+    ): SiteBuilder
+
+    public companion object {
+        public val INDEX_PAGE_TOKEN: NameToken = NameToken("index")
+        public val UP_PAGE_TOKEN: NameToken = NameToken("..")
+    }
+}
+
+context(SiteBuilder) public val siteBuilder: SiteBuilder get() = this@SiteBuilder
+
+public inline fun SiteBuilder.route(
+    route: Name,
+    dataOverride: DataTree<*>? = null,
+    metaOverride: Meta? = null,
+    setAsRoot: Boolean = false,
+    block: SiteBuilder.() -> Unit,
+) {
+    route(route, dataOverride, metaOverride, setAsRoot).apply(block)
+}
+
+public inline fun SiteBuilder.route(
+    route: String,
+    dataOverride: DataTree<*>? = null,
+    metaOverride: Meta? = null,
+    setAsRoot: Boolean = false,
+    block: SiteBuilder.() -> Unit,
+) {
+    route(route.parseAsName(), dataOverride, metaOverride, setAsRoot).apply(block)
+}
+
+
+///**
+// * Create a stand-alone site at a given node
+// */
+//public fun SiteBuilder.site(route: Name, dataRoot: DataTree<*>, block: SiteBuilder.() -> Unit) {
+//    val mountedData = data.copy(
+//        data = dataRoot,
+//        baseUrlPath = data.resolveRef(route.tokens.joinToString(separator = "/")),
+//        meta = Laminate(dataRoot.meta, data.meta) //layering dataRoot meta over existing data
+//    )
+//    route(route) {
+//        withData(mountedData).block()
+//    }
+//}
+
+//TODO move to DF
+public fun DataTree.Companion.empty(meta: Meta = Meta.EMPTY): DataTree<Any> = object : DataTree<Any> {
+    override val items: Map<NameToken, DataTreeItem<Any>> get() = emptyMap()
+    override val dataType: KType get() = typeOf<Any>()
+    override val meta: Meta get() = meta
+}

snark-html/src/main/kotlin/space/kscience/snark/html/SiteLayout.kt

@@ -0,0 +1,150 @@
+package space.kscience.snark.html
+
+import kotlinx.html.body
+import kotlinx.html.head
+import kotlinx.html.title
+import space.kscience.dataforge.data.Data
+import space.kscience.dataforge.data.DataTreeItem
+import space.kscience.dataforge.data.getItem
+import space.kscience.dataforge.meta.Meta
+import space.kscience.dataforge.meta.get
+import space.kscience.dataforge.meta.getIndexed
+import space.kscience.dataforge.meta.string
+import space.kscience.dataforge.misc.Type
+import space.kscience.dataforge.names.Name
+import space.kscience.dataforge.names.NameToken
+import space.kscience.dataforge.names.asName
+import space.kscience.dataforge.names.parseAsName
+import space.kscience.snark.html.SiteLayout.Companion.ASSETS_KEY
+import space.kscience.snark.html.SiteLayout.Companion.INDEX_PAGE_TOKEN
+import space.kscience.snark.html.SiteLayout.Companion.LAYOUT_KEY
+import java.nio.file.Path
+import kotlin.reflect.typeOf
+
+internal fun SiteBuilder.assetsFrom(rootMeta: Meta) {
+    rootMeta.getIndexed("resource".asName()).forEach { (_, meta) ->
+
+        val path by meta.string()
+        val remotePath by meta.string()
+
+        path?.let { resourcePath ->
+            //If remote path provided, use a single resource
+            remotePath?.let {
+                assetResourceFile(it, resourcePath)
+                return@forEach
+            }
+
+            //otherwise use package resources
+            assetResourceDirectory(resourcePath)
+        }
+    }
+
+    rootMeta.getIndexed("file".asName()).forEach { (_, meta) ->
+        val remotePath by meta.string { error("File remote path is not provided") }
+        val path by meta.string { error("File path is not provided") }
+        assetFile(remotePath, Path.of(path))
+    }
+
+    rootMeta.getIndexed("directory".asName()).forEach { (_, meta) ->
+        val path by meta.string { error("Directory path is not provided") }
+        assetDirectory("", Path.of(path))
+    }
+}
+
+public typealias DataRenderer = SiteBuilder.(name: Name, data: Data<Any>) -> Unit
+
+/**
+ * Recursively renders the data items in [data]. If [LAYOUT_KEY] is defined in an item, use it to load
+ * layout from the context, otherwise render children nodes as name segments and individual data items using [dataRenderer].
+ */
+public fun SiteBuilder.pages(
+    data: DataTreeItem<*>,
+    dataRenderer: DataRenderer = SiteLayout.defaultDataRenderer,
+) {
+    val layoutMeta = data.meta[LAYOUT_KEY]
+    if (layoutMeta != null) {
+        //use layout if it is defined
+        snark.siteLayout(layoutMeta).render(data)
+    } else {
+        when (data) {
+            is DataTreeItem.Node -> {
+                data.tree.items.forEach { (token, item) ->
+                    //Don't apply index token
+                    if (token == INDEX_PAGE_TOKEN) {
+                        pages(item, dataRenderer)
+                    } else if (item is DataTreeItem.Leaf) {
+                        dataRenderer(this, token.asName(), item.data)
+                    } else {
+                        route(token.asName()) {
+                            pages(item, dataRenderer)
+                        }
+                    }
+                }
+            }
+            is DataTreeItem.Leaf -> {
+                dataRenderer.invoke(this, Name.EMPTY, data.data)
+            }
+        }
+        data.meta[ASSETS_KEY]?.let {
+            assetsFrom(it)
+        }
+    }
+    //TODO watch for changes
+}
+
+/**
+ * Render all pages in a node with given name
+ */
+public fun SiteBuilder.pages(
+    dataPath: Name,
+    remotePath: Name = dataPath,
+    dataRenderer: DataRenderer = SiteLayout.defaultDataRenderer,
+) {
+    val item = data.getItem(dataPath) ?: error("No data found by name $dataPath")
+    route(remotePath) {
+        pages(item, dataRenderer)
+    }
+}
+
+public fun SiteBuilder.pages(
+    dataPath: String,
+    remotePath: Name = dataPath.parseAsName(),
+    dataRenderer: DataRenderer = SiteLayout.defaultDataRenderer,
+) {
+    pages(dataPath.parseAsName(), remotePath, dataRenderer = dataRenderer)
+}
+
+
+@Type(SiteLayout.TYPE)
+public fun interface SiteLayout {
+
+    context(SiteBuilder) public fun render(item: DataTreeItem<*>)
+
+    public companion object {
+        public const val TYPE: String = "snark.layout"
+        public const val LAYOUT_KEY: String = "layout"
+        public const val ASSETS_KEY: String = "assets"
+        public val INDEX_PAGE_TOKEN: NameToken = NameToken("index")
+
+        public val defaultDataRenderer: SiteBuilder.(name: Name, data: Data<*>) -> Unit = { name: Name, data: Data<*> ->
+            if (data.type == typeOf<HtmlData>()) {
+                page(name) {
+                    head {
+                        title = data.meta["title"].string ?: "Untitled page"
+                    }
+                    body {
+                        @Suppress("UNCHECKED_CAST")
+                        htmlData(data as HtmlData)
+                    }
+                }
+            }
+        }
+    }
+}
+
+
+public object DefaultSiteLayout : SiteLayout {
+    context(SiteBuilder) override fun render(item: DataTreeItem<*>) {
+        pages(item)
+    }
+}

snark-html/src/main/kotlin/space/kscience/snark/html/SnarkPlugin.kt

@@ -0,0 +1,117 @@
+package space.kscience.snark.html
+
+import io.ktor.utils.io.core.readBytes
+import space.kscience.dataforge.context.*
+import space.kscience.dataforge.data.DataTree
+import space.kscience.dataforge.io.*
+import space.kscience.dataforge.io.yaml.YamlMetaFormat
+import space.kscience.dataforge.io.yaml.YamlPlugin
+import space.kscience.dataforge.meta.Meta
+import space.kscience.dataforge.meta.get
+import space.kscience.dataforge.meta.string
+import space.kscience.dataforge.misc.DFExperimental
+import space.kscience.dataforge.names.Name
+import space.kscience.dataforge.names.asName
+import space.kscience.dataforge.names.parseAsName
+import space.kscience.dataforge.workspace.FileData
+import space.kscience.dataforge.workspace.readDataDirectory
+import space.kscience.snark.SnarkParser
+import java.nio.file.Path
+import kotlin.io.path.extension
+import kotlin.reflect.KClass
+import kotlin.reflect.KType
+import kotlin.reflect.typeOf
+
+@PublishedApi
+internal class SnarkParserWrapper<R : Any>(
+    val reader: IOReader<R>,
+    override val type: KType,
+    override val fileExtensions: Set<String>,
+) : SnarkParser<R> {
+    override fun parse(context: Context, meta: Meta, bytes: ByteArray): R = bytes.asBinary().readWith(reader)
+}
+
+/**
+ * Create a generic parser from reader
+ */
+@Suppress("FunctionName")
+public inline fun <reified R : Any> SnarkParser(
+    reader: IOReader<R>,
+    vararg fileExtensions: String,
+): SnarkParser<R> = SnarkParserWrapper(reader, typeOf<R>(), fileExtensions.toSet())
+
+public class SnarkPlugin : AbstractPlugin() {
+    private val yaml by require(YamlPlugin)
+    public val io: IOPlugin get() = yaml.io
+
+    override val tag: PluginTag get() = Companion.tag
+
+    internal val parsers: Map<Name, SnarkParser<Any>> by lazy {
+        context.gather(SnarkParser.TYPE, true)
+    }
+
+    private val siteLayouts: Map<Name, SiteLayout> by lazy {
+        context.gather(SiteLayout.TYPE, true)
+    }
+
+    private val textTransformations: Map<Name, TextTransformation> by lazy {
+        context.gather(TextTransformation.TYPE, true)
+    }
+
+    internal fun siteLayout(layoutMeta: Meta): SiteLayout {
+        val layoutName = layoutMeta.string
+            ?: layoutMeta["name"].string ?: error("Layout name not defined in $layoutMeta")
+        return siteLayouts[layoutName.parseAsName()] ?: error("Layout with name $layoutName not found in $this")
+    }
+
+    internal fun textTransformation(transformationMeta: Meta): TextTransformation {
+        val transformationName = transformationMeta.string
+            ?: transformationMeta["name"].string ?: error("Transformation name not defined in $transformationMeta")
+        return textTransformations[transformationName.parseAsName()]
+            ?: error("Text transformation with name $transformationName not found in $this")
+    }
+
+    override fun content(target: String): Map<Name, Any> = when (target) {
+        SnarkParser.TYPE -> mapOf(
+            "html".asName() to SnarkHtmlParser,
+            "markdown".asName() to SnarkMarkdownParser,
+            "json".asName() to SnarkParser(JsonMetaFormat, "json"),
+            "yaml".asName() to SnarkParser(YamlMetaFormat, "yaml", "yml"),
+            "png".asName() to SnarkParser(ImageIOReader, "png"),
+            "jpg".asName() to SnarkParser(ImageIOReader, "jpg", "jpeg"),
+            "gif".asName() to SnarkParser(ImageIOReader, "gif"),
+        )
+        TextTransformation.TYPE -> mapOf(
+            "basic".asName() to BasicTextTransformation
+        )
+        else -> super.content(target)
+    }
+
+    public companion object : PluginFactory<SnarkPlugin> {
+        override val tag: PluginTag = PluginTag("snark")
+        override val type: KClass<out SnarkPlugin> = SnarkPlugin::class
+
+        override fun build(context: Context, meta: Meta): SnarkPlugin = SnarkPlugin()
+
+        private val byteArrayIOReader = IOReader {
+            readBytes()
+        }
+
+        internal val byteArraySnarkParser = SnarkParser(byteArrayIOReader)
+    }
+}
+
+@OptIn(DFExperimental::class)
+public fun SnarkPlugin.readDirectory(path: Path): DataTree<Any> = io.readDataDirectory(path) { dataPath, meta ->
+    val fileExtension = meta[FileData.META_FILE_EXTENSION_KEY].string ?: dataPath.extension
+    val parser: SnarkParser<Any> = parsers.values.filter { parser ->
+        fileExtension in parser.fileExtensions
+    }.maxByOrNull {
+        it.priority
+    } ?: run {
+        logger.warn { "The parser is not found for file $dataPath with meta $meta" }
+        SnarkPlugin.byteArraySnarkParser
+    }
+
+    parser.reader(context, meta)
+}

snark-html/src/main/kotlin/space/kscience/snark/html/SnarkTextParser.kt

@@ -0,0 +1,70 @@
+package space.kscience.snark.html
+
+import io.ktor.util.asStream
+import io.ktor.utils.io.core.Input
+import kotlinx.html.div
+import kotlinx.html.unsafe
+import org.intellij.markdown.flavours.commonmark.CommonMarkFlavourDescriptor
+import org.intellij.markdown.html.HtmlGenerator
+import org.intellij.markdown.parser.MarkdownParser
+import space.kscience.dataforge.context.Context
+import space.kscience.dataforge.io.IOReader
+import space.kscience.dataforge.meta.Meta
+import space.kscience.dataforge.meta.get
+import space.kscience.snark.SnarkParser
+import java.awt.image.BufferedImage
+import javax.imageio.ImageIO
+import kotlin.reflect.KType
+import kotlin.reflect.typeOf
+
+public abstract class SnarkTextParser<R> : SnarkParser<R> {
+    public abstract fun parseText(text: String, meta: Meta): R
+
+    override fun parse(context: Context, meta: Meta, bytes: ByteArray): R =
+        parseText(bytes.decodeToString(), meta)
+
+    public fun transformText(text: String, meta: Meta, page: Page): String =
+        meta[TextTransformation.TEXT_TRANSFORMATION_KEY]?.let {
+            with(page) { page.snark.textTransformation(it).transform(text) }
+        } ?: text
+}
+
+
+internal object SnarkHtmlParser : SnarkTextParser<HtmlFragment>() {
+    override val fileExtensions: Set<String> = setOf("html")
+    override val type: KType = typeOf<HtmlFragment>()
+
+    override fun parseText(text: String, meta: Meta): HtmlFragment = HtmlFragment { page ->
+        div {
+            unsafe { +transformText(text, meta, page) }
+        }
+    }
+}
+
+internal object SnarkMarkdownParser : SnarkTextParser<HtmlFragment>() {
+    override val fileExtensions: Set<String> = setOf("markdown", "mdown", "mkdn", "mkd", "md")
+    override val type: KType = typeOf<HtmlFragment>()
+
+    private val markdownFlavor = CommonMarkFlavourDescriptor()
+    private val markdownParser = MarkdownParser(markdownFlavor)
+
+    override fun parseText(text: String, meta: Meta): HtmlFragment {
+        val parsedTree = markdownParser.buildMarkdownTreeFromString(text)
+        val htmlString = HtmlGenerator(text, parsedTree, markdownFlavor).generateHtml()
+
+        return HtmlFragment { page ->
+
+            div {
+                unsafe {
+                    +SnarkHtmlParser.transformText(htmlString, meta, page)
+                }
+            }
+        }
+    }
+}
+
+internal object ImageIOReader : IOReader<BufferedImage> {
+    override val type: KType get() = typeOf<BufferedImage>()
+
+    override fun readObject(input: Input): BufferedImage = ImageIO.read(input.asStream())
+}

snark-html/src/main/kotlin/space/kscience/snark/html/StaticSiteBuilder.kt

@@ -0,0 +1,125 @@
+package space.kscience.snark.html
+
+import kotlinx.html.HTML
+import kotlinx.html.html
+import kotlinx.html.stream.createHTML
+import space.kscience.dataforge.data.DataTree
+import space.kscience.dataforge.meta.Meta
+import space.kscience.dataforge.meta.withDefault
+import space.kscience.dataforge.names.Name
+import space.kscience.dataforge.names.isEmpty
+import java.nio.file.Files
+import java.nio.file.Path
+import kotlin.contracts.InvocationKind
+import kotlin.contracts.contract
+import kotlin.io.path.*
+
+
+internal class StaticSiteBuilder(
+    override val snark: SnarkPlugin,
+    override val data: DataTree<*>,
+    override val siteMeta: Meta,
+    private val baseUrl: String,
+    private val path: Path,
+) : SiteBuilder {
+    private fun Path.copyRecursively(target: Path) {
+        Files.walk(this).forEach { source: Path ->
+            val destination: Path = target.resolve(source.relativeTo(this))
+            if (!destination.isDirectory()) {
+                //avoid re-creating directories
+                source.copyTo(destination, true)
+            }
+        }
+    }
+
+    override fun assetFile(remotePath: String, file: Path) {
+        val targetPath = path.resolve(remotePath)
+        targetPath.parent.createDirectories()
+        file.copyTo(targetPath, true)
+    }
+
+    override fun assetDirectory(remotePath: String, directory: Path) {
+        val targetPath = path.resolve(remotePath)
+        targetPath.parent.createDirectories()
+        directory.copyRecursively(targetPath)
+    }
+
+    override fun assetResourceFile(remotePath: String, resourcesPath: String) {
+        val targetPath = path.resolve(remotePath)
+        targetPath.parent.createDirectories()
+        javaClass.getResource(resourcesPath)?.let { Path.of(it.toURI()) }?.copyTo(targetPath, true)
+    }
+
+    override fun assetResourceDirectory(resourcesPath: String) {
+        path.parent.createDirectories()
+        javaClass.getResource(resourcesPath)?.let { Path.of(it.toURI()) }?.copyRecursively(path)
+    }
+
+    private fun resolveRef(baseUrl: String, ref: String) = if (baseUrl.isEmpty()) {
+        ref
+    } else if (ref.isEmpty()) {
+        baseUrl
+    } else {
+        "${baseUrl.removeSuffix("/")}/$ref"
+    }
+
+    inner class StaticPage : Page {
+        override val data: DataTree<*> get() = this@StaticSiteBuilder.data
+        override val pageMeta: Meta get() = this@StaticSiteBuilder.siteMeta
+        override val snark: SnarkPlugin get() = this@StaticSiteBuilder.snark
+
+
+        override fun resolveRef(ref: String): String = resolveRef(baseUrl, ref)
+
+        override fun resolvePageRef(pageName: Name): String = resolveRef(
+            pageName.toWebPath() + ".html"
+        )
+    }
+
+
+    override fun page(route: Name, content: context(Page, HTML) () -> Unit) {
+        val htmlBuilder = createHTML()
+
+        htmlBuilder.html {
+            content(StaticPage(), this)
+        }
+
+        val newPath = if (route.isEmpty()) {
+            path.resolve("index.html")
+        } else {
+            path.resolve(route.toWebPath() + ".html")
+        }
+
+        newPath.parent.createDirectories()
+        newPath.writeText(htmlBuilder.finalize())
+    }
+
+    override fun route(
+        routeName: Name,
+        dataOverride: DataTree<*>?,
+        metaOverride: Meta?,
+        setAsRoot: Boolean,
+    ): SiteBuilder = StaticSiteBuilder(
+        snark = snark,
+        data = dataOverride ?: data,
+        siteMeta = metaOverride?.withDefault(siteMeta) ?: siteMeta,
+        baseUrl = if (setAsRoot) {
+            resolveRef(baseUrl, routeName.toWebPath())
+        } else {
+            baseUrl
+        },
+        path = path.resolve(routeName.toWebPath())
+    )
+}
+
+public fun SnarkPlugin.renderStatic(
+    outputPath: Path,
+    data: DataTree<*> = DataTree.empty(),
+    siteUrl: String = outputPath.absolutePathString().replace("\\", "/"),
+    block: SiteBuilder.() -> Unit,
+) {
+    contract {
+        callsInPlace(block, InvocationKind.EXACTLY_ONCE)
+    }
+    StaticSiteBuilder(this, data, meta, siteUrl, outputPath).block()
+}

snark-html/src/main/kotlin/space/kscience/snark/html/TextTransformation.kt

@@ -0,0 +1,38 @@
+package space.kscience.snark.html
+
+import space.kscience.dataforge.misc.Type
+import space.kscience.dataforge.names.NameToken
+
+@Type(TextTransformation.TYPE)
+public fun interface TextTransformation {
+    context(Page) public fun transform(text: String): String
+
+    public companion object {
+        public const val TYPE: String = "snark.textTransformation"
+        public val TEXT_TRANSFORMATION_KEY: NameToken = NameToken("transformation")
+    }
+}
+
+public object BasicTextTransformation : TextTransformation {
+
+    private val regex = "\\\$\\{(\\w*)(?>\\(\"(.*)\"\\))?\\}".toRegex()
+
+    context(Page) override fun transform(text: String): String {
+        return text.replace(regex) { match ->
+            when (match.groups[1]!!.value) {
+                "homeRef" -> homeRef
+                "resolveRef" -> {
+                    val refString = match.groups[2]?.value ?: error("resolveRef requires a string (quoted) argument")
+                    resolveRef(refString)
+                }
+                "resolvePageRef" -> {
+                    val refString = match.groups[2]?.value ?: error("resolvePageRef requires a string (quoted) argument")
+                    resolvePageRef(refString)
+                }
+                else -> match.value
+            }
+        }
+    }
+}
+
+

snark-html/src/main/resources/application.conf

@@ -0,0 +1,12 @@
+ktor {
+    application {
+        modules = [ ru.mipt.spc.ApplicationKt.spcModule ]
+    }
+
+    deployment {
+        port = 7080
+        watch = ["classes", "data/"]
+    }
+
+    development = true
+}

snark-html/src/main/resources/logback.xml

@@ -0,0 +1,29 @@
+<configuration>
+    <timestamp key="bySecond" datePattern="yyyyMMdd'T'HHmmss"/>
+
+    <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
+        <encoder>
+            <pattern>%d{YYYY-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n</pattern>
+        </encoder>
+    </appender>
+
+    <root level="trace">
+        <appender-ref ref="STDOUT"/>
+    </root>
+
+    <appender name="FILE" class="ch.qos.logback.core.FileAppender">
+        <!-- use the previously created timestamp to create a uniquely
+             named log file -->
+        <file>logs/${bySecond}.txt</file>
+        <encoder>
+            <pattern>%d{YYYY-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n</pattern>
+        </encoder>
+    </appender>
+
+    <root level="DEBUG">
+        <appender-ref ref="FILE" />
+    </root>
+
+    <logger name="org.eclipse.jetty" level="INFO"/>
+    <logger name="io.netty" level="INFO"/>
+</configuration>

snark-ktor/build.gradle.kts

@@ -0,0 +1,16 @@
+plugins {
+    id("ru.mipt.npm.gradle.jvm")
+}
+
+val dataforgeVersion: String by rootProject.extra
+val ktorVersion = ru.mipt.npm.gradle.KScienceVersions.ktorVersion
+
+dependencies {
+    api(projects.snarkHtml)
+
+    api("io.ktor:ktor-server-core:$ktorVersion")
+    api("io.ktor:ktor-server-html-builder:$ktorVersion")
+    api("io.ktor:ktor-server-host-common:$ktorVersion")
+
+    testApi("io.ktor:ktor-server-tests:$ktorVersion")
+}

snark-ktor/src/main/kotlin/space/kscience/snark/ktor/KtorSiteBuilder.kt

@@ -0,0 +1,136 @@
+package space.kscience.snark.ktor
+
+import io.ktor.http.URLBuilder
+import io.ktor.http.URLProtocol
+import io.ktor.server.application.Application
+import io.ktor.server.application.call
+import io.ktor.server.html.respondHtml
+import io.ktor.server.http.content.*
+import io.ktor.server.plugins.origin
+import io.ktor.server.request.host
+import io.ktor.server.request.port
+import io.ktor.server.routing.Route
+import io.ktor.server.routing.createRouteFromPath
+import io.ktor.server.routing.get
+import io.ktor.server.routing.routing
+import kotlinx.html.HTML
+import space.kscience.dataforge.data.DataTree
+import space.kscience.dataforge.meta.Meta
+import space.kscience.dataforge.meta.withDefault
+import space.kscience.dataforge.names.Name
+import space.kscience.dataforge.names.cutLast
+import space.kscience.dataforge.names.endsWith
+import space.kscience.snark.html.*
+import java.nio.file.Path
+import kotlin.contracts.InvocationKind
+import kotlin.contracts.contract
+
+@PublishedApi
+internal class KtorSiteBuilder(
+    override val snark: SnarkPlugin,
+    override val data: DataTree<*>,
+    override val siteMeta: Meta,
+    private val baseUrl: String,
+    private val ktorRoute: Route,
+) : SiteBuilder {
+
+    override fun assetFile(remotePath: String, file: Path) {
+        ktorRoute.file(remotePath, file.toFile())
+    }
+
+    override fun assetDirectory(remotePath: String, directory: Path) {
+        ktorRoute.static(remotePath) {
+            files(directory.toFile())
+        }
+    }
+
+    private fun resolveRef(baseUrl: String, ref: String) = if (baseUrl.isEmpty()) {
+        ref
+    } else if (ref.isEmpty()) {
+        baseUrl
+    } else {
+        "${baseUrl.removeSuffix("/")}/$ref"
+    }
+
+
+    inner class KtorPage(
+        val pageBaseUrl: String,
+        override val pageMeta: Meta = this@KtorSiteBuilder.siteMeta,
+    ) : Page {
+        override val snark: SnarkPlugin get() = this@KtorSiteBuilder.snark
+        override val data: DataTree<*> get() = this@KtorSiteBuilder.data
+
+        override fun resolveRef(ref: String): String = resolveRef(pageBaseUrl, ref)
+
+        override fun resolvePageRef(pageName: Name): String = if (pageName.endsWith(SiteBuilder.INDEX_PAGE_TOKEN)) {
+            resolveRef(pageName.cutLast().toWebPath())
+        } else {
+            resolveRef(pageName.toWebPath())
+        }
+    }
+
+    override fun page(route: Name, content: context(Page, HTML)() -> Unit) {
+        ktorRoute.get(route.toWebPath()) {
+            call.respondHtml {
+                val request = call.request
+                //substitute host for url for backwards calls
+                val url = URLBuilder(baseUrl).apply {
+                    protocol = URLProtocol.createOrDefault(request.origin.scheme)
+                    host = request.host()
+                    port = request.port()
+                }
+                val pageBuilder = KtorPage(url.buildString())
+                content(pageBuilder, this)
+            }
+        }
+    }
+
+    override fun route(
+        routeName: Name,
+        dataOverride: DataTree<*>?,
+        metaOverride: Meta?,
+        setAsRoot: Boolean,
+    ): SiteBuilder = KtorSiteBuilder(
+        snark = snark,
+        data = dataOverride ?: data,
+        siteMeta = metaOverride?.withDefault(siteMeta) ?: siteMeta,
+        baseUrl = if (setAsRoot) {
+            resolveRef(baseUrl, routeName.toWebPath())
+        } else {
+            baseUrl
+        },
+        ktorRoute = ktorRoute.createRouteFromPath(routeName.toWebPath())
+    )
+
+
+    override fun assetResourceFile(remotePath: String, resourcesPath: String) {
+        ktorRoute.resource(resourcesPath, resourcesPath)
+    }
+
+    override fun assetResourceDirectory(resourcesPath: String) {
+        ktorRoute.resources(resourcesPath)
+    }
+}
+
+public inline fun Route.snarkSite(
+    snark: SnarkPlugin,
+    data: DataTree<*>,
+    meta: Meta = data.meta,
+    block: SiteBuilder.() -> Unit,
+) {
+    contract {
+        callsInPlace(block, InvocationKind.EXACTLY_ONCE)
+    }
+    block(KtorSiteBuilder(snark, data, meta, "", this@snarkSite))
+}
+
+public fun Application.snarkSite(
+    snark: SnarkPlugin,
+    data: DataTree<*> = DataTree.empty(),
+    meta: Meta = data.meta,
+    block: SiteBuilder.() -> Unit,
+) {
+    routing {
+        snarkSite(snark, data, meta, block)
+    }
+}