[WIP] refactor in progress

gradle.properties

@@ -1,3 +1,3 @@
 kotlin.code.style=official
 
-toolsVersion=0.15.1-kotlin-1.9.21
+toolsVersion=0.15.2-kotlin-1.9.21

snark-core/src/commonMain/kotlin/space/kscience/snark/Snark.kt

@@ -34,7 +34,7 @@ public class Snark : WorkspacePlugin() {
         context.gather(TextProcessor.DF_TYPE, true)
     }
 
-    public fun textProcessor(transformationMeta: Meta): TextProcessor {
+    public fun preprocessor(transformationMeta: Meta): TextProcessor {
         val transformationName = transformationMeta.string
             ?: transformationMeta["name"].string ?: error("Transformation name not defined in $transformationMeta")
         return textProcessors[transformationName.parseAsName()]

snark-core/src/jvmMain/kotlin/space/kscience/snark/snarkWorkspace.kt

@@ -27,15 +27,13 @@ import kotlin.io.path.toPath
 private fun IOPlugin.readResources(
     vararg resources: String,
     classLoader: ClassLoader = Thread.currentThread().contextClassLoader,
-): DataTree<Binary> {
+): DataTree<Binary> = DataTree {
-//    require(resource.isNotBlank()) {"Can't mount root resource tree as data root"}
+    //    require(resource.isNotBlank()) {"Can't mount root resource tree as data root"}
-    return DataTree {
+    resources.forEach { resource ->
-        resources.forEach { resource ->
+        val path = classLoader.getResource(resource)?.toURI()?.toPath() ?: error(
-            val path = classLoader.getResource(resource)?.toURI()?.toPath() ?: error(
+            "Resource with name $resource is not resolved"
-                "Resource with name $resource is not resolved"
+        )
-            )
+        node(resource, readRawDirectory(path))
-            node(resource, readRawDirectory(path))
-        }
     }
 }
 

snark-html/src/jvmMain/kotlin/space/kscience/snark/html/HtmlPage.kt

@@ -10,18 +10,23 @@ import space.kscience.dataforge.meta.Meta
 import space.kscience.dataforge.names.Name
 
 public fun interface HtmlPage {
-    public suspend fun HTML.renderPage(page: PageContext, data: DataSet<*>)
 
-    public companion object{
+    context(PageContextWithData)
-        public suspend fun createHtmlString(pageContext: PageContext, page: HtmlPage, data: DataSet<*>): String{
+    public fun HTML.renderPage()
-            return createHTML().run {
+
-                HTML(kotlinx.html.emptyMap, this, null).visitTagAndFinalize(this) {
+    public companion object {
+        public fun createHtmlString(
+            pageContext: PageContext,
+            dataSet: DataSet<*>,
+            page: HtmlPage,
+        ): String = createHTML().run {
+            HTML(kotlinx.html.emptyMap, this, null).visitTagAndFinalize(this) {
+                with(PageContextWithData(pageContext, dataSet)) {
                     with(page) {
-                        renderPage(pageContext, data)
+                        renderPage()
                     }
                 }
             }
-
         }
     }
 }
@@ -29,14 +34,16 @@ public fun interface HtmlPage {
 
 // data builders
 
-public fun DataSetBuilder<Any>.page(name: Name, pageMeta: Meta = Meta.EMPTY, block: HTML.(pageContext: PageContext, pageData: DataSet<Any>) -> Unit) {
+public fun DataSetBuilder<Any>.page(
+    name: Name,
+    pageMeta: Meta = Meta.EMPTY,
+    block: context(PageContextWithData) HTML.() -> Unit,
+) {
     val page = HtmlPage(block)
     static<HtmlPage>(name, page, pageMeta)
 }
 
 
-
-
 //                if (data.type == typeOf<HtmlData>()) {
 //                    val languageMeta: Meta = Language.forName(name)
 //

snark-html/src/jvmMain/kotlin/space/kscience/snark/html/HtmlSite.kt

@@ -11,15 +11,16 @@ import space.kscience.dataforge.names.asName
 import space.kscience.dataforge.names.parseAsName
 
 public fun interface HtmlSite {
-    public suspend fun SiteContext.renderSite(data: DataSet<Any>)
+    context(SiteContextWithData)
+    public fun renderSite()
 }
 
 public fun DataSetBuilder<Any>.site(
     name: Name,
     siteMeta: Meta,
-    block: (siteContext: SiteContext, siteData: DataSet<Any>) -> Unit,
+    block: (siteContext: SiteContext, data: DataSet<Any>) -> Unit,
 ) {
-    static(name, HtmlSite(block), siteMeta)
+    static(name, HtmlSite { block(site, siteData) }, siteMeta)
 }
 
 //public fun DataSetBuilder<Any>.site(name: Name, block: DataSetBuilder<Any>.() -> Unit) {

snark-html/src/jvmMain/kotlin/space/kscience/snark/html/Language.kt

@@ -3,13 +3,17 @@ package space.kscience.snark.html
 import space.kscience.dataforge.data.DataSet
 import space.kscience.dataforge.data.branch
 import space.kscience.dataforge.meta.*
-import space.kscience.dataforge.names.*
+import space.kscience.dataforge.names.Name
+import space.kscience.dataforge.names.asName
+import space.kscience.dataforge.names.parseAsName
+import space.kscience.dataforge.names.plus
 import space.kscience.snark.SnarkBuilder
 import space.kscience.snark.html.Language.Companion.SITE_LANGUAGES_KEY
 import space.kscience.snark.html.Language.Companion.SITE_LANGUAGE_KEY
 
 
 public class Language : Scheme() {
+
     /**
      * Language key override
      */
@@ -20,6 +24,11 @@ public class Language : Scheme() {
      */
     public var prefix: String? by string()
 
+    /**
+     * An override for data path. By default uses [prefix]
+     */
+    public var dataPath: String? by string()
+
     /**
      * Target page name with a given language key
      */
@@ -33,21 +42,21 @@ public class Language : Scheme() {
 
         public val LANGUAGE_KEY: Name = "language".asName()
 
-        public val LANGUAGES_KEY: Name = "languages".asName()
+        public val LANGUAGE_MAP_KEY: Name = "languageMap".asName()
 
         public val SITE_LANGUAGE_KEY: Name = SiteContext.SITE_META_KEY + LANGUAGE_KEY
 
-        public val SITE_LANGUAGES_KEY: Name = SiteContext.SITE_META_KEY + LANGUAGES_KEY
+        public val SITE_LANGUAGES_KEY: Name = SiteContext.SITE_META_KEY + LANGUAGE_MAP_KEY
 
         public const val DEFAULT_LANGUAGE: String = "en"
-//
+
 //        /**
 //         * Automatically build a language map for a data piece with given [name] based on existence of appropriate data nodes.
 //         */
-//        context(SiteContext)
+//        context(PageContextWithData)
-//        public fun forName(name: Name): Meta = Meta {
+//        public fun languageMapFor(name: Name): Meta = Meta {
 //            val currentLanguagePrefix = languages[language]?.get(Language::prefix.name)?.string ?: language
-//            val fullName = (route.removeFirstOrNull(currentLanguagePrefix.asName()) ?: route) + name
+//            val fullName = (site.route.removeFirstOrNull(currentLanguagePrefix.asName()) ?: site.route) + name
 //            languages.forEach { (key, meta) ->
 //                val languagePrefix: String = meta[Language::prefix.name].string ?: key
 //                val nameWithLanguage: Name = if (languagePrefix.isBlank()) {
@@ -55,7 +64,7 @@ public class Language : Scheme() {
 //                } else {
 //                    languagePrefix.asName() + fullName
 //                }
-//                if (resolveData.getItem(name) != null) {
+//                if (data.resolveHtmlOrNull(name) != null) {
 //                    key put meta.asMutableMeta().apply {
 //                        Language::target.name put nameWithLanguage.toString()
 //                    }
@@ -65,6 +74,8 @@ public class Language : Scheme() {
     }
 }
 
+public fun Language(prefix: String): Language = Language { this.prefix = prefix }
+
 public val SiteContext.languages: Map<String, Meta>
     get() = siteMeta[SITE_LANGUAGES_KEY]?.items?.mapKeys { it.key.toStringUnescaped() } ?: emptyMap()
 
@@ -74,8 +85,17 @@ public val SiteContext.language: String
 public val SiteContext.languagePrefix: Name
     get() = languages[language]?.let { it[Language::prefix.name].string ?: language }?.parseAsName() ?: Name.EMPTY
 
+/**
+ * Create a multiple sites for different languages. All sites use the same [content], but rely on different data
+ *
+ * @param data a common data root for all sites
+ */
 @SnarkBuilder
-public suspend fun SiteContext.multiLanguageSite(data: DataSet<Any>, languageMap: Map<String, Language>, site: HtmlSite) {
+public fun SiteContext.multiLanguageSite(
+    data: DataSet<*>,
+    languageMap: Map<String, Language>,
+    content: HtmlSite,
+) {
     languageMap.forEach { (languageKey, language) ->
         val prefix = language.prefix ?: languageKey
         val languageSiteMeta = Meta {
@@ -86,7 +106,12 @@ public suspend fun SiteContext.multiLanguageSite(data: DataSet<Any>, languageMap
                 }
             }
         }
-        site(prefix.parseAsName(), data.branch(prefix), siteMeta = Laminate(languageSiteMeta, siteMeta), site)
+        site(
+            prefix.parseAsName(),
+            data.branch(language.dataPath ?: prefix),
+            siteMeta = Laminate(languageSiteMeta, siteMeta),
+            content
+        )
     }
 }
 
@@ -99,11 +124,11 @@ public val PageContext.language: String
 /**
  * Mapping of language keys to other language versions of this page
  */
-public val PageContext.languages: Map<String, Meta>
+public fun PageContext.getLanguageMap(): Map<String, Meta> =
-    get() = pageMeta[Language.LANGUAGES_KEY]?.items?.mapKeys { it.key.toStringUnescaped() } ?: emptyMap()
+    pageMeta[Language.LANGUAGE_MAP_KEY]?.items?.mapKeys { it.key.toStringUnescaped() } ?: emptyMap()
 
 public fun PageContext.localisedPageRef(pageName: Name, relative: Boolean = false): String {
-    val prefix = languages[language]?.get(Language::prefix.name)?.string?.parseAsName() ?: Name.EMPTY
+    val prefix = getLanguageMap()[language]?.get(Language::prefix.name)?.string?.parseAsName() ?: Name.EMPTY
     return resolvePageRef(prefix + pageName, relative)
 }
 

snark-html/src/jvmMain/kotlin/space/kscience/snark/html/PageContext.kt

@@ -1,13 +1,13 @@
 package space.kscience.snark.html
 
-import kotlinx.html.HTML
+import space.kscience.dataforge.data.DataSet
-import space.kscience.dataforge.context.Context
-import space.kscience.dataforge.context.ContextAware
-import space.kscience.dataforge.data.*
 import space.kscience.dataforge.meta.Meta
 import space.kscience.dataforge.meta.get
 import space.kscience.dataforge.meta.string
-import space.kscience.dataforge.names.*
+import space.kscience.dataforge.names.Name
+import space.kscience.dataforge.names.asName
+import space.kscience.dataforge.names.hasIndex
+import space.kscience.dataforge.names.parseAsName
 import space.kscience.snark.SnarkBuilder
 import space.kscience.snark.SnarkContext
 
@@ -54,4 +54,7 @@ public fun PageContext.resolvePageRef(pageName: String): String = resolvePageRef
 
 public val PageContext.homeRef: String get() = resolvePageRef(SiteContext.INDEX_PAGE_TOKEN.asName())
 
-public val PageContext.name: Name? get() = pageMeta["name"].string?.parseAsName()
+public val PageContext.name: Name? get() = pageMeta["name"].string?.parseAsName()
+
+
+public class PageContextWithData(private val pageContext: PageContext, public val data: DataSet<*>): PageContext by pageContext

snark-html/src/jvmMain/kotlin/space/kscience/snark/html/PageFragment.kt

@@ -1,6 +1,5 @@
 package space.kscience.snark.html
 
-import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.runBlocking
 import kotlinx.html.FlowContent
 import space.kscience.dataforge.data.*
@@ -14,16 +13,26 @@ import space.kscience.dataforge.names.plus
 import space.kscience.dataforge.names.startsWith
 import space.kscience.snark.SnarkContext
 
-public fun interface DataFragment {
+public fun interface PageFragment {
-    public suspend fun FlowContent.renderFragment(page: PageContext, data: DataSet<*>)
+
+    context(PageContextWithData)
+    public fun FlowContent.renderFragment()
+}
+
+context(PageContextWithData)
+public fun FlowContent.fragment(fragment: PageFragment): Unit{
+    with(fragment) {
+        renderFragment()
+    }
 }
 
 
-context(PageContext)
+context(PageContextWithData)
-public fun FlowContent.htmlData(data: DataSet<*>, fragment: Data<DataFragment>): Unit = runBlocking(Dispatchers.IO) {
+public fun FlowContent.fragment(data: Data<PageFragment>): Unit = runBlocking {
-    with(fragment.await()) { renderFragment(page, data) }
+    fragment(data.await())
 }
 
+
 context(SnarkContext)
 public val Data<*>.id: String
     get() = meta["id"]?.string ?: "block[${hashCode()}]"
@@ -45,8 +54,8 @@ public val Data<*>.published: Boolean
  * Resolve a Html builder by its full name
  */
 context(SnarkContext)
-public fun DataSet<*>.resolveHtmlOrNull(name: Name): Data<DataFragment>? {
+public fun DataSet<*>.resolveHtmlOrNull(name: Name): Data<PageFragment>? {
-    val resolved = (getByType<DataFragment>(name) ?: getByType<DataFragment>(name + SiteContext.INDEX_PAGE_TOKEN))
+    val resolved = (getByType<PageFragment>(name) ?: getByType<PageFragment>(name + SiteContext.INDEX_PAGE_TOKEN))
 
     return resolved?.takeIf {
         it.published //TODO add language confirmation
@@ -54,10 +63,10 @@ public fun DataSet<*>.resolveHtmlOrNull(name: Name): Data<DataFragment>? {
 }
 
 context(SnarkContext)
-public fun DataSet<*>.resolveHtmlOrNull(name: String): Data<DataFragment>? = resolveHtmlOrNull(name.parseAsName())
+public fun DataSet<*>.resolveHtmlOrNull(name: String): Data<PageFragment>? = resolveHtmlOrNull(name.parseAsName())
 
 context(SnarkContext)
-public fun DataSet<*>.resolveHtml(name: String): Data<DataFragment> = resolveHtmlOrNull(name)
+public fun DataSet<*>.resolveHtml(name: String): Data<PageFragment> = resolveHtmlOrNull(name)
     ?: error("Html fragment with name $name is not resolved")
 
 /**
@@ -66,7 +75,7 @@ public fun DataSet<*>.resolveHtml(name: String): Data<DataFragment> = resolveHtm
 context(SnarkContext)
 public fun DataSet<*>.resolveAllHtml(
     predicate: (name: Name, meta: Meta) -> Boolean,
-): Map<Name, Data<DataFragment>> = filterByType<DataFragment> { name, meta ->
+): Map<Name, Data<PageFragment>> = filterByType<PageFragment> { name, meta ->
     predicate(name, meta)
             && meta["published"].string != "false"
     //TODO add language confirmation
@@ -76,6 +85,6 @@ context(SnarkContext)
 public fun DataSet<*>.findHtmlByContentType(
     contentType: String,
     baseName: Name = Name.EMPTY,
-): Map<Name, Data<DataFragment>> = resolveAllHtml { name, meta ->
+): Map<Name, Data<PageFragment>> = resolveAllHtml { name, meta ->
     name.startsWith(baseName) && meta["content_type"].string == contentType
 }

snark-html/src/jvmMain/kotlin/space/kscience/snark/html/Postprocessor.kt

@@ -42,13 +42,16 @@ public class WebPageTextProcessor(private val page: PageContext) : TextProcessor
 
 }
 
-public class WebPagePostprocessor<out R>(
+/**
+ * A tag consumer wrapper that wraps existing [TagConsumer] and adds postprocessing.
+ *
+ */
+public class Postprocessor<out R>(
     public val page: PageContext,
     private val consumer: TagConsumer<R>,
+    private val processor: TextProcessor = WebPageTextProcessor(page),
 ) : TagConsumer<R> by consumer {
 
-    private val processor = WebPageTextProcessor(page)
-
     override fun onTagAttributeChange(tag: Tag, attribute: String, value: String?) {
         if (tag is A && attribute == "href" && value != null) {
             consumer.onTagAttributeChange(tag, attribute, processor.process(value))
@@ -73,9 +76,10 @@ public class WebPagePostprocessor<out R>(
     }
 }
 
-public inline fun FlowContent.withSnarkPage(page: PageContext, block: FlowContent.() -> Unit) {
+context(PageContext)
+public inline fun FlowContent.postprocess(block: FlowContent.() -> Unit) {
     val fc = object : FlowContent by this {
-        override val consumer: TagConsumer<*> = WebPagePostprocessor(page, this@withSnarkPage.consumer)
+        override val consumer: TagConsumer<*> = Postprocessor(page, this@postprocess.consumer)
     }
     fc.block()
 }

snark-html/src/jvmMain/kotlin/space/kscience/snark/html/SiteContext.kt

@@ -1,6 +1,5 @@
 package space.kscience.snark.html
 
-import kotlinx.html.HTML
 import space.kscience.dataforge.data.*
 import space.kscience.dataforge.io.Binary
 import space.kscience.dataforge.meta.Meta
@@ -35,34 +34,43 @@ public interface SiteContext : SnarkContext {
      * @param route The route name of the static file relative to the site root.
      * @param data The data object containing the binary data for the static file.
      */
-    public suspend fun static(route: Name, data: Data<Binary>)
+    public fun static(route: Name, data: Data<Binary>)
 
 
     /**
-     * Create a single page at given [route]. If route is empty, create an index page at current route.
+     * Create a single page at given [route]. If the route is empty, create an index page the current route.
      *
      * @param pageMeta additional page meta. [PageContext] will use both it and [siteMeta]
      */
     @SnarkBuilder
-    public suspend fun page(
+    public fun page(
         route: Name,
-        data: DataSet<Any>,
+        data: DataSet<*>,
         pageMeta: Meta = Meta.EMPTY,
-        htmlPage: HtmlPage,
+        content: HtmlPage,
     )
 
+    /**
+     * Create a route block with its own data. Does not change base url
+     */
+    @SnarkBuilder
+    public fun route(
+        route: Name,
+        data: DataSet<*>,
+        siteMeta: Meta = Meta.EMPTY,
+        content: HtmlSite,
+    )
 
     /**
      * Creates a sub-site and sets it as site base url
      * @param route mount site at [rootName]
-     * @param dataPrefix prefix path for data used in this site
      */
     @SnarkBuilder
-    public suspend fun site(
+    public fun site(
         route: Name,
-        data: DataSet<Any>,
+        data: DataSet<*>,
         siteMeta: Meta = Meta.EMPTY,
-        htmlSite: HtmlSite,
+        content: HtmlSite,
     )
 
 
@@ -73,13 +81,14 @@ public interface SiteContext : SnarkContext {
     }
 }
 
-public suspend fun SiteContext.static(dataSet: DataSet<Binary>, prefix: Name = Name.EMPTY) {
+public fun SiteContext.static(dataSet: DataSet<Binary>, prefix: Name = Name.EMPTY) {
     dataSet.forEach { (name, data) ->
         static(prefix + name, data)
     }
 }
 
-public suspend fun SiteContext.static(dataSet: DataSet<*>, branch: String, prefix: String = branch) {
+
+public fun SiteContext.static(dataSet: DataSet<*>, branch: String, prefix: String = branch) {
     val branchName = branch.parseAsName()
     val prefixName = prefix.parseAsName()
     val binaryType = typeOf<Binary>()
@@ -91,20 +100,50 @@ public suspend fun SiteContext.static(dataSet: DataSet<*>, branch: String, prefi
     }
 }
 
-@SnarkBuilder
+
-public suspend fun SiteContext.page(
-    route: Name,
-    data: DataSet<Any>,
-    pageMeta: Meta = Meta.EMPTY,
-    htmlPage: HTML.(page: PageContext, data: DataSet<Any>) -> Unit,
-): Unit = page(route, data, pageMeta, HtmlPage(htmlPage))
 
 context(SiteContext)
 public val site: SiteContext
     get() = this@SiteContext
 
 
-public suspend fun SiteContext.renderPages(data: DataSet<Any>): Unit {
+/**
+ * A wrapper for site context that allows convenient site building experience
+ */
+public class SiteContextWithData(private val site: SiteContext, public val siteData: DataSet<*>) : SiteContext by site
+
+
+@SnarkBuilder
+public fun SiteContextWithData.static(branch: String, prefix: String = branch): Unit = static(siteData, branch, prefix)
+
+
+@SnarkBuilder
+public fun SiteContextWithData.page(
+    route: Name = Name.EMPTY,
+    pageMeta: Meta = Meta.EMPTY,
+    content: HtmlPage,
+): Unit = page(route, siteData, pageMeta, content)
+
+@SnarkBuilder
+public suspend fun SiteContextWithData.route(
+    route: String,
+    data: DataSet<*> = siteData,
+    siteMeta: Meta = Meta.EMPTY,
+    content: HtmlSite,
+): Unit = route(route.parseAsName(), data, siteMeta,content)
+
+@SnarkBuilder
+public suspend fun SiteContextWithData.site(
+    route: String,
+    data: DataSet<*> = siteData,
+    siteMeta: Meta = Meta.EMPTY,
+    content: HtmlSite,
+): Unit = site(route.parseAsName(), data, siteMeta,content)
+
+/**
+ * Render all pages and sites found in the data
+ */
+public suspend fun SiteContext.renderPages(data: DataSet<*>): Unit {
 
     // Render all sub-sites
     data.filterByType<HtmlSite>().forEach { siteData: NamedData<HtmlSite> ->

snark-html/src/jvmMain/kotlin/space/kscience/snark/html/SnarkHtml.kt

@@ -4,6 +4,7 @@ package space.kscience.snark.html
 
 import io.ktor.http.ContentType
 import kotlinx.io.readByteArray
+import space.kscience.dataforge.actions.Action
 import space.kscience.dataforge.context.*
 import space.kscience.dataforge.data.*
 import space.kscience.dataforge.io.IOPlugin
@@ -15,10 +16,12 @@ import space.kscience.dataforge.meta.Meta
 import space.kscience.dataforge.meta.get
 import space.kscience.dataforge.meta.string
 import space.kscience.dataforge.misc.DFExperimental
-import space.kscience.dataforge.names.*
+import space.kscience.dataforge.names.Name
+import space.kscience.dataforge.names.NameToken
+import space.kscience.dataforge.names.asName
+import space.kscience.dataforge.names.replaceLast
 import space.kscience.dataforge.provider.dfId
 import space.kscience.dataforge.workspace.*
-import space.kscience.snark.ImageIOReader
 import space.kscience.snark.Snark
 import space.kscience.snark.SnarkReader
 import space.kscience.snark.TextProcessor
@@ -27,6 +30,13 @@ import kotlin.io.path.Path
 import kotlin.io.path.extension
 
 
+public fun <T : Any, R : Any> DataSet<T>.transform(action: Action<T, R>, meta: Meta = Meta.EMPTY): DataSet<R> =
+    action.execute(this, meta)
+
+public fun <T : Any> TaskResultBuilder<T>.fill(dataSet: DataSet<T>) {
+    node(Name.EMPTY, dataSet)
+}
+
 /**
  * A plugin used for rendering a [DataTree] as HTML
  */
@@ -43,56 +53,52 @@ public class SnarkHtml : WorkspacePlugin() {
             "markdown".asName() to MarkdownReader,
             "json".asName() to SnarkReader(JsonMetaFormat, ContentType.Application.Json.toString()),
             "yaml".asName() to SnarkReader(YamlMetaFormat, "text/yaml", "yaml"),
-            "png".asName() to SnarkReader(ImageIOReader, ContentType.Image.PNG.toString()),
+//            "png".asName() to SnarkReader(ImageIOReader, ContentType.Image.PNG.toString()),
-            "jpg".asName() to SnarkReader(ImageIOReader, ContentType.Image.JPEG.toString()),
+//            "jpg".asName() to SnarkReader(ImageIOReader, ContentType.Image.JPEG.toString()),
-            "gif".asName() to SnarkReader(ImageIOReader, ContentType.Image.GIF.toString()),
+//            "gif".asName() to SnarkReader(ImageIOReader, ContentType.Image.GIF.toString()),
-            "svg".asName() to SnarkReader(IOReader.binary, ContentType.Image.SVG.toString(), "svg"),
+//            "svg".asName() to SnarkReader(IOReader.binary, ContentType.Image.SVG.toString(), "svg"),
-            "raw".asName() to SnarkReader(
+//            "raw".asName() to SnarkReader(
-                IOReader.binary,
+//                IOReader.binary,
-                "css",
+//                "css",
-                "js",
+//                "js",
-                "javascript",
+//                "javascript",
-                "scss",
+//                "scss",
-                "woff",
+//                "woff",
-                "woff2",
+//                "woff2",
-                "ttf",
+//                "ttf",
-                "eot"
+//                "eot"
-            )
+//            )
         )
 
         else -> super.content(target)
     }
 
-    public val preprocess: TaskReference<String> by task<String> {
+    public val read: TaskReference<String> by task<String>{
-        pipeFrom<String, String>(dataByType<String>()) { text, _, meta ->
+
-            meta[TextProcessor.TEXT_TRANSFORMATION_KEY]?.let {
-                snark.textProcessor(it).process(text)
-            } ?: text
-        }
     }
 
     public val parse: TaskReference<Any> by task<Any> {
-        from(preprocess).forEach { (dataName, data) ->
+        from(read).forEach { (dataName, data) ->
             //remove extensions for data files
             val filePath = meta[FileData.FILE_PATH_KEY]?.string ?: dataName.toString()
             val fileType = URLConnection.guessContentTypeFromName(filePath) ?: Path(filePath).extension
-            val newName = dataName.replaceLast {
-                if (fileType in setOf("md", "html", "yaml", "json")) {
-                    NameToken(it.body.substringBeforeLast("."), it.index)
-                } else {
-                    it
-                }
-            }
             val parser = snark.readers.values.filter { parser ->
                 fileType in parser.types
             }.maxByOrNull {
                 it.priority
             } ?: run {
-                logger.debug { "The parser is not found for file $filePath with meta $meta" }
+                logger.debug { "The parser is not found for file $filePath with meta $meta. Passing data without parsing" }
-                byteArraySnarkParser
+                data(dataName, data)
+                return@forEach
             }
+            val newName = dataName.replaceLast {
+                NameToken(it.body.substringBeforeLast("."), it.index)
+            }
+            val preprocessor = meta[TextProcessor.TEXT_TRANSFORMATION_KEY]?.let{snark.preprocessor(it)}
+
             data(newName, data.map { string: String ->
-                parser.readFrom(string)
+                val preprocessed = preprocessor?.process(string) ?: string
+                parser.readFrom(preprocessed)
             })
         }
     }

snark-html/src/jvmMain/kotlin/space/kscience/snark/html/readers.kt

@@ -11,25 +11,25 @@ import space.kscience.snark.SnarkReader
 import kotlin.reflect.KType
 import kotlin.reflect.typeOf
 
-public object HtmlReader : SnarkReader<DataFragment> {
+public object HtmlReader : SnarkReader<PageFragment> {
     override val types: Set<String> = setOf("html")
 
-    override fun readFrom(source: String): DataFragment = DataFragment { _, _ ->
+    override fun readFrom(source: String): PageFragment = PageFragment {
         div {
             unsafe { +source }
         }
     }
 
-    override fun readFrom(source: Source): DataFragment = readFrom(source.readString())
+    override fun readFrom(source: Source): PageFragment = readFrom(source.readString())
-    override val type: KType = typeOf<DataFragment>()
+    override val type: KType = typeOf<PageFragment>()
 }
 
-public object MarkdownReader : SnarkReader<DataFragment> {
+public object MarkdownReader : SnarkReader<PageFragment> {
-    override val type: KType = typeOf<DataFragment>()
+    override val type: KType = typeOf<PageFragment>()
 
     override val types: Set<String> = setOf("text/markdown", "md", "markdown")
 
-    override fun readFrom(source: String): DataFragment = DataFragment { _, _ ->
+    override fun readFrom(source: String): PageFragment = PageFragment {
         val parsedTree = markdownParser.buildMarkdownTreeFromString(source)
         val htmlString = HtmlGenerator(source, parsedTree, markdownFlavor).generateHtml()
 
@@ -43,9 +43,9 @@ public object MarkdownReader : SnarkReader<DataFragment> {
     private val markdownFlavor = CommonMarkFlavourDescriptor()
     private val markdownParser = MarkdownParser(markdownFlavor)
 
-    override fun readFrom(source: Source): DataFragment = readFrom(source.readString())
+    override fun readFrom(source: Source): PageFragment = readFrom(source.readString())
 
-    public val snarkReader: SnarkReader<DataFragment> = SnarkReader(this, "text/markdown")
+    public val snarkReader: SnarkReader<PageFragment> = SnarkReader(this, "text/markdown")
 
 }
 

snark-html/src/jvmMain/kotlin/space/kscience/snark/html/static/StaticSiteContext.kt

@@ -1,7 +1,7 @@
 package space.kscience.snark.html.static
 
-import kotlinx.html.html
+import kotlinx.coroutines.Dispatchers
-import kotlinx.html.stream.createHTML
+import kotlinx.coroutines.runBlocking
 import kotlinx.io.asSink
 import kotlinx.io.buffered
 import space.kscience.dataforge.data.*
@@ -14,8 +14,6 @@ import space.kscience.dataforge.names.plus
 import space.kscience.dataforge.workspace.FileData
 import space.kscience.snark.html.*
 import java.nio.file.Path
-import kotlin.contracts.InvocationKind
-import kotlin.contracts.contract
 import kotlin.io.path.*
 import kotlin.reflect.typeOf
 
@@ -63,7 +61,8 @@ internal class StaticSiteContext(
 //    }
 
     @OptIn(ExperimentalPathApi::class)
-    override suspend fun static(route: Name, data: Data<Binary>) {
+    override fun static(route: Name, data: Data<Binary>) {
+        //if data is a file, copy it
         data.meta[FileData.FILE_PATH_KEY]?.string?.let {
             val file = Path.of(it)
             val targetPath = outputPath.resolve(route.toWebPath())
@@ -75,9 +74,11 @@ internal class StaticSiteContext(
 
         if (data.type != typeOf<Binary>()) error("Can't directly serve file of type ${data.type}")
         val targetPath = outputPath.resolve(route.toWebPath())
-        val binary = data.await()
+        runBlocking(Dispatchers.IO) {
-        targetPath.outputStream().asSink().buffered().use {
+            val binary = data.await()
-            it.writeBinary(binary)
+            targetPath.outputStream().asSink().buffered().use {
+                it.writeBinary(binary)
+            }
         }
     }
 
@@ -99,7 +100,7 @@ internal class StaticSiteContext(
         )
     }
 
-    override suspend fun page(route: Name, data: DataSet<Any>, pageMeta: Meta, htmlPage: HtmlPage) {
+    override fun page(route: Name, data: DataSet<Any>, pageMeta: Meta, content: HtmlPage) {
 
 
         val modifiedPageMeta = pageMeta.toMutableMeta().apply {
@@ -115,17 +116,40 @@ internal class StaticSiteContext(
         newPath.parent.createDirectories()
 
         val pageContext = StaticPageContext(this, Laminate(modifiedPageMeta, siteMeta))
-        newPath.writeText(HtmlPage.createHtmlString(pageContext,htmlPage, data))
+        newPath.writeText(HtmlPage.createHtmlString(pageContext, data, content))
     }
 
-    override suspend fun site(route: Name, data: DataSet<Any>, siteMeta: Meta, htmlSite: HtmlSite) {
+    override fun route(route: Name, data: DataSet<*>, siteMeta: Meta, content: HtmlSite) {
-        with(htmlSite) {
+        val siteContextWithData = SiteContextWithData(
+            StaticSiteContext(
+                siteMeta = Laminate(siteMeta, this@StaticSiteContext.siteMeta),
+                baseUrl = baseUrl,
+                route = route,
+                outputPath = outputPath.resolve(route.toWebPath())
+            ),
+            data
+        )
+        with(content) {
+            with(siteContextWithData) {
+                renderSite()
+            }
+        }
+    }
+
+    override fun site(route: Name, data: DataSet<Any>, siteMeta: Meta, content: HtmlSite) {
+        val siteContextWithData = SiteContextWithData(
             StaticSiteContext(
                 siteMeta = Laminate(siteMeta, this@StaticSiteContext.siteMeta),
                 baseUrl = if (baseUrl == "") "" else resolveRef(baseUrl, route.toWebPath()),
                 route = Name.EMPTY,
                 outputPath = outputPath.resolve(route.toWebPath())
-            ).renderSite(data)
+            ),
+            data
+        )
+        with(content) {
+            with(siteContextWithData) {
+                renderSite()
+            }
         }
     }
 
@@ -136,15 +160,21 @@ internal class StaticSiteContext(
  * Use [siteUrl] as a base for all resolved URLs. By default, use [outputPath] absolute path as a base.
  *
  */
-public fun SnarkHtml.staticSite(
+@Suppress("UnusedReceiverParameter")
+public suspend fun SnarkHtml.staticSite(
     data: DataSet<*>,
     outputPath: Path,
     siteUrl: String = outputPath.absolutePathString().replace("\\", "/"),
     siteMeta: Meta = data.meta,
-    block: SiteContext.() -> Unit,
+    content: HtmlSite,
 ) {
-    contract {
+    val siteContextWithData = SiteContextWithData(
-        callsInPlace(block, InvocationKind.EXACTLY_ONCE)
+        StaticSiteContext(siteMeta, siteUrl, Name.EMPTY, outputPath),
+        data
+    )
+    with(content){
+        with(siteContextWithData) {
+            renderSite()
+        }
     }
-    StaticSiteContext(siteMeta, siteUrl, Name.EMPTY, outputPath).block()
 }

snark-ktor/src/main/kotlin/space/kscience/snark/ktor/KtorSiteContext.kt

@@ -2,7 +2,6 @@ package space.kscience.snark.ktor
 
 import io.ktor.http.*
 import io.ktor.http.content.TextContent
-import io.ktor.server.application.Application
 import io.ktor.server.application.call
 import io.ktor.server.http.content.staticFiles
 import io.ktor.server.plugins.origin
@@ -11,14 +10,12 @@ import io.ktor.server.response.respondBytes
 import io.ktor.server.routing.Route
 import io.ktor.server.routing.createRouteFromPath
 import io.ktor.server.routing.get
-import io.ktor.server.routing.routing
 import space.kscience.dataforge.context.Context
 import space.kscience.dataforge.context.ContextAware
 import space.kscience.dataforge.context.error
 import space.kscience.dataforge.context.logger
 import space.kscience.dataforge.data.Data
 import space.kscience.dataforge.data.DataSet
-import space.kscience.dataforge.data.DataTree
 import space.kscience.dataforge.data.await
 import space.kscience.dataforge.io.Binary
 import space.kscience.dataforge.io.toByteArray
@@ -33,8 +30,6 @@ import space.kscience.dataforge.names.plus
 import space.kscience.dataforge.workspace.FileData
 import space.kscience.snark.html.*
 import java.nio.file.Path
-import kotlin.contracts.InvocationKind
-import kotlin.contracts.contract
 import kotlin.reflect.typeOf
 
 //public fun CommonAttributeGroupFacade.css(block: CssBuilder.() -> Unit) {
@@ -50,7 +45,7 @@ public class KtorSiteContext(
 ) : SiteContext, ContextAware {
 
 
-    override suspend fun static(route: Name, data: Data<Binary>) {
+    override fun static(route: Name, data: Data<Binary>) {
         data.meta[FileData.FILE_PATH_KEY]?.string?.let {
             val file = try {
                 Path.of(it).toFile()
@@ -111,7 +106,7 @@ public class KtorSiteContext(
         }
     }
 
-    override suspend fun page(route: Name, data: DataSet<Any>, pageMeta: Meta, htmlPage: HtmlPage) {
+    override fun page(route: Name, data: DataSet<*>, pageMeta: Meta, content: HtmlPage) {
         ktorRoute.get(route.toWebPath()) {
             val request = call.request
             //substitute host for url for backwards calls
@@ -128,50 +123,75 @@ public class KtorSiteContext(
             val pageContext =
                 KtorPageContext(this@KtorSiteContext, url.buildString(), Laminate(modifiedPageMeta, siteMeta))
             //render page in suspend environment
-            val html = HtmlPage.createHtmlString(pageContext, htmlPage, data)
+            val html = HtmlPage.createHtmlString(pageContext, data, content)
 
             call.respond(TextContent(html, ContentType.Text.Html.withCharset(Charsets.UTF_8), HttpStatusCode.OK))
         }
     }
 
-    override suspend fun site(route: Name, data: DataSet<Any>, siteMeta: Meta, htmlSite: HtmlSite) {
+    override fun route(route: Name, data: DataSet<*>, siteMeta: Meta, content: HtmlSite) {
-        with(htmlSite) {
+        val siteContext = SiteContextWithData(
+            KtorSiteContext(
+                context,
+                siteMeta = Laminate(siteMeta, this@KtorSiteContext.siteMeta),
+                baseUrl = baseUrl,
+                route = route,
+                ktorRoute = ktorRoute.createRouteFromPath(route.toWebPath())
+            ),
+            data
+        )
+        with(content) {
+            with(siteContext) {
+                renderSite()
+            }
+        }
+    }
+
+    override fun site(route: Name, data: DataSet<*>, siteMeta: Meta, content: HtmlSite) {
+        val siteContext = SiteContextWithData(
             KtorSiteContext(
                 context,
                 siteMeta = Laminate(siteMeta, this@KtorSiteContext.siteMeta),
                 baseUrl = resolveRef(baseUrl, route.toWebPath()),
                 route = Name.EMPTY,
                 ktorRoute = ktorRoute.createRouteFromPath(route.toWebPath())
-            ).renderSite(data)
+            ),
+            data
+        )
+        with(content) {
+            with(siteContext) {
+                renderSite()
+            }
         }
     }
 
 }
 
-private fun Route.site(
+public fun Route.site(
     context: Context,
-    data: DataTree<*>,
+    data: DataSet<*>,
     baseUrl: String = "",
     siteMeta: Meta = data.meta,
-    block: KtorSiteContext.() -> Unit,
+    content: HtmlSite,
 ) {
-    contract {
+    val siteContext = SiteContextWithData(
-        callsInPlace(block, InvocationKind.EXACTLY_ONCE)
+        KtorSiteContext(context, siteMeta, baseUrl, route = Name.EMPTY, this@Route),
-    }
+        data
-    block(KtorSiteContext(context, siteMeta, baseUrl, route = Name.EMPTY, this@Route))
+    )
-}
+    with(content) {
-
+        with(siteContext) {
-public fun Application.site(
+            renderSite()
-    context: Context,
+        }
-    data: DataTree<*>,
-    baseUrl: String = "",
-    siteMeta: Meta = data.meta,
-    block: SiteContext.() -> Unit,
-) {
-    contract {
-        callsInPlace(block, InvocationKind.EXACTLY_ONCE)
-    }
-    routing {
-        site(context, data, baseUrl, siteMeta, block)
     }
 }
+//
+//public suspend fun Application.site(
+//    context: Context,
+//    data: DataSet<*>,
+//    baseUrl: String = "",
+//    siteMeta: Meta = data.meta,
+//    content: HtmlSite,
+//) {
+//    routing {}.site(context, data, baseUrl, siteMeta, content)
+//
+//}