Update to DataForge 0.8

build.gradle.kts

@@ -1,4 +1,5 @@
-import space.kscience.gradle.*
+import space.kscience.gradle.useApache2Licence
+import space.kscience.gradle.useSPCTeam
 
 plugins {
     id("space.kscience.gradle.project")
@@ -14,7 +15,7 @@ allprojects {
     }
 }
 
-val dataforgeVersion by extra("0.8.0-dev-1")
+val dataforgeVersion by extra("0.8.0")
 
 ksciencePublish {
     pom("https://github.com/SciProgCentre/snark") {

gradle.properties

@@ -1,3 +1,3 @@
 kotlin.code.style=official
 
-toolsVersion=0.15.2-kotlin-1.9.21
+toolsVersion=0.15.2-kotlin-1.9.22

snark-core/src/commonMain/kotlin/space/kscience/snark/ReWrapAction.kt

@@ -5,9 +5,7 @@ import space.kscience.dataforge.data.*
 import space.kscience.dataforge.meta.Meta
 import space.kscience.dataforge.meta.MutableMeta
 import space.kscience.dataforge.meta.copy
-import space.kscience.dataforge.names.Name
-import space.kscience.dataforge.names.NameToken
-import space.kscience.dataforge.names.replaceLast
+import space.kscience.dataforge.names.*
 import kotlin.reflect.KType
 import kotlin.reflect.typeOf
 
@@ -19,20 +17,19 @@ public class ReWrapAction<R : Any>(
     private val newMeta: MutableMeta.(name: Name) -> Unit = {},
     private val newName: (name: Name, meta: Meta?) -> Name,
 ) : AbstractAction<R, R>(type) {
-    override fun DataSetBuilder<R>.generate(data: DataSet<R>, meta: Meta) {
+    override fun DataSink<R>.generate(data: DataTree<R>, meta: Meta) {
         data.forEach { namedData ->
-            data(
+            put(
                 newName(namedData.name, namedData.meta),
                 namedData.data.withMeta(namedData.meta.copy { newMeta(namedData.name) })
             )
         }
     }
 
-    override fun DataSourceBuilder<R>.update(dataSet: DataSet<R>, meta: Meta, updateKey: Name) {
-        val datum = dataSet[updateKey]
-        data(
-            newName(updateKey, datum?.meta),
-            datum?.withMeta(datum.meta.copy { newMeta(updateKey) })
+    override fun DataSink<R>.update(source: DataTree<R>, meta: Meta, namedData: NamedData<R>) {
+        put(
+            newName(namedData.name, namedData.meta),
+            namedData.withMeta(namedData.meta.copy { newMeta(namedData.name) })
         )
     }
 
@@ -50,11 +47,15 @@ public class ReWrapAction<R : Any>(
                 }
             }
         }
+
+        public inline fun <reified R : Any> removeIndex(): ReWrapAction<R> = ReWrapAction<R>(typeOf<R>()) { name, _ ->
+            if (name.endsWith("index")) name.cutLast() else name
+        }
     }
 }
 
 public inline fun <reified R : Any> ReWrapAction(
     noinline newMeta: MutableMeta.(name: Name) -> Unit = {},
-    noinline newName: (Name, Meta?) -> Name
+    noinline newName: (Name, Meta?) -> Name,
 ): ReWrapAction<R> = ReWrapAction(typeOf<R>(), newMeta, newName)
 

snark-core/src/jvmMain/kotlin/space/kscience/snark/ImageIOReader.kt

@@ -5,8 +5,6 @@ import kotlinx.io.asInputStream
 import space.kscience.dataforge.io.IOReader
 import java.awt.image.BufferedImage
 import javax.imageio.ImageIO
-import kotlin.reflect.KType
-import kotlin.reflect.typeOf
 
 /**
  * The ImageIOReader class is an implementation of the IOReader interface specifically for reading images using the ImageIO library.
@@ -15,7 +13,5 @@ import kotlin.reflect.typeOf
  * @property type The KType of the data to be read by the ImageIOReader.
  */
 public object ImageIOReader : IOReader<BufferedImage> {
-    override val type: KType get() = typeOf<BufferedImage>()
-
     override fun readFrom(source: Source): BufferedImage = ImageIO.read(source.asInputStream())
 }

snark-core/src/jvmMain/kotlin/space/kscience/snark/extensions.kt

@@ -1,16 +1,12 @@
 package space.kscience.snark
 
-import space.kscience.dataforge.data.DataSet
+
+import space.kscience.dataforge.data.DataSource
+import space.kscience.dataforge.data.DataTree
 import space.kscience.dataforge.data.filterByType
 import space.kscience.dataforge.names.Name
-import space.kscience.dataforge.names.parseAsName
 import space.kscience.dataforge.names.startsWith
-import kotlin.reflect.typeOf
 
-public inline fun <reified R : Any> DataSet<*>.branch(
+public inline fun <reified R : Any> DataTree<*>.branch(
     branchName: Name,
-): DataSet<R> = filterByType(typeOf<R>()) { name, _ -> name.startsWith(branchName) }
-
-public inline fun <reified R : Any> DataSet<*>.branch(
-    branchName: String,
-): DataSet<R> = filterByType(typeOf<R>()) { name, _ -> name.startsWith(branchName.parseAsName()) }
+): DataSource<R> = filterByType<R> { name, _, _ -> name.startsWith(branchName) }

snark-core/src/jvmMain/kotlin/space/kscience/snark/snarkWorkspace.kt

@@ -2,38 +2,36 @@
 
 package space.kscience.snark
 
-import space.kscience.dataforge.data.DataTree
-import space.kscience.dataforge.data.node
-import space.kscience.dataforge.io.Binary
-import space.kscience.dataforge.io.IOPlugin
+import space.kscience.dataforge.data.branch
 import space.kscience.dataforge.meta.*
 import space.kscience.dataforge.misc.DFExperimental
+import space.kscience.dataforge.names.parseAsName
 import space.kscience.dataforge.workspace.Workspace
 import space.kscience.dataforge.workspace.WorkspaceBuilder
-import space.kscience.dataforge.workspace.readRawDirectory
+import space.kscience.dataforge.workspace.directory
+import space.kscience.dataforge.workspace.resources
 import kotlin.io.path.Path
-import kotlin.io.path.toPath
 
-
-/**
- * Reads the specified resources and returns a [DataTree] containing the data.
- *
- * @param resources The names of the resources to read.
- * @param classLoader The class loader to use for loading the resources. By default, it uses the current thread's context class loader.
- * @return A DataTree containing the data read from the resources.
- */
-private fun IOPlugin.readResources(
-    vararg resources: String,
-    classLoader: ClassLoader = Thread.currentThread().contextClassLoader,
-): DataTree<Binary> = DataTree {
-    //    require(resource.isNotBlank()) {"Can't mount root resource tree as data root"}
-    resources.forEach { resource ->
-        val path = classLoader.getResource(resource)?.toURI()?.toPath() ?: error(
-            "Resource with name $resource is not resolved"
-        )
-        node(resource, readRawDirectory(path))
-    }
-}
+//
+///**
+// * Reads the specified resources and returns a [DataTree] containing the data.
+// *
+// * @param resources The names of the resources to read.
+// * @param classLoader The class loader to use for loading the resources. By default, it uses the current thread's context class loader.
+// * @return A DataTree containing the data read from the resources.
+// */
+//private fun IOPlugin.readResources(
+//    vararg resources: String,
+//    classLoader: ClassLoader = Thread.currentThread().contextClassLoader,
+//): DataTree<Binary> = DataTree {
+//    //    require(resource.isNotBlank()) {"Can't mount root resource tree as data root"}
+//    resources.forEach { resource ->
+//        val path = classLoader.getResource(resource)?.toURI()?.toPath() ?: error(
+//            "Resource with name $resource is not resolved"
+//        )
+//        node(resource, readRawDirectory(path))
+//    }
+//}
 
 public fun Snark.workspace(
     meta: Meta,
@@ -45,14 +43,14 @@ public fun Snark.workspace(
         meta.getIndexed("directory").forEach { (index, directoryMeta) ->
             val dataDirectory = directoryMeta["path"].string ?: error("Directory path not defined")
             val nodeName = directoryMeta["name"].string ?: directoryMeta.string ?: index ?: ""
-            val data = io.readRawDirectory(Path(dataDirectory))
-            node(nodeName, data)
+            directory(io, nodeName.parseAsName(), Path((dataDirectory)))
         }
         meta.getIndexed("resource").forEach { (index, resourceMeta) ->
             val resource = resourceMeta["path"]?.stringList ?: listOf("/")
             val nodeName = resourceMeta["name"].string ?: resourceMeta.string ?: index ?: ""
-            val data: DataTree<Binary> = io.readResources(*resource.toTypedArray())
-            node(nodeName, data)
+            branch(nodeName) {
+                resources(io, *resource.toTypedArray())
+            }
         }
     }
 

snark-html/src/jvmMain/kotlin/space/kscience/snark/html/HtmlPage.kt

@@ -3,9 +3,9 @@ package space.kscience.snark.html
 import kotlinx.html.HTML
 import kotlinx.html.stream.createHTML
 import kotlinx.html.visitTagAndFinalize
-import space.kscience.dataforge.data.DataSet
-import space.kscience.dataforge.data.DataSetBuilder
-import space.kscience.dataforge.data.static
+import space.kscience.dataforge.data.DataSink
+import space.kscience.dataforge.data.DataTree
+import space.kscience.dataforge.data.wrap
 import space.kscience.dataforge.meta.Meta
 import space.kscience.dataforge.names.Name
 
@@ -17,11 +17,11 @@ public fun interface HtmlPage {
     public companion object {
         public fun createHtmlString(
             pageContext: PageContext,
-            dataSet: DataSet<*>,
+            dataSet: DataTree<*>?,
             page: HtmlPage,
         ): String = createHTML().run {
             HTML(kotlinx.html.emptyMap, this, null).visitTagAndFinalize(this) {
-                with(PageContextWithData(pageContext, dataSet)) {
+                with(PageContextWithData(pageContext, dataSet ?: DataTree.EMPTY)) {
                     with(page) {
                         renderPage()
                     }
@@ -34,13 +34,13 @@ public fun interface HtmlPage {
 
 // data builders
 
-public fun DataSetBuilder<Any>.page(
+public fun DataSink<Any>.page(
     name: Name,
     pageMeta: Meta = Meta.EMPTY,
     block: context(PageContextWithData) HTML.() -> Unit,
 ) {
     val page = HtmlPage(block)
-    static<HtmlPage>(name, page, pageMeta)
+    wrap<HtmlPage>(name, page, pageMeta)
 }
 
 

snark-html/src/jvmMain/kotlin/space/kscience/snark/html/HtmlSite.kt

@@ -1,8 +1,8 @@
 package space.kscience.snark.html
 
-import space.kscience.dataforge.data.DataSet
-import space.kscience.dataforge.data.DataSetBuilder
-import space.kscience.dataforge.data.static
+import space.kscience.dataforge.data.DataSink
+import space.kscience.dataforge.data.DataTree
+import space.kscience.dataforge.data.wrap
 import space.kscience.dataforge.meta.Meta
 import space.kscience.dataforge.meta.getIndexed
 import space.kscience.dataforge.meta.string
@@ -15,23 +15,23 @@ public fun interface HtmlSite {
     public fun renderSite()
 }
 
-public fun DataSetBuilder<Any>.site(
+public fun DataSink<Any>.site(
     name: Name,
     siteMeta: Meta,
-    block: (siteContext: SiteContext, data: DataSet<Any>) -> Unit,
+    block: (siteContext: SiteContext, data: DataTree<*>?) -> Unit,
 ) {
-    static(name, HtmlSite { block(site, siteData) }, siteMeta)
+    wrap(name, HtmlSite { block(site, siteData) }, siteMeta)
 }
 
 //public fun DataSetBuilder<Any>.site(name: Name, block: DataSetBuilder<Any>.() -> Unit) {
 //    node(name, block)
 //}
 
-internal fun DataSetBuilder<Any>.assetsFrom(rootMeta: Meta) {
+internal fun DataSink<Any>.assetsFrom(rootMeta: Meta) {
     rootMeta.getIndexed("file".asName()).forEach { (_, meta) ->
         val webName: String? by meta.string()
         val name by meta.string { error("File path is not provided") }
         val fileName = name.parseAsName()
-        static(fileName, webName?.parseAsName() ?: fileName)
+        wrap(fileName, webName?.parseAsName() ?: fileName)
     }
 }

snark-html/src/jvmMain/kotlin/space/kscience/snark/html/Language.kt

@@ -1,15 +1,14 @@
 package space.kscience.snark.html
 
-import space.kscience.dataforge.data.DataSet
-import space.kscience.dataforge.data.branch
+import space.kscience.dataforge.actions.AbstractAction
+import space.kscience.dataforge.actions.transform
+import space.kscience.dataforge.data.*
 import space.kscience.dataforge.meta.*
-import space.kscience.dataforge.names.Name
-import space.kscience.dataforge.names.asName
-import space.kscience.dataforge.names.parseAsName
-import space.kscience.dataforge.names.plus
+import space.kscience.dataforge.names.*
 import space.kscience.snark.SnarkBuilder
 import space.kscience.snark.html.Language.Companion.SITE_LANGUAGES_KEY
 import space.kscience.snark.html.Language.Companion.SITE_LANGUAGE_KEY
+import kotlin.reflect.typeOf
 
 
 public class Language : Scheme() {
@@ -81,9 +80,37 @@ public val SiteContext.languages: Map<String, Meta>
 
 public val SiteContext.language: String
     get() = siteMeta[SITE_LANGUAGE_KEY].string ?: Language.DEFAULT_LANGUAGE
+//
+//public val SiteContext.languagePrefix: Name
+//    get() = languages[language]?.let { it[Language::prefix.name].string ?: language }?.parseAsName() ?: Name.EMPTY
 
-public val SiteContext.languagePrefix: Name
-    get() = languages[language]?.let { it[Language::prefix.name].string ?: language }?.parseAsName() ?: Name.EMPTY
+
+private class LanguageBranchAction(val prefix: Name) : AbstractAction<Any, Any>(typeOf<Any>()) {
+    override fun DataSink<Any>.generate(data: DataTree<Any>, meta: Meta) {
+        data.forEach { (name, item) ->
+            val nameWithoutPrefix = name.removeFirstOrNull(prefix)
+            if (nameWithoutPrefix != null) {
+                // put language item as is
+                put(nameWithoutPrefix, item)
+            } else if (data[name + prefix] == null) {
+                // put if language item is missing
+                put(name, item)
+            }
+        }
+    }
+
+    override fun DataSink<Any>.update(source: DataTree<Any>, meta: Meta, namedData: NamedData<Any>) {
+        val name = namedData.name
+        val nameWithoutPrefix = name.removeFirstOrNull(prefix)
+        if (nameWithoutPrefix != null) {
+            // put language item as is
+            put(nameWithoutPrefix, namedData.data)
+        } else if (source[name + prefix] == null) {
+            // put if language item is missing
+            put(name, namedData.data)
+        }
+    }
+}
 
 /**
  * Create a multiple sites for different languages. All sites use the same [content], but rely on different data
@@ -92,7 +119,7 @@ public val SiteContext.languagePrefix: Name
  */
 @SnarkBuilder
 public fun SiteContext.multiLanguageSite(
-    data: DataSet<*>,
+    data: DataTree<*>,
     languageMap: Map<String, Language>,
     content: HtmlSite,
 ) {
@@ -108,7 +135,7 @@ public fun SiteContext.multiLanguageSite(
         }
         site(
             prefix.parseAsName(),
-            data.branch(language.dataPath ?: prefix),
+            data.filterByType<Any>().transform(LanguageBranchAction(Name.parse(language.dataPath ?: prefix))),
             siteMeta = Laminate(languageSiteMeta, siteMeta),
             content
         )

snark-html/src/jvmMain/kotlin/space/kscience/snark/html/PageContext.kt

@@ -1,6 +1,6 @@
 package space.kscience.snark.html
 
-import space.kscience.dataforge.data.DataSet
+import space.kscience.dataforge.data.DataTree
 import space.kscience.dataforge.meta.Meta
 import space.kscience.dataforge.meta.get
 import space.kscience.dataforge.meta.string
@@ -57,4 +57,7 @@ public val PageContext.homeRef: String get() = resolvePageRef(SiteContext.INDEX_
 public val PageContext.name: Name? get() = pageMeta["name"].string?.parseAsName()
 
 
-public class PageContextWithData(private val pageContext: PageContext, public val data: DataSet<*>): PageContext by pageContext
+public class PageContextWithData(
+    private val pageContext: PageContext,
+    public val data: DataTree<*>,
+) : PageContext by pageContext

snark-html/src/jvmMain/kotlin/space/kscience/snark/html/PageFragment.kt

@@ -15,20 +15,19 @@ import space.kscience.snark.SnarkContext
 
 public fun interface PageFragment {
 
-    context(PageContextWithData)
-    public fun FlowContent.renderFragment()
+    context(PageContextWithData, FlowContent) public fun renderFragment()
 }
 
-context(PageContextWithData)
-public fun FlowContent.fragment(fragment: PageFragment): Unit{
+context(PageContextWithData, FlowContent)
+public fun fragment(fragment: PageFragment): Unit {
     with(fragment) {
         renderFragment()
     }
 }
 
 
-context(PageContextWithData)
-public fun FlowContent.fragment(data: Data<PageFragment>): Unit = runBlocking {
+context(PageContextWithData, FlowContent)
+public fun fragment(data: Data<PageFragment>): Unit = runBlocking {
     fragment(data.await())
 }
 
@@ -54,7 +53,7 @@ public val Data<*>.published: Boolean
  * Resolve a Html builder by its full name
  */
 context(SnarkContext)
-public fun DataSet<*>.resolveHtmlOrNull(name: Name): Data<PageFragment>? {
+public fun DataTree<*>.resolveHtmlOrNull(name: Name): Data<PageFragment>? {
     val resolved = (getByType<PageFragment>(name) ?: getByType<PageFragment>(name + SiteContext.INDEX_PAGE_TOKEN))
 
     return resolved?.takeIf {
@@ -63,26 +62,26 @@ public fun DataSet<*>.resolveHtmlOrNull(name: Name): Data<PageFragment>? {
 }
 
 context(SnarkContext)
-public fun DataSet<*>.resolveHtmlOrNull(name: String): Data<PageFragment>? = resolveHtmlOrNull(name.parseAsName())
+public fun DataTree<*>.resolveHtmlOrNull(name: String): Data<PageFragment>? = resolveHtmlOrNull(name.parseAsName())
 
 context(SnarkContext)
-public fun DataSet<*>.resolveHtml(name: String): Data<PageFragment> = resolveHtmlOrNull(name)
+public fun DataTree<*>.resolveHtml(name: String): Data<PageFragment> = resolveHtmlOrNull(name)
     ?: error("Html fragment with name $name is not resolved")
 
 /**
  * Find all Html blocks using given name/meta filter
  */
 context(SnarkContext)
-public fun DataSet<*>.resolveAllHtml(
+public fun DataTree<*>.resolveAllHtml(
     predicate: (name: Name, meta: Meta) -> Boolean,
-): Map<Name, Data<PageFragment>> = filterByType<PageFragment> { name, meta ->
+): Map<Name, Data<PageFragment>> = filterByType<PageFragment> { name, meta, _ ->
     predicate(name, meta)
             && meta["published"].string != "false"
     //TODO add language confirmation
 }.asSequence().associate { it.name to it.data }
 
 context(SnarkContext)
-public fun DataSet<*>.findHtmlByContentType(
+public fun DataTree<*>.findHtmlByContentType(
     contentType: String,
     baseName: Name = Name.EMPTY,
 ): Map<Name, Data<PageFragment>> = resolveAllHtml { name, meta ->

snark-html/src/jvmMain/kotlin/space/kscience/snark/html/ParseAction.kt

@@ -0,0 +1,50 @@
+package space.kscience.snark.html
+
+import space.kscience.dataforge.actions.AbstractAction
+import space.kscience.dataforge.data.*
+import space.kscience.dataforge.io.Binary
+import space.kscience.dataforge.io.toByteArray
+import space.kscience.dataforge.meta.Meta
+import space.kscience.dataforge.meta.get
+import space.kscience.snark.TextProcessor
+import kotlin.reflect.typeOf
+
+public class ParseAction(private val snarkHtml: SnarkHtml) :
+    AbstractAction<Binary, PageFragment>(typeOf<PageFragment>()) {
+
+    private fun parseOne(data: NamedData<Binary>, actionMeta: Meta): NamedData<PageFragment>? = with(snarkHtml) {
+        val contentType = getContentType(data.name, data.meta)
+
+        val parser = snark.readers.values.filterIsInstance<SnarkHtmlReader>().filter { parser ->
+            contentType in parser.types
+        }.maxByOrNull {
+            it.priority
+        }
+
+        //ignore data for which parser is not found
+        if (parser != null) {
+            val preprocessor = meta[TextProcessor.TEXT_PREPROCESSOR_KEY]?.let { snark.preprocessor(it) }
+            data.transform {
+                if (preprocessor == null) {
+                    parser.readFrom(it)
+                } else {
+                    //TODO provide encoding
+                    val string = it.toByteArray().decodeToString()
+                    parser.readFrom(preprocessor.process(string))
+                }
+            }.named(data.name)
+        } else {
+            null
+        }
+    }
+
+    override fun DataSink<PageFragment>.generate(data: DataTree<Binary>, meta: Meta) {
+        data.forEach {
+            parseOne(it, meta)?.let { put(it) }
+        }
+    }
+
+    override fun DataSink<PageFragment>.update(source: DataTree<Binary>, meta: Meta, namedData: NamedData<Binary>) {
+        parseOne(namedData,meta)?.let { put(it) }
+    }
+}

snark-html/src/jvmMain/kotlin/space/kscience/snark/html/SiteContext.kt

@@ -8,8 +8,6 @@ import space.kscience.dataforge.meta.string
 import space.kscience.dataforge.names.*
 import space.kscience.snark.SnarkBuilder
 import space.kscience.snark.SnarkContext
-import kotlin.reflect.full.isSubtypeOf
-import kotlin.reflect.typeOf
 
 
 /**
@@ -45,7 +43,7 @@ public interface SiteContext : SnarkContext {
     @SnarkBuilder
     public fun page(
         route: Name,
-        data: DataSet<*>,
+        data: DataTree<*>?,
         pageMeta: Meta = Meta.EMPTY,
         content: HtmlPage,
     )
@@ -56,7 +54,7 @@ public interface SiteContext : SnarkContext {
     @SnarkBuilder
     public fun route(
         route: Name,
-        data: DataSet<*>,
+        data: DataTree<*>?,
         siteMeta: Meta = Meta.EMPTY,
         content: HtmlSite,
     )
@@ -68,7 +66,7 @@ public interface SiteContext : SnarkContext {
     @SnarkBuilder
     public fun site(
         route: Name,
-        data: DataSet<*>,
+        data: DataTree<*>?,
         siteMeta: Meta = Meta.EMPTY,
         content: HtmlSite,
     )
@@ -81,27 +79,21 @@ public interface SiteContext : SnarkContext {
     }
 }
 
-public fun SiteContext.static(dataSet: DataSet<Binary>, prefix: Name = Name.EMPTY) {
+public fun SiteContext.static(dataSet: DataTree<Binary>, prefix: Name = Name.EMPTY) {
     dataSet.forEach { (name, data) ->
         static(prefix + name, data)
     }
 }
 
-
-public fun SiteContext.static(dataSet: DataSet<*>, branch: String, prefix: String = branch) {
+public fun SiteContext.static(dataSet: DataTree<*>, branch: String, prefix: String = branch) {
     val branchName = branch.parseAsName()
     val prefixName = prefix.parseAsName()
-    val binaryType = typeOf<Binary>()
-    dataSet.forEach { (name, data) ->
-        @Suppress("UNCHECKED_CAST")
-        if (name.startsWith(branchName) && data.type.isSubtypeOf(binaryType)) {
-            static(prefixName + name, data as Data<Binary>)
-        }
+    dataSet.branch(branchName)?.filterByType<Binary>()?.forEach {
+        static(prefixName + it.name, it.data)
     }
 }
 
 
-
 context(SiteContext)
 public val site: SiteContext
     get() = this@SiteContext
@@ -110,7 +102,7 @@ public val site: SiteContext
 /**
  * A wrapper for site context that allows convenient site building experience
  */
-public class SiteContextWithData(private val site: SiteContext, public val siteData: DataSet<*>) : SiteContext by site
+public class SiteContextWithData(private val site: SiteContext, public val siteData: DataTree<*>) : SiteContext by site
 
 
 @SnarkBuilder
@@ -127,23 +119,23 @@ public fun SiteContextWithData.page(
 @SnarkBuilder
 public fun SiteContextWithData.route(
     route: String,
-    data: DataSet<*> = siteData,
+    data: DataTree<*>? = siteData,
     siteMeta: Meta = Meta.EMPTY,
     content: HtmlSite,
-): Unit = route(route.parseAsName(), data, siteMeta,content)
+): Unit = route(route.parseAsName(), data, siteMeta, content)
 
 @SnarkBuilder
 public fun SiteContextWithData.site(
     route: String,
-    data: DataSet<*> = siteData,
+    data: DataTree<*>? = siteData,
     siteMeta: Meta = Meta.EMPTY,
     content: HtmlSite,
-): Unit = site(route.parseAsName(), data, siteMeta,content)
+): Unit = site(route.parseAsName(), data, siteMeta, content)
 
 /**
  * Render all pages and sites found in the data
  */
-public suspend fun SiteContext.renderPages(data: DataSet<*>): Unit {
+public suspend fun SiteContext.renderPages(data: DataTree<*>): Unit {
 
     // Render all sub-sites
     data.filterByType<HtmlSite>().forEach { siteData: NamedData<HtmlSite> ->
@@ -151,7 +143,7 @@ public suspend fun SiteContext.renderPages(data: DataSet<*>): Unit {
         val dataPrefix = siteData.meta["site.dataPath"].string?.asName() ?: Name.EMPTY
         site(
             route = siteData.meta["site.route"].string?.asName() ?: siteData.name,
-            data.branch(dataPrefix),
+            data.branch(dataPrefix) ?: DataTree.EMPTY,
             siteMeta = siteData.meta,
             siteData.await()
         )
@@ -162,7 +154,7 @@ public suspend fun SiteContext.renderPages(data: DataSet<*>): Unit {
         val dataPrefix = pageData.meta["page.dataPath"].string?.asName() ?: Name.EMPTY
         page(
             route = pageData.meta["page.route"].string?.asName() ?: pageData.name,
-            data.branch(dataPrefix),
+            data.branch(dataPrefix) ?: DataTree.EMPTY,
             pageMeta = pageData.meta,
             pageData.await()
         )

snark-html/src/jvmMain/kotlin/space/kscience/snark/html/SnarkHtml.kt

@@ -6,16 +6,18 @@ import io.ktor.http.ContentType
 import kotlinx.coroutines.CoroutineScope
 import kotlinx.io.readByteArray
 import space.kscience.dataforge.actions.Action
-import space.kscience.dataforge.actions.mapping
+import space.kscience.dataforge.actions.transform
 import space.kscience.dataforge.context.Context
 import space.kscience.dataforge.context.PluginFactory
 import space.kscience.dataforge.context.PluginTag
 import space.kscience.dataforge.data.*
-import space.kscience.dataforge.io.*
+import space.kscience.dataforge.io.Binary
+import space.kscience.dataforge.io.IOPlugin
+import space.kscience.dataforge.io.IOReader
+import space.kscience.dataforge.io.JsonMetaFormat
 import space.kscience.dataforge.io.yaml.YamlMetaFormat
 import space.kscience.dataforge.io.yaml.YamlPlugin
 import space.kscience.dataforge.meta.Meta
-import space.kscience.dataforge.meta.get
 import space.kscience.dataforge.meta.set
 import space.kscience.dataforge.meta.string
 import space.kscience.dataforge.misc.DFExperimental
@@ -26,24 +28,19 @@ import space.kscience.dataforge.workspace.*
 import space.kscience.snark.ReWrapAction
 import space.kscience.snark.Snark
 import space.kscience.snark.SnarkReader
-import space.kscience.snark.TextProcessor
 import java.net.URLConnection
 import kotlin.io.path.Path
 import kotlin.io.path.extension
 
 
-public fun <T : Any, R : Any> DataSet<T>.transform(action: Action<T, R>, meta: Meta = Meta.EMPTY): DataSet<R> =
+public fun <T : Any, R : Any> DataTree<T>.transform(action: Action<T, R>, meta: Meta = Meta.EMPTY): DataTree<R> =
     action.execute(this, meta)
 
-public fun <T : Any> DataSetBuilder<T>.fill(dataSet: DataSet<T>) {
-    node(Name.EMPTY, dataSet)
-}
-
 /**
  * A plugin used for rendering a [DataTree] as HTML
  */
 public class SnarkHtml : WorkspacePlugin() {
-    private val snark by require(Snark)
+    public val snark: Snark by require(Snark)
     private val yaml by require(YamlPlugin)
     public val io: IOPlugin get() = snark.io
 
@@ -60,61 +57,30 @@ public class SnarkHtml : WorkspacePlugin() {
         else -> super.content(target)
     }
 
-    private fun getContentType(name: Name, meta: Meta): String = meta[CONTENT_TYPE_KEY].string ?: run {
+    internal fun getContentType(name: Name, meta: Meta): String = meta[CONTENT_TYPE_KEY].string ?: run {
         val filePath = meta[FileData.FILE_PATH_KEY]?.string ?: name.toString()
         URLConnection.guessContentTypeFromName(filePath) ?: Path(filePath).extension
     }
 
 
-    public val prepareHeaderAction: Action<Any, Any> = ReWrapAction.removeExtensions("html", "md") { name ->
+    public val prepareHeaderAction: ReWrapAction<Any> = ReWrapAction.removeExtensions<Any>("html", "md") { name ->
         val contentType = getContentType(name, this)
         set(CONTENT_TYPE_KEY, contentType)
     }
 
-    public val parseAction: Action<Any, Any> = Action.mapping {
-        val contentType = getContentType(name, meta)
+    public val removeIndexAction: ReWrapAction<Any> = ReWrapAction.removeIndex<Any>()
 
-        val parser = snark.readers.values.filter { parser ->
-            contentType in parser.types
-        }.maxByOrNull {
-            it.priority
-        }
-        //pass data for which parser is not found
-        if (parser == null) {
-            result { it }
-        } else {
-
-            val preprocessor = meta[TextProcessor.TEXT_PREPROCESSOR_KEY]?.let { snark.preprocessor(it) }
-            result {
-
-                when (it) {
-                    is CharSequence -> {
-                        val string = it.toString()
-                        val preprocessed = preprocessor?.process(string) ?: string
-                        parser.readFrom(preprocessed)
-                    }
-
-                    is Binary -> {
-                        if (preprocessor == null) {
-                            parser.readFrom(it)
-                        } else {
-                            //TODO provide encoding
-                            val string = it.toByteArray().decodeToString()
-                            parser.readFrom(preprocessor.process(string))
-                        }
-                    }
-
-                    // bypass for non textual-data
-                    else -> it
-                }
-            }
-        }
-    }
+    public val parseAction: Action<Binary, PageFragment> = ParseAction(this)
+    private val allDataNotNull: DataSelector<Any>
+        get() = DataSelector { workspace, _ -> workspace.data.filterByType() }
 
     public val parse: TaskReference<Any> by task<Any>({
         description = "Parse all data for which reader is resolved"
     }) {
-        fill(from(allData).transform(parseAction, taskMeta))
+        //put all data
+        putAll(from(allDataNotNull))
+        //override parsed data
+        putAll(from(allDataNotNull).filterByType<Binary>().transform(parseAction))
     }
 
 
@@ -136,15 +102,18 @@ public class SnarkHtml : WorkspacePlugin() {
 
 
 public fun SnarkHtml.readSiteData(
-    binaries: DataSource<Binary>,
+    binaries: DataTree<Binary>,
     meta: Meta = Meta.EMPTY,
-): DataSet<Any> = binaries.transform(parseAction, meta)
+): DataTree<Any> = ObservableDataTree(context) {
+    //put all binaries
+    putAll(binaries)
+    //override ones which could be parsed
+    putAll(binaries.transform(parseAction, meta))
+}.transform(prepareHeaderAction, meta).transform(removeIndexAction, meta)
 
 
 public fun SnarkHtml.readSiteData(
     coroutineScope: CoroutineScope,
     meta: Meta = Meta.EMPTY,
-    builder: context(IOPlugin)  DataSourceBuilder<Any>.() -> Unit,
-): DataSet<Any> = DataSource(coroutineScope) { builder(io, this) }
-    .transform(prepareHeaderAction, meta)
-    .transform(parseAction, meta)
+    builder: DataSink<Binary>.() -> Unit,
+): DataTree<Any> = readSiteData(ObservableDataTree(coroutineScope) { builder() }, meta)

snark-html/src/jvmMain/kotlin/space/kscience/snark/html/readers.kt

@@ -8,10 +8,11 @@ import org.intellij.markdown.flavours.commonmark.CommonMarkFlavourDescriptor
 import org.intellij.markdown.html.HtmlGenerator
 import org.intellij.markdown.parser.MarkdownParser
 import space.kscience.snark.SnarkReader
-import kotlin.reflect.KType
-import kotlin.reflect.typeOf
 
-public object HtmlReader : SnarkReader<PageFragment> {
+
+public interface SnarkHtmlReader: SnarkReader<PageFragment>
+
+public object HtmlReader : SnarkHtmlReader {
     override val types: Set<String> = setOf("html")
 
     override fun readFrom(source: String): PageFragment = PageFragment {
@@ -21,12 +22,9 @@ public object HtmlReader : SnarkReader<PageFragment> {
     }
 
     override fun readFrom(source: Source): PageFragment = readFrom(source.readString())
-    override val type: KType = typeOf<PageFragment>()
 }
 
-public object MarkdownReader : SnarkReader<PageFragment> {
-    override val type: KType = typeOf<PageFragment>()
-
+public object MarkdownReader : SnarkHtmlReader {
     override val types: Set<String> = setOf("text/markdown", "md", "markdown")
 
     override fun readFrom(source: String): PageFragment = PageFragment {

snark-html/src/jvmMain/kotlin/space/kscience/snark/html/static/StaticSiteContext.kt

@@ -100,7 +100,7 @@ internal class StaticSiteContext(
         )
     }
 
-    override fun page(route: Name, data: DataSet<Any>, pageMeta: Meta, content: HtmlPage) {
+    override fun page(route: Name, data: DataTree<*>?, pageMeta: Meta, content: HtmlPage) {
 
 
         val modifiedPageMeta = pageMeta.toMutableMeta().apply {
@@ -119,7 +119,7 @@ internal class StaticSiteContext(
         newPath.writeText(HtmlPage.createHtmlString(pageContext, data, content))
     }
 
-    override fun route(route: Name, data: DataSet<*>, siteMeta: Meta, content: HtmlSite) {
+    override fun route(route: Name, data: DataTree<*>?, siteMeta: Meta, content: HtmlSite) {
         val siteContextWithData = SiteContextWithData(
             StaticSiteContext(
                 siteMeta = Laminate(siteMeta, this@StaticSiteContext.siteMeta),
@@ -127,7 +127,7 @@ internal class StaticSiteContext(
                 route = route,
                 outputPath = outputPath.resolve(route.toWebPath())
             ),
-            data
+            data ?: DataTree.EMPTY
         )
         with(content) {
             with(siteContextWithData) {
@@ -136,7 +136,7 @@ internal class StaticSiteContext(
         }
     }
 
-    override fun site(route: Name, data: DataSet<Any>, siteMeta: Meta, content: HtmlSite) {
+    override fun site(route: Name, data: DataTree<*>?, siteMeta: Meta, content: HtmlSite) {
         val siteContextWithData = SiteContextWithData(
             StaticSiteContext(
                 siteMeta = Laminate(siteMeta, this@StaticSiteContext.siteMeta),
@@ -144,7 +144,7 @@ internal class StaticSiteContext(
                 route = Name.EMPTY,
                 outputPath = outputPath.resolve(route.toWebPath())
             ),
-            data
+            data ?: DataTree.EMPTY
         )
         with(content) {
             with(siteContextWithData) {
@@ -162,17 +162,17 @@ internal class StaticSiteContext(
  */
 @Suppress("UnusedReceiverParameter")
 public suspend fun SnarkHtml.staticSite(
-    data: DataSet<*>,
+    data: DataTree<*>?,
     outputPath: Path,
     siteUrl: String = outputPath.absolutePathString().replace("\\", "/"),
-    siteMeta: Meta = data.meta,
+    siteMeta: Meta = data?.meta ?: Meta.EMPTY,
     content: HtmlSite,
 ) {
     val siteContextWithData = SiteContextWithData(
         StaticSiteContext(siteMeta, siteUrl, Name.EMPTY, outputPath),
-        data
+        data ?: DataTree.EMPTY
     )
-    with(content){
+    with(content) {
         with(siteContextWithData) {
             renderSite()
         }

snark-ktor/src/jvmMain/kotlin/space/kscience/snark/ktor/KtorSiteContext.kt

@@ -15,8 +15,9 @@ import space.kscience.dataforge.context.ContextAware
 import space.kscience.dataforge.context.error
 import space.kscience.dataforge.context.logger
 import space.kscience.dataforge.data.Data
-import space.kscience.dataforge.data.DataSet
+import space.kscience.dataforge.data.DataTree
 import space.kscience.dataforge.data.await
+import space.kscience.dataforge.data.meta
 import space.kscience.dataforge.io.Binary
 import space.kscience.dataforge.io.toByteArray
 import space.kscience.dataforge.meta.Laminate
@@ -106,7 +107,7 @@ public class KtorSiteContext(
         }
     }
 
-    override fun page(route: Name, data: DataSet<*>, pageMeta: Meta, content: HtmlPage) {
+    override fun page(route: Name, data: DataTree<*>?, pageMeta: Meta, content: HtmlPage) {
         ktorRoute.get(route.toWebPath()) {
             val request = call.request
             //substitute host for url for backwards calls
@@ -129,7 +130,7 @@ public class KtorSiteContext(
         }
     }
 
-    override fun route(route: Name, data: DataSet<*>, siteMeta: Meta, content: HtmlSite) {
+    override fun route(route: Name, data: DataTree<*>?, siteMeta: Meta, content: HtmlSite) {
         val siteContext = SiteContextWithData(
             KtorSiteContext(
                 context,
@@ -138,7 +139,7 @@ public class KtorSiteContext(
                 route = route,
                 ktorRoute = ktorRoute.createRouteFromPath(route.toWebPath())
             ),
-            data
+            data ?: DataTree.EMPTY
         )
         with(content) {
             with(siteContext) {
@@ -147,7 +148,7 @@ public class KtorSiteContext(
         }
     }
 
-    override fun site(route: Name, data: DataSet<*>, siteMeta: Meta, content: HtmlSite) {
+    override fun site(route: Name, data: DataTree<*>?, siteMeta: Meta, content: HtmlSite) {
         val siteContext = SiteContextWithData(
             KtorSiteContext(
                 context,
@@ -156,7 +157,7 @@ public class KtorSiteContext(
                 route = Name.EMPTY,
                 ktorRoute = ktorRoute.createRouteFromPath(route.toWebPath())
             ),
-            data
+            data?: DataTree.EMPTY
         )
         with(content) {
             with(siteContext) {
@@ -169,14 +170,14 @@ public class KtorSiteContext(
 
 public fun Route.site(
     context: Context,
-    data: DataSet<*>,
+    data: DataTree<*>?,
     baseUrl: String = "",
-    siteMeta: Meta = data.meta,
+    siteMeta: Meta = data?.meta ?: Meta.EMPTY,
     content: HtmlSite,
 ) {
     val siteContext = SiteContextWithData(
         KtorSiteContext(context, siteMeta, baseUrl, route = Name.EMPTY, this@Route),
-        data
+        data?: DataTree.EMPTY
     )
     with(content) {
         with(siteContext) {