Research + Team

build.gradle.kts

@@ -3,6 +3,7 @@ import ru.mipt.npm.gradle.KScienceVersions
 plugins {
     id("ru.mipt.npm.gradle.project")
     id("ru.mipt.npm.gradle.jvm")
+    id("org.hidetake.ssh") version "2.10.1"
     application
 }
 
@@ -21,7 +22,7 @@ application {
 }
 
 
-val dataforgeVersion by extra("0.6.0-dev-7")
+val dataforgeVersion by extra("0.6.0-dev-9")
 val ktorVersion = KScienceVersions.ktorVersion
 
 dependencies {
@@ -42,11 +43,11 @@ dependencies {
 
 kotlin {
     explicitApi = org.jetbrains.kotlin.gradle.dsl.ExplicitApiMode.Disabled
-    sourceSets.all {
-        languageSettings {
-            languageVersion  = "1.7"
-            apiVersion = "1.7"
-        }
+}
+
+tasks.withType<org.jetbrains.kotlin.gradle.tasks.KotlinCompile>{
+    kotlinOptions{
+        freeCompilerArgs = freeCompilerArgs + "-Xcontext-receivers"
     }
 }
 

data/home/content/people/Nozik.md

@@ -0,0 +1,75 @@
+---
+type: team
+title: Alexander Nozik
+id: nozik
+order: 1
+image: images/people/nozik_2.png
+language: en
+---
+
+
+* PhD in particle physics.
+* Director of [Scientific Programming Centre](/).
+* Senior researcher at https://npm.mipt.ru.
+* (ex) Team lead at [JetBrains Research](https://research.jetbrains.org/groups/npm/).
+* Google developer expert in Kotlin.
+
+<blockquote>
+<ul>
+  <li> I am against the war in Ukraine. It must be stopped. </li> 
+  <li> I did not vote for Putin. I do not think that Putin is the legitimate president of Russia. </li>
+  <li> I hope for the better future. I publicly express my position in <a href="https://twitter.com/noraltavir">my twitter</a>.</li>
+  <li> I do not speak for the whole laboratory. </li>
+</ul>
+</blockquote>  
+
+## About me
+
+When I finished the school, I had two choices: I liked physics and I liked programming. In the end I decided that physics is more interesting. Now I am doing programming in physics. Currently I am doing software and physical data analysis for a number of experiments in particle physics (Troitsk nu-mass, IAXO and other experiments). I am leading development of scientific libraries and leading the development of new simulation software and models.
+
+The laboratory at MIPT, we created in the last years includes young passionate researchers in different fields connected to particle and nuclear physics methods. The aim of the laboratory is to prepare the new generation of researchers, familiar with modern scientific tools and able to develop new ones.
+
+## Social
+
+* [ResearchGate profile](https://www.researchgate.net/profile/Alexander_Nozik)
+* [Twitter](https://twitter.com/noraltavir)
+* [Telegram](https://t.me/noraltavir)
+
+## Education
+
+* Lyceum “2nd school” 1997 - 2002
+* MIPT (bachelor, problems of physics and energetics) 2002 - 2006. Finished with honors.
+* MIPT (master, fundamental particles and cosmology) 2006 - 2008
+* [PhD in physics](https://www.researchgate.net/publication/260058278_Rezultaty_obrabotki_dannyh_eksperimenta_Troick_nu-mass_po_pramomu_izmereniu_massy_elektronnogo_nejtrino), INR RAS 2012 .
+
+## Work experience
+
+* INR RAS Laboratory assistant, 2004–2008
+* INR RAS Researcher, 2008–2012
+* INR RAS Senior researcher, 2012–present
+* MIPT Assistant, 2013–2018, The department of general physics laboratory practice and seminars. Special courses on programming and statistics.
+* MIPT Associate professor, 2019–present
+* MIPT Senior researcher, 2019–present, Deputy head of the nuclear physics methods laboratory.
+* JetBrains Research, 2020–present, Head of the research group.
+
+## The code
+
+* Private GitHub account: https://github.com/altavir
+* Laboratory / centre GitHub account: https://github.com/mipt-npm
+
+## Achievements
+
+* The best (until 2020) limit on electron neutrino mass: https://arxiv.org/abs/1108.5034
+* The best limit on sterile neutrino contribution to the electron neutrino with masses up to 2 keV: https://arxiv.org/abs/1307.5687, https://arxiv.org/abs/1703.10779
+* The data acquisition and analysis software in Troitsk nu-mass experiment.
+* The Reactor model for TGF generation in thunderclouds.
+* KMath library: https://github.com/mipt-npm/kmath.
+
+## Publications
+* ORCID: https://orcid.org/0000-0001-9075-0080
+* Scopus ID: 24071435300
+* Web of Science ID: H-3844-2019
+
+## Keywords
+
+Particle physics, Neutrino, Data analysis, Mathematical statistics, Scientific programming, Java, Kotlin

data/home/content/people/Nozik[info].md

@@ -0,0 +1,5 @@
+* PhD in particle physics.
+* Director of [Scientific Programming Centre](/).
+* Senior researcher at https://npm.mipt.ru.
+* (ex) Team lead at [JetBrains Research](https://research.jetbrains.org/groups/npm/).
+* Google developer expert in Kotlin.

data/home/content/projects/kmath.md

@@ -0,0 +1,10 @@
+---
+type: project
+title: KMath
+order: 2
+language: en
+---
+
+An experimental Kotlin library for mathematical operations, built on the principle of context-oriented programming using mathematical abstractions.
+
+[Repository and documentation](https://github.com/altavir/kmath)

data/home/content/projects/kmath[info].md

@@ -0,0 +1 @@
+An experimental Kotlin library for mathematical operations, built on the principle of context-oriented programming using mathematical abstractions.

gradle.properties

@@ -1,3 +1,3 @@
 kotlin.code.style=official
 
-toolsVersion=0.11.4-kotlin-1.6.20
+toolsVersion=0.11.5-kotlin-1.6.21

src/main/kotlin/ru/mipt/spc/master.kt

@@ -268,7 +268,7 @@ internal fun Application.spcMaster(context: Context, dataPath: Path, prefix: Str
 
     val snark = context.fetch(SnarkPlugin)
 
-    val magProgPageContext = snark.parse(prefix, dataPath.resolve("content"))
+    val magProgPageContext: PageContext = snark.parse(prefix, dataPath.resolve("content"))
 
     routing {
         route(prefix) {

src/main/kotlin/ru/mipt/spc/spcHome.kt

@@ -11,166 +11,20 @@ import io.ktor.server.routing.*
 import kotlinx.html.*
 import space.kscience.dataforge.context.Context
 import space.kscience.dataforge.context.fetch
+import space.kscience.dataforge.meta.Meta
 import space.kscience.dataforge.meta.get
 import space.kscience.dataforge.meta.string
+import space.kscience.dataforge.names.Name
 import space.kscience.dataforge.names.parseAsName
 import space.kscience.snark.*
 import java.nio.file.Path
 
 
-private const val SPC_TITLE = "Scientific Programming Centre"
-
-context(PageContext) private fun HTML.spcHead(title: String = SPC_TITLE) {
-    head {
-        title {
-            +title
-        }
-        meta {
-            charset = "utf-8"
-        }
-        meta {
-            name = "viewport"
-            content = "width=device-width, initial-scale=1, user-scalable=no"
-        }
-        link(rel = "stylesheet", href = resolveRef("assets/css/main.css"))
-        noScript {
-            link(rel = "stylesheet", href = resolveRef("assets/css/noscript.css"))
-        }
-    }
-}
-
-context(PageContext) private fun FlowContent.spcHomeMenu() {
-    nav {
-        id = "menu"
-        ul("links") {
-            li {
-                a {
-                    href = homeRef
-                    +"""Home"""
-                }
-            }
-            li {
-                a {
-                    href = resolveRef("magprog")
-                    +"""Master"""
-                }
-            }
-            li {
-                a {
-                    href = resolveRef("research")
-                    +"""Research"""
-                }
-            }
-            li {
-                a {
-                    href = resolveRef("consulting")
-                    +"""Consulting"""
-                }
-            }
-            li {
-                a {
-                    href = resolveRef("team")
-                    +"""Team"""
-                }
-            }
-        }
-//        ul("actions stacked") {
-//            li {
-//                a(classes = "button primary fit") {
-//                    href = "#"
-//                    +"""Get Started"""
-//                }
-//            }
-//            li {
-//                a(classes = "button fit") {
-//                    href = "#"
-//                    +"""Log In"""
-//                }
-//            }
-//        }
-    }
-}
-
-context(PageContext) private fun FlowContent.spcFooter() {
-    footer {
-        id = "footer"
-        div("inner") {
-            ul("icons") {
-//                li {
-//                    a(classes = "icon brands alt fa-twitter") {
-//                        href = "#"
-//                        span("label") { +"""Twitter""" }
-//                    }
-//                }
-//                li {
-//                    a(classes = "icon brands alt fa-facebook-f") {
-//                        href = "#"
-//                        span("label") { +"""Facebook""" }
-//                    }
-//                }
-//                li {
-//                    a(classes = "icon brands alt fa-instagram") {
-//                        href = "#"
-//                        span("label") { +"""Instagram""" }
-//                    }
-//                }
-                li {
-                    a(classes = "icon brands alt fa-github") {
-                        href = "https://github.com/mipt-npm"
-                        span("label") { +"""GitHub""" }
-                    }
-                }
-//                li {
-//                    a(classes = "icon brands alt fa-linkedin-in") {
-//                        href = "#"
-//                        span("label") { +"""LinkedIn""" }
-//                    }
-//                }
-            }
-            ul("copyright") {
-                li { +"""SPC""" }
-                li {
-                    +"""Design:"""
-                    a {
-                        href = "https://html5up.net"
-                        +"""HTML5 UP"""
-                    }
-                }
-            }
-        }
-    }
-}
-
-context(PageContext) private fun FlowContent.wrapper(contentBody: FlowContent.() -> Unit) {
-    div {
-        id = "wrapper"
-        // Header
-        header("alt") {
-            id = "header"
-            a(classes = "logo") {
-                href = homeRef
-                strong { +"""SPC""" }
-                span { +"""Scientific Programming Centre""" }
-            }
-            nav {
-                a {
-                    href = "#menu"
-                    +"""Menu"""
-                }
-            }
-        }
-        // Menu
-        spcHomeMenu()
-        //Body
-        contentBody()
-        // Footer
-        spcFooter()
-    }
-}
-
-
-context(PageContext) private fun HTML.spcPage(data: HtmlData) {
-    val title = data.meta["title"].string ?: SPC_TITLE
+context(PageContext) internal fun HTML.spcPageContent(
+    meta: Meta,
+    title: String = meta["title"].string ?: SPC_TITLE,
+    fragment: FlowContent.() -> Unit,
+) {
     spcHead(title)
     body("is-preload") {
         wrapper {
@@ -183,7 +37,7 @@ context(PageContext) private fun HTML.spcPage(data: HtmlData) {
                         header("major") {
                             h1 { +title }
                         }
-                        data.meta["image"].string?.let { imagePath ->
+                        meta["image"].string?.let { imagePath ->
                             span("image main") {
                                 img {
                                     src = resolveRef(imagePath)
@@ -191,7 +45,7 @@ context(PageContext) private fun HTML.spcPage(data: HtmlData) {
                                 }
                             }
                         }
-                        htmlData(data)
+                        fragment()
                     }
                 }
             }
@@ -201,26 +55,41 @@ context(PageContext) private fun HTML.spcPage(data: HtmlData) {
     }
 }
 
-context(PageContext) private fun Route.spcPage(subRoute: String, data: HtmlData) {
+
+context(PageContext) internal fun Route.spcPage(subRoute: String, meta: Meta, fragment: FlowContent.() -> Unit) {
     get(subRoute) {
         withRequest(call.request) {
             call.respondHtml {
-                spcPage(data)
+                spcPageContent(meta, fragment = fragment)
             }
         }
     }
 }
 
-context(PageContext) private fun Route.spcPage(subRoute: String, dataPath: String = subRoute) {
-    val data = resolveHtml(dataPath.parseAsName())
+context(PageContext) internal fun Route.spcPage(
+    subRoute: String,
+    dataPath: Name = subRoute.parseAsName(),
+    more: FlowContent.() -> Unit = {},
+) {
+    val data = resolveHtml(dataPath)
     if (data != null) {
-        spcPage(subRoute, data)
+        spcPage(subRoute, data.meta) {
+            htmlData(data)
+            more()
+        }
     } else {
         application.log.error("Content for page with path $dataPath not found")
     }
 
 }
 
+context(PageContext) internal fun Route.spcPage(
+    name: Name,
+    more: FlowContent.() -> Unit = {},
+) {
+    spcPage(name.tokens.joinToString("/"), name, more)
+}
+
 context(PageContext) private fun HTML.spcHome() {
     spcHead()
     body("is-preload") {
@@ -420,8 +289,9 @@ internal fun Application.spcHome(context: Context, rootPath: Path, prefix: Strin
                 }
 
                 spcPage("consulting")
-                spcPage("research")
-                spcPage("team")
+
+                spcLanding("team") { _, m -> m["type"].string == "team" }
+                spcLanding("research"){ _, m -> m["type"].string == "project" }
             }
         }
     }

src/main/kotlin/ru/mipt/spc/spcLanding.kt

@@ -0,0 +1,119 @@
+package ru.mipt.spc
+
+import html5up.forty.fortyScripts
+import io.ktor.server.application.call
+import io.ktor.server.html.respondHtml
+import io.ktor.server.routing.Route
+import io.ktor.server.routing.get
+import kotlinx.html.*
+import space.kscience.dataforge.meta.Meta
+import space.kscience.dataforge.meta.get
+import space.kscience.dataforge.meta.string
+import space.kscience.dataforge.names.Name
+import space.kscience.dataforge.names.parseAsName
+import space.kscience.dataforge.names.withIndex
+import space.kscience.snark.*
+
+context(PageContext) private fun FlowContent.spcTeamContent(
+    landing: HtmlData,
+    content: Map<Name, HtmlData>,
+) {
+    // Banner
+    // Note: The "styleN" class below should match that of the header element.
+    section("style2") {
+        id = "banner"
+        div("inner") {
+            span("image") {
+                img {
+                    src = "images/pic07.jpg"
+                    alt = ""
+                }
+            }
+            header("major") {
+                h1 { +(landing.meta["title"].string ?: "???") }
+            }
+            div("content") {
+                htmlData(landing)
+            }
+        }
+    }
+
+    // Main
+    div {
+        id = "main"
+        content.forEach { (name, data) ->
+            val ref = resolveRef(name)
+
+            section("spotlights") {
+                id = data.meta["id"].string ?: name.toString()
+                section {
+                    data.meta["image"].string?.let { imagePath ->
+                        a(classes = "image") {
+                            href = ref
+                            img {
+                                src = resolveRef(imagePath)
+                                alt = name.toString()
+                                attributes["data-position"] = "center center"
+                            }
+                        }
+                    }
+                    div("content") {
+                        div("inner") {
+                            header("major") {
+                                h3 { +(data.meta["title"].string ?: "???") }
+                            }
+                            resolveHtml(name.withIndex("info"))?.let {
+                                htmlData(it)
+                            }
+                            ul("actions") {
+                                li {
+                                    a(classes = "button") {
+                                        href = ref
+                                        +"""Learn more"""
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
+}
+
+
+context(PageContext) internal fun Route.spcLanding(
+    name: String,
+    contentFilter: (Name, Meta) -> Boolean,
+) {
+    val body = resolveHtml(name.parseAsName()) ?: error("Could not find body for $name")
+    val content = resolveAllHtml(contentFilter)
+
+    val meta = body.meta
+    get(name) {
+        withRequest(call.request) {
+            call.respondHtml {
+                val title = meta["title"].string ?: SPC_TITLE
+                spcHead(title)
+                body("is-preload") {
+                    wrapper {
+                        spcTeamContent(body, content)
+                    }
+
+                    fortyScripts()
+                }
+            }
+        }
+    }
+    content.forEach { (name, contentBody) ->
+        get(name.tokens.joinToString("/")) {
+            withRequest(call.request) {
+                call.respondHtml {
+                    spcPageContent(contentBody.meta) {
+                        htmlData(contentBody)
+                    }
+                }
+            }
+        }
+    }
+}

src/main/kotlin/ru/mipt/spc/spcMisc.kt

@@ -0,0 +1,157 @@
+package ru.mipt.spc
+
+import kotlinx.html.*
+import space.kscience.snark.PageContext
+import space.kscience.snark.homeRef
+import space.kscience.snark.resolveRef
+
+
+internal const val SPC_TITLE = "Scientific Programming Centre"
+
+context(PageContext) internal fun HTML.spcHead(title: String = SPC_TITLE) {
+    head {
+        title {
+            +title
+        }
+        meta {
+            charset = "utf-8"
+        }
+        meta {
+            name = "viewport"
+            content = "width=device-width, initial-scale=1, user-scalable=no"
+        }
+        link(rel = "stylesheet", href = resolveRef("assets/css/main.css"))
+        noScript {
+            link(rel = "stylesheet", href = resolveRef("assets/css/noscript.css"))
+        }
+    }
+}
+
+context(PageContext) internal fun FlowContent.spcHomeMenu() {
+    nav {
+        id = "menu"
+        ul("links") {
+            li {
+                a {
+                    href = homeRef
+                    +"""Home"""
+                }
+            }
+            li {
+                a {
+                    href = resolveRef("magprog")
+                    +"""Master"""
+                }
+            }
+            li {
+                a {
+                    href = resolveRef("research")
+                    +"""Research"""
+                }
+            }
+            li {
+                a {
+                    href = resolveRef("consulting")
+                    +"""Consulting"""
+                }
+            }
+            li {
+                a {
+                    href = resolveRef("team")
+                    +"""Team"""
+                }
+            }
+        }
+//        ul("actions stacked") {
+//            li {
+//                a(classes = "button primary fit") {
+//                    href = "#"
+//                    +"""Get Started"""
+//                }
+//            }
+//            li {
+//                a(classes = "button fit") {
+//                    href = "#"
+//                    +"""Log In"""
+//                }
+//            }
+//        }
+    }
+}
+
+context(PageContext) internal fun FlowContent.spcFooter() {
+    footer {
+        id = "footer"
+        div("inner") {
+            ul("icons") {
+//                li {
+//                    a(classes = "icon brands alt fa-twitter") {
+//                        href = "#"
+//                        span("label") { +"""Twitter""" }
+//                    }
+//                }
+//                li {
+//                    a(classes = "icon brands alt fa-facebook-f") {
+//                        href = "#"
+//                        span("label") { +"""Facebook""" }
+//                    }
+//                }
+//                li {
+//                    a(classes = "icon brands alt fa-instagram") {
+//                        href = "#"
+//                        span("label") { +"""Instagram""" }
+//                    }
+//                }
+                li {
+                    a(classes = "icon brands alt fa-github") {
+                        href = "https://github.com/mipt-npm"
+                        span("label") { +"""GitHub""" }
+                    }
+                }
+//                li {
+//                    a(classes = "icon brands alt fa-linkedin-in") {
+//                        href = "#"
+//                        span("label") { +"""LinkedIn""" }
+//                    }
+//                }
+            }
+            ul("copyright") {
+                li { +"""SPC""" }
+                li {
+                    +"""Design:"""
+                    a {
+                        href = "https://html5up.net"
+                        +"""HTML5 UP"""
+                    }
+                }
+            }
+        }
+    }
+}
+
+context(PageContext) internal fun FlowContent.wrapper(contentBody: FlowContent.() -> Unit) {
+    div {
+        id = "wrapper"
+        // Header
+        header("alt") {
+            id = "header"
+            a(classes = "logo") {
+                href = homeRef
+                strong { +"""SPC""" }
+                span { +"""Scientific Programming Centre""" }
+            }
+            nav {
+                a {
+                    href = "#menu"
+                    +"""Menu"""
+                }
+            }
+        }
+        // Menu
+        spcHomeMenu()
+        //Body
+        contentBody()
+        // Footer
+        spcFooter()
+    }
+}

src/main/kotlin/space/kscience/snark/DirectoryDataTree.kt

@@ -1,105 +0,0 @@
-package space.kscience.snark
-
-import kotlinx.coroutines.flow.SharedFlow
-import kotlinx.coroutines.flow.SharingStarted
-import kotlinx.coroutines.flow.flow
-import kotlinx.coroutines.flow.shareIn
-import space.kscience.dataforge.data.Data
-import space.kscience.dataforge.data.DataSource
-import space.kscience.dataforge.data.DataTree
-import space.kscience.dataforge.data.DataTreeItem
-import space.kscience.dataforge.io.IOPlugin
-import space.kscience.dataforge.io.readEnvelopeFile
-import space.kscience.dataforge.io.readMetaFile
-import space.kscience.dataforge.io.toByteArray
-import space.kscience.dataforge.meta.Meta
-import space.kscience.dataforge.meta.copy
-import space.kscience.dataforge.names.*
-import java.nio.file.Path
-import java.nio.file.StandardWatchEventKinds
-import java.nio.file.attribute.BasicFileAttributes
-import kotlin.coroutines.CoroutineContext
-import kotlin.io.path.*
-import kotlin.reflect.KType
-import kotlin.reflect.typeOf
-
-
-class DirectoryDataTree(val io: IOPlugin, val path: Path) : DataTree<ByteArray>, DataSource<ByteArray> {
-
-    override val coroutineContext: CoroutineContext get() = io.context.coroutineContext
-
-    override val dataType: KType
-        get() = typeOf<ByteArray>()
-
-    override val meta: Meta get() = try {
-        //TODO replace by readMetaFileOrNull
-        io.readMetaFile(path)
-    } catch (ise: java.lang.IllegalStateException){
-        Meta.EMPTY
-    }
-
-    private fun readFile(filePath: Path): Data<ByteArray> {
-        val envelope = io.readEnvelopeFile(filePath, readNonEnvelopes = true)
-        val meta = envelope.meta.copy {
-            META_FILE_PATH_KEY put filePath.toString()
-            META_FILE_EXTENSION_KEY put filePath.extension
-
-            val attributes = filePath.readAttributes<BasicFileAttributes>()
-            META_FILE_UPDATE_TIME_KEY put attributes.lastModifiedTime().toInstant().toString()
-            META_FILE_CREATE_TIME_KEY put attributes.creationTime().toInstant().toString()
-        }
-        return Data(meta) {
-            envelope.data?.toByteArray() ?: ByteArray(0)
-        }
-    }
-
-
-    private fun Path.toName(): Name = Name(flatMap {it.nameWithoutExtension.parseAsName().tokens})
-
-    override val items: Map<NameToken, DataTreeItem<ByteArray>>
-        get() = path.listDirectoryEntries().associate { childPath ->
-            //val fileName = childPath.fileName.nameWithoutExtension
-
-            val item: DataTreeItem<ByteArray> = if (childPath.isDirectory()) {
-                DataTreeItem.Node(DirectoryDataTree(io, childPath))
-            } else {
-                DataTreeItem.Leaf(readFile(childPath))
-            }
-
-            val name = childPath.fileName.toName()//Name.parse(fileName)
-            if (name.length == 1) {
-                name.first() to item
-            } else {
-                TODO("Segmented names are not supported")
-            }
-        }
-
-    override val updates: SharedFlow<Name> by lazy {
-        val watchService = path.fileSystem.newWatchService()
-
-        path.register(
-            watchService,
-            StandardWatchEventKinds.ENTRY_CREATE,
-            StandardWatchEventKinds.ENTRY_MODIFY,
-            StandardWatchEventKinds.ENTRY_DELETE
-        )
-
-        flow {
-            while (true) {
-                val key = watchService.take() ?: break
-                key.pollEvents().map { it.context() }.filterIsInstance<Path>().forEach {
-                    emit(it.toName())
-                }
-                key.reset()
-            }
-        }.shareIn(this, SharingStarted.Eagerly)
-    }
-
-    companion object {
-        val META_FILE_KEY = "file".asName()
-        val META_FILE_PATH_KEY = META_FILE_KEY + "path"
-        val META_FILE_EXTENSION_KEY = META_FILE_KEY + "extension"
-        val META_FILE_CREATE_TIME_KEY = META_FILE_KEY + "created"
-        val META_FILE_UPDATE_TIME_KEY = META_FILE_KEY + "update"
-    }
-}

src/main/kotlin/space/kscience/snark/HtmlData.kt

@@ -1,5 +1,6 @@
 package space.kscience.snark
 
+import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.runBlocking
 import kotlinx.html.FlowContent
 import kotlinx.html.TagConsumer
@@ -23,11 +24,11 @@ val HtmlData.language: String? get() = meta["language"].string?.lowercase()
 
 val HtmlData.order: Int? get() = meta["order"]?.int
 
-fun TagConsumer<*>.htmlData(data: HtmlData) = runBlocking {
+fun TagConsumer<*>.htmlData(data: HtmlData) = runBlocking(Dispatchers.IO) {
     data.await().invoke(this@htmlData)
 }
 
-fun FlowContent.htmlData(data: HtmlData) = runBlocking {
+fun FlowContent.htmlData(data: HtmlData) = runBlocking(Dispatchers.IO) {
     data.await().invoke(consumer)
 }
 

src/main/kotlin/space/kscience/snark/PageContext.kt

@@ -6,7 +6,6 @@ import io.ktor.server.plugins.origin
 import io.ktor.server.request.ApplicationRequest
 import io.ktor.server.request.host
 import io.ktor.server.request.port
-import space.kscience.dataforge.actions.invoke
 import space.kscience.dataforge.data.*
 import space.kscience.dataforge.meta.Meta
 import space.kscience.dataforge.meta.get
@@ -24,6 +23,8 @@ data class PageContext(val path: String, val pageMeta: Meta, val data: DataSet<*
  */
 fun PageContext.resolveRef(name: String): String = "${path.removeSuffix("/")}/$name"
 
+fun PageContext.resolveRef(name: Name): String = "${path.removeSuffix("/")}/${name.tokens.joinToString ("/")}"
+
 /**
  * Resolve a Html builder by its full name
  */
@@ -53,9 +54,7 @@ internal val Data<*>.published: Boolean get() = meta["published"].string != "fal
 fun PageContext(rootUrl: String, data: DataSet<*>): PageContext = PageContext(rootUrl, data.meta, data)
 
 fun SnarkPlugin.parse(rootUrl: String, path: Path): PageContext {
-    val directoryDataTree = DirectoryDataTree(io, path)
-
-    val parsedData: DataSet<Any> = parseAction(directoryDataTree)
+    val parsedData: DataSet<Any> = readDirectory(path)
 
     return PageContext(rootUrl, parsedData)
 }

src/main/kotlin/space/kscience/snark/SnarkHtmlParser.kt

@@ -1,20 +1,20 @@
 package space.kscience.snark
 
 import io.ktor.http.ContentType
+import io.ktor.utils.io.core.Input
 import kotlinx.html.div
 import kotlinx.html.unsafe
-import space.kscience.dataforge.meta.Meta
 import kotlin.reflect.KType
 import kotlin.reflect.typeOf
 
-object SnarkHtmlParser:SnarkParser<HtmlFragment> {
+object SnarkHtmlParser : SnarkParser<HtmlFragment> {
     override val contentType: ContentType = ContentType.Text.Html
     override val fileExtensions: Set<String> = setOf("html")
-    override val resultType: KType = typeOf<HtmlFragment>()
+    override val type: KType = typeOf<HtmlFragment>()
 
-    override suspend fun parse(bytes: ByteArray, meta: Meta): HtmlFragment = {
-        div{
-           unsafe { +bytes.decodeToString() }
+    override fun readObject(input: Input): HtmlFragment = {
+        div {
+            unsafe { +input.readText() }
         }
     }
 }

src/main/kotlin/space/kscience/snark/SnarkJsonParser.kt

@@ -1,17 +0,0 @@
-package space.kscience.snark
-
-import io.ktor.http.ContentType
-import kotlinx.serialization.json.Json
-import space.kscience.dataforge.meta.Meta
-import space.kscience.dataforge.meta.toMeta
-import kotlin.reflect.KType
-import kotlin.reflect.typeOf
-
-object SnarkJsonParser: SnarkParser<Meta> {
-    override val contentType: ContentType = ContentType.Application.Json
-    override val fileExtensions: Set<String> = setOf("json")
-    override val resultType: KType= typeOf<Meta>()
-
-    override suspend fun parse(bytes: ByteArray, meta: Meta): Meta =
-        Json.parseToJsonElement(bytes.decodeToString()).toMeta()
-}

src/main/kotlin/space/kscience/snark/SnarkMarkdownParser.kt

@@ -1,25 +1,25 @@
 package space.kscience.snark
 
 import io.ktor.http.ContentType
+import io.ktor.utils.io.core.Input
 import kotlinx.html.div
 import kotlinx.html.unsafe
 import org.intellij.markdown.flavours.commonmark.CommonMarkFlavourDescriptor
 import org.intellij.markdown.html.HtmlGenerator
 import org.intellij.markdown.parser.MarkdownParser
-import space.kscience.dataforge.meta.Meta
 import kotlin.reflect.KType
 import kotlin.reflect.typeOf
 
 object SnarkMarkdownParser:SnarkParser<HtmlFragment> {
     override val contentType: ContentType = ContentType.Text.Html
     override val fileExtensions: Set<String> = setOf("markdown", "mdown", "mkdn", "mkd", "md")
-    override val resultType: KType = typeOf<HtmlFragment>()
+    override val type: KType = typeOf<HtmlFragment>()
 
     private val markdownFlavor = CommonMarkFlavourDescriptor()
     private val markdownParser = MarkdownParser(markdownFlavor)
 
-    override suspend fun parse(bytes: ByteArray, meta: Meta): HtmlFragment {
-        val src = bytes.decodeToString()
+    override fun readObject(input: Input): HtmlFragment  {
+        val src = input.readText()
         val parsedTree = markdownParser.buildMarkdownTreeFromString(src)
         val htmlString = HtmlGenerator(src, parsedTree, markdownFlavor).generateHtml()
 

src/main/kotlin/space/kscience/snark/SnarkPlugin.kt

@@ -1,9 +1,14 @@
 package space.kscience.snark
 
 import io.ktor.http.ContentType
-import space.kscience.dataforge.actions.Action
-import space.kscience.dataforge.actions.map
+import io.ktor.util.extension
+import io.ktor.utils.io.core.readBytes
 import space.kscience.dataforge.context.*
+import space.kscience.dataforge.data.DataTree
+import space.kscience.dataforge.io.IOReader
+import space.kscience.dataforge.io.JsonMetaFormat
+import space.kscience.dataforge.io.asBinary
+import space.kscience.dataforge.io.yaml.YamlMetaFormat
 import space.kscience.dataforge.io.yaml.YamlPlugin
 import space.kscience.dataforge.meta.Meta
 import space.kscience.dataforge.meta.get
@@ -12,20 +17,24 @@ import space.kscience.dataforge.misc.DFExperimental
 import space.kscience.dataforge.misc.Type
 import space.kscience.dataforge.names.Name
 import space.kscience.dataforge.names.asName
+import space.kscience.dataforge.workspace.FileData
+import space.kscience.dataforge.workspace.readDataDirectory
+import java.nio.file.Path
 import kotlin.reflect.KClass
 import kotlin.reflect.KType
+import kotlin.reflect.typeOf
 
 @Type(SnarkParser.TYPE)
-interface SnarkParser<R : Any> {
+interface SnarkParser<out R : Any> : IOReader<R> {
     val contentType: ContentType
 
     val fileExtensions: Set<String>
 
     val priority: Int get() = DEFAULT_PRIORITY
 
-    val resultType: KType
-
-    suspend fun parse(bytes: ByteArray, meta: Meta): R
+    suspend fun parse(bytes: ByteArray, meta: Meta): R = bytes.asBinary().read {
+        readObject(this)
+    }
 
     companion object {
         const val TYPE = "snark.parser"
@@ -33,6 +42,20 @@ interface SnarkParser<R : Any> {
     }
 }
 
+@PublishedApi
+internal class SnarkParserWrapper<R : Any>(
+    val reader: IOReader<R>,
+    override val type: KType,
+    override val contentType: ContentType,
+    override val fileExtensions: Set<String>,
+) : SnarkParser<R>, IOReader<R> by reader
+
+
+inline fun <reified R : Any> SnarkParser(
+    reader: IOReader<R>,
+    contentType: ContentType,
+    vararg fileExtensions: String,
+): SnarkParser<R> = SnarkParserWrapper(reader, typeOf<R>(), contentType, fileExtensions.toSet())
 
 @OptIn(DFExperimental::class)
 class SnarkPlugin : AbstractPlugin() {
@@ -45,31 +68,46 @@ class SnarkPlugin : AbstractPlugin() {
         context.gather(SnarkParser.TYPE, true)
     }
 
-    val parseAction = Action.map {
+//    val parseAction: Action<ByteArray, Any> = Action.map {
+//        val parser: SnarkParser<*>? = parsers.values.filter { parser ->
+//            parser.contentType.toString() == meta["contentType"].string ||
+//                    meta[META_FILE_EXTENSION_KEY].string in parser.fileExtensions
+//        }.maxByOrNull {
+//            it.priority
+//        }
+//
+//        //ensure that final type is correct
+//        if (parser == null) {
+//            logger.warn { "The parser is not found for data with meta $meta" }
+//            result { it }
+//        } else {
+//            result(parser.resultType) { bytes ->
+//                parser.parse(bytes, meta)
+//            }
+//        }
+//    }
+
+    fun readDirectory(path: Path): DataTree<Any> = io.readDataDirectory(path) { dataPath, meta ->
+        val fileExtension = meta[FileData.META_FILE_EXTENSION_KEY].string ?: dataPath.extension
         val parser: SnarkParser<*>? = parsers.values.filter { parser ->
-            parser.contentType.toString() == meta["contentType"].string ||
-                    meta[DirectoryDataTree.META_FILE_EXTENSION_KEY].string in parser.fileExtensions
+            fileExtension in parser.fileExtensions
         }.maxByOrNull {
             it.priority
         }
 
-        //ensure that final type is correct
-        if (parser == null) {
-            logger.warn { "The parser is not found for data with meta $meta" }
-            result { it }
-        } else {
-            result(parser.resultType) { bytes ->
-                parser.parse(bytes, meta)
-            }
+        parser ?: run {
+            logger.warn { "The parser is not found for file $dataPath with meta $meta" }
+            byteArrayIOReader
         }
     }
 
+
     override fun content(target: String): Map<Name, Any> = when (target) {
         SnarkParser.TYPE -> mapOf(
             "html".asName() to SnarkHtmlParser,
             "markdown".asName() to SnarkMarkdownParser,
-            "json".asName() to SnarkJsonParser,
-            "yaml".asName() to SnarkYamlParser
+            "json".asName() to SnarkParser(JsonMetaFormat, ContentType.Application.Json, "json"),
+            "yaml".asName() to SnarkParser(YamlMetaFormat, ContentType.Application.Json, "yaml", "yml")
         )
         else -> super.content(target)
     }
@@ -79,5 +117,9 @@ class SnarkPlugin : AbstractPlugin() {
         override val type: KClass<out SnarkPlugin> = SnarkPlugin::class
 
         override fun build(context: Context, meta: Meta): SnarkPlugin = SnarkPlugin()
+
+        private val byteArrayIOReader = IOReader {
+            readBytes()
+        }
     }
 }

src/main/kotlin/space/kscience/snark/SnarkYamlParser.kt

@@ -1,18 +0,0 @@
-package space.kscience.snark
-
-import io.ktor.http.ContentType
-import space.kscience.dataforge.io.asBinary
-import space.kscience.dataforge.io.readObject
-import space.kscience.dataforge.io.yaml.YamlMetaFormat
-import space.kscience.dataforge.meta.Meta
-import kotlin.reflect.KType
-import kotlin.reflect.typeOf
-
-object SnarkYamlParser : SnarkParser<Meta> {
-    override val contentType: ContentType = ContentType.Application.Json
-    override val fileExtensions: Set<String> = setOf("yaml", "yml")
-    override val resultType: KType = typeOf<Meta>()
-
-    override suspend fun parse(bytes: ByteArray, meta: Meta): Meta =
-        YamlMetaFormat.readObject(bytes.asBinary())
-}