Refactor

build.gradle.kts

@@ -1,4 +1,3 @@
-import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
 import ru.mipt.npm.gradle.KScienceVersions
 
 plugins {
@@ -21,11 +20,11 @@ application {
     applicationDefaultJvmArgs = listOf("-Dio.ktor.development=$isDevelopment")
 }
 
-tasks.withType<KotlinCompile>{
-    kotlinOptions{
-        freeCompilerArgs = freeCompilerArgs + "-Xcontext-receivers"
-    }
-}
+//tasks.withType<KotlinCompile>{
+//    kotlinOptions{
+//        freeCompilerArgs = freeCompilerArgs + "-Xcontext-receivers"
+//    }
+//}
 
 val dataforgeVersion by extra("0.6.0-dev-3")
 val ktorVersion  = KScienceVersions.ktorVersion

src/main/kotlin/ru/mipt/Application.kt

@@ -16,12 +16,12 @@ import io.ktor.server.routing.route
 import io.ktor.server.routing.routing
 import ru.mipt.plugins.configureTemplating
 import ru.mipt.spc.magprog.DataSetSiteContext
-import ru.mipt.spc.magprog.DirectoryDataTree
 import ru.mipt.spc.magprog.SiteContext
 import ru.mipt.spc.magprog.magProgPage
 import space.kscience.dataforge.context.Context
 import space.kscience.dataforge.io.io
 import space.kscience.dataforge.io.yaml.YamlPlugin
+import space.kscience.snark.DirectoryDataTree
 import java.nio.file.Path
 
 

src/main/kotlin/ru/mipt/snapshot.kt

@@ -0,0 +1,12 @@
+package ru.mipt
+
+//private fun snapshotRoute(route: Route, path: Path){
+//    route.children.forEach {
+//        it.
+//    }
+//}
+//
+//
+//fun Application.snapshotTo(path: Path): Job = launch{
+//    pluginOrNull(Routing)?.children
+//}

src/main/kotlin/ru/mipt/spc/magprog/DataSetSiteContext.kt

@@ -7,7 +7,6 @@ import kotlinx.serialization.json.Json
 import org.intellij.markdown.flavours.commonmark.CommonMarkFlavourDescriptor
 import org.intellij.markdown.html.HtmlGenerator
 import org.intellij.markdown.parser.MarkdownParser
-import ru.mipt.spc.magprog.DirectoryDataTree.Companion.META_FILE_EXTENSION_KEY
 import space.kscience.dataforge.context.Context
 import space.kscience.dataforge.data.*
 import space.kscience.dataforge.io.asBinary
@@ -19,6 +18,7 @@ import space.kscience.dataforge.meta.string
 import space.kscience.dataforge.meta.toMeta
 import space.kscience.dataforge.misc.DFInternal
 import space.kscience.dataforge.names.Name
+import space.kscience.snark.DirectoryDataTree.Companion.META_FILE_EXTENSION_KEY
 import kotlin.reflect.KType
 import kotlin.reflect.full.isSubtypeOf
 import kotlin.reflect.typeOf
@@ -35,12 +35,12 @@ class DataSetSiteContext(
     private val markdownParser = MarkdownParser(markdownFlavor)
 
     //TODO replace by a plugin
-    private suspend fun Data<ByteArray>.toHtmlBlock(): HtmlBlock {
+    private suspend fun Data<ByteArray>.toHtmlBlock(): HtmlData {
         val fileType = meta[META_FILE_EXTENSION_KEY].string
         val src = await().decodeToString()
 
         return when (fileType) {
-            "html" -> HtmlBlock(meta) {
+            "html" -> HtmlData(meta) {
                 div {
                     unsafe {
                         +src
@@ -48,7 +48,7 @@ class DataSetSiteContext(
                 }
             }
 
-            "markdown", "mdown", "mkdn", "mkd", "md" -> HtmlBlock(meta) {
+            "markdown", "mdown", "mkdn", "mkd", "md" -> HtmlData(meta) {
                 div("markdown") {
                     val parsedTree = markdownParser.buildMarkdownTreeFromString(src)
 
@@ -93,11 +93,11 @@ class DataSetSiteContext(
     override fun <T : Any> resolveAll(type: KType, filter: (name: Name, meta: Meta) -> Boolean): DataSet<T> =
         dataSet.select(type, filter = filter)
 
-    override fun resolveHtml(name: Name): HtmlBlock? = runBlocking {
+    override fun resolveHtml(name: Name): HtmlData? = runBlocking {
         resolve<ByteArray>(name)?.takeIf { it.published }?.toHtmlBlock()
     }
 
-    override fun resolveAllHtml(filter: (name: Name, meta: Meta) -> Boolean): Map<Name, HtmlBlock> = runBlocking {
+    override fun resolveAllHtml(filter: (name: Name, meta: Meta) -> Boolean): Map<Name, HtmlData> = runBlocking {
         buildMap {
             resolveAll<ByteArray>(filter).dataSequence().filter { it.published }.forEach {
                 put(it.name, it.toHtmlBlock())

src/main/kotlin/ru/mipt/spc/magprog/HtmlBlock.kt

@@ -1,25 +0,0 @@
-package ru.mipt.spc.magprog
-
-import kotlinx.html.FlowContent
-import space.kscience.dataforge.meta.*
-
-enum class Language {
-    EN,
-    RU
-}
-
-interface HtmlBlock {
-    val meta: Meta
-    val content: FlowContent.() -> Unit
-}
-
-fun HtmlBlock(meta: Meta, content: FlowContent.() -> Unit ) = object: HtmlBlock{
-    override val meta: Meta = meta
-    override val content: FlowContent.() -> Unit = content
-}
-
-val HtmlBlock.id: String get() = meta["id"]?.string ?: "block[${hashCode()}]"
-val HtmlBlock.language: Language get() = meta["language"]?.enum<Language>() ?: Language.RU
-
-val HtmlBlock.order: Int? get() = meta["order"]?.int
-

src/main/kotlin/ru/mipt/spc/magprog/HtmlData.kt

@@ -0,0 +1,34 @@
+package ru.mipt.spc.magprog
+
+import kotlinx.coroutines.runBlocking
+import kotlinx.html.FlowContent
+import kotlinx.html.TagConsumer
+import space.kscience.dataforge.data.Data
+import space.kscience.dataforge.data.await
+import space.kscience.dataforge.meta.*
+
+enum class Language {
+    EN,
+    RU
+}
+
+//TODO replace by VisionForge type
+typealias HtmlFragment = TagConsumer<*>.() -> Unit
+
+typealias HtmlData = Data<HtmlFragment>
+
+fun HtmlData(meta: Meta, content: TagConsumer<*>.() -> Unit): HtmlData = Data(content, meta)
+
+val HtmlData.id: String get() = meta["id"]?.string ?: "block[${hashCode()}]"
+val HtmlData.language: Language get() = meta["language"]?.enum<Language>() ?: Language.RU
+
+val HtmlData.order: Int? get() = meta["order"]?.int
+
+fun TagConsumer<*>.htmlData(data: HtmlData) = runBlocking {
+    data.await().invoke(this@htmlData)
+}
+
+fun FlowContent.htmlData(data: HtmlData) = runBlocking {
+    data.await().invoke(consumer)
+}
+

src/main/kotlin/ru/mipt/spc/magprog/Person.kt

@@ -4,7 +4,7 @@ import kotlinx.css.*
 import kotlinx.html.*
 import space.kscience.dataforge.meta.string
 
-class Person(val block: HtmlBlock) : HtmlBlock by block {
+class Person(val block: HtmlData) : HtmlData by block {
     val name: String by meta.string { error("Mentor name is not defined") }
     val photo: String? by meta.string()
 }
@@ -27,7 +27,7 @@ private fun FlowContent.personCards(list: List<Person>, prefix: String) {
                     h2 {
                         a(href = "#${prefix}_${mentor.id}") { +mentor.name }
                     }
-                    with(mentor) { content() }
+                    htmlData(mentor.block)
                 }
             }
         }

src/main/kotlin/ru/mipt/spc/magprog/SiteContext.kt

@@ -28,12 +28,12 @@ interface SiteContext: ContextAware {
     /**
      * Resolve a Html builder by its full name
      */
-    fun resolveHtml(name: Name): HtmlBlock?
+    fun resolveHtml(name: Name): HtmlData?
 
     /**
      * Find all Html blocks using given name/meta filter
      */
-    fun resolveAllHtml(filter: (name: Name, meta: Meta) -> Boolean): Map<Name, HtmlBlock>
+    fun resolveAllHtml(filter: (name: Name, meta: Meta) -> Boolean): Map<Name, HtmlData>
 }
 
 @OptIn(DFInternal::class)

src/main/kotlin/ru/mipt/spc/magprog/magProgPage.kt

@@ -6,8 +6,10 @@ import kotlinx.html.*
 import space.kscience.dataforge.data.await
 import space.kscience.dataforge.meta.Meta
 import space.kscience.dataforge.meta.get
+import space.kscience.dataforge.meta.getIndexed
 import space.kscience.dataforge.meta.string
 import space.kscience.dataforge.names.Name
+import space.kscience.dataforge.names.asName
 import space.kscience.dataforge.names.plus
 
 //fun CssBuilder.magProgCss() {
@@ -26,9 +28,9 @@ class MagProgSection(
     val id: String,
     val title: String,
     val style: String,
-    override val content: FlowContent.() -> Unit,
-) : HtmlBlock {
-    override val meta: Meta
+    val content: FlowContent.() -> Unit,
+)  {
+    val meta: Meta
         get() = Meta {
             "id" put id
             "title" put title
@@ -47,13 +49,14 @@ private fun wrapSection(
 }
 
 private fun wrapSection(
-    block: HtmlBlock,
+    block: HtmlData,
     idOverride: String? = null,
 ): MagProgSection = wrapSection(
     idOverride ?: block.id,
     block.meta["section_title"]?.string ?: error("Section without title"),
-    block.content
-)
+){
+    htmlData(block)
+}
 
 private val CONTENT_NODE_NAME = Name.EMPTY//"content".asName()
 private val INTRO_PATH: Name = CONTENT_NODE_NAME + "intro"
@@ -68,7 +71,7 @@ context(SiteContext) private fun FlowContent.programSection() {
     val recommendedBlock = resolveHtml(RECOMMENDED_COURSES_PATH)!!
     div("inner") {
         h2 { +"Учебная программа" }
-        with(programBlock) { content() }
+        htmlData(programBlock)
         button(classes = "fit btn btn-primary btn-lg") {
             attributes["data-bs-toggle"] = "collapse"
             attributes["data-bs-target"] = "#recommended-courses-collapse-text"
@@ -79,7 +82,7 @@ context(SiteContext) private fun FlowContent.programSection() {
         div("collapse pt-3") {
             id = "recommended-courses-collapse-text"
             div("card card-body") {
-                with(recommendedBlock) { content() }
+                htmlData(recommendedBlock)
             }
         }
     }
@@ -91,7 +94,7 @@ context(SiteContext) private fun FlowContent.partners() {
     div("inner") {
         h2 { +"Партнеры" }
         div("features") {
-            partnersData.items.values.forEach { partner ->
+            partnersData.getIndexed("content".asName()).values.forEach { partner ->
                 section {
                     a(href = partner["link"].string, target = "_blank") {
                         rel = "noreferrer"
@@ -205,7 +208,7 @@ context(SiteContext) fun HTML.magProgPage() {
             id = "footer"
             div("inner") {
                 ul("menu") {
-                    li { +"""© Untitled. All rights reserved.""" }
+                    li { +"""© SPC. All rights reserved.""" }
                     li {
                         +"""Design:"""
                         a {

src/main/kotlin/space/kscience/snark/DirectoryDataTree.kt

@@ -1,9 +1,8 @@
-package ru.mipt.spc.magprog
+package space.kscience.snark
 
 import space.kscience.dataforge.data.Data
 import space.kscience.dataforge.data.DataTree
 import space.kscience.dataforge.data.DataTreeItem
-import space.kscience.dataforge.data.StaticData
 import space.kscience.dataforge.io.IOPlugin
 import space.kscience.dataforge.io.readEnvelopeFile
 import space.kscience.dataforge.io.readMetaFile
@@ -21,22 +20,6 @@ import kotlin.io.path.nameWithoutExtension
 import kotlin.reflect.KType
 import kotlin.reflect.typeOf
 
-//internal object ByteArrayIOFormat : IOFormat<ByteArray> {
-//
-//    override val type: KType = typeOf<ByteArray>()
-//
-//    override fun writeObject(output: Output, obj: ByteArray) {
-//        output.writeFully(obj)
-//    }
-//
-//    override fun readObject(input: Input): ByteArray = input.readBytes()
-//
-//    override fun toMeta(): Meta = Meta {
-//        IOFormat.NAME_KEY put "ByteArray"
-//    }
-//
-//}
-
 
 class DirectoryDataTree(val io: IOPlugin, val path: Path) : DataTree<ByteArray> {
     override val dataType: KType
@@ -51,7 +34,9 @@ class DirectoryDataTree(val io: IOPlugin, val path: Path) : DataTree<ByteArray>
             META_FILE_EXTENSION_KEY put filePath.extension
             //TODO add other file information
         }
-        return StaticData(typeOf<ByteArray>(), envelope.data?.toByteArray() ?: ByteArray(0), meta)
+        return Data(meta){
+            envelope.data?.toByteArray() ?: ByteArray(0)
+        }
     }
 
 

src/main/kotlin/space/kscience/snark/SnarkPlugin.kt

@@ -0,0 +1,59 @@
+package space.kscience.snark
+
+import io.ktor.http.ContentType
+import space.kscience.dataforge.actions.Action
+import space.kscience.dataforge.actions.map
+import space.kscience.dataforge.context.*
+import space.kscience.dataforge.io.yaml.YamlPlugin
+import space.kscience.dataforge.meta.Meta
+import space.kscience.dataforge.meta.get
+import space.kscience.dataforge.meta.string
+import space.kscience.dataforge.misc.Type
+import space.kscience.dataforge.names.Name
+import kotlin.reflect.KClass
+import kotlin.reflect.KType
+
+@Type(SnarkParser.TYPE)
+interface SnarkParser<R : Any> {
+    val contentType: ContentType
+
+    val fileExtensions: Set<String>
+
+    val priority: Int
+
+    val resultType: KType
+
+    suspend fun parse(bytes: ByteArray): R
+
+    companion object {
+        const val TYPE = "snark.parser"
+    }
+}
+
+
+class SnarkPlugin : AbstractPlugin() {
+    val yaml by require(YamlPlugin)
+    val io get() = yaml.io
+
+    override val tag: PluginTag get() = Companion.tag
+
+    private val parsers: Map<Name, SnarkParser<*>> by lazy { context.gather(SnarkParser.TYPE, true) }
+
+    private val parseAction = Action.map<ByteArray, Any> {
+        result { bytes ->
+            parsers.values.filter { parser ->
+                parser.contentType.toString() == meta["contentType"].string ||
+                        meta[DirectoryDataTree.META_FILE_EXTENSION_KEY].string in parser.fileExtensions
+            }.maxByOrNull {
+                it.priority
+            }?.parse(bytes) ?: bytes
+        }
+    }
+
+    companion object : PluginFactory<SnarkPlugin> {
+        override val tag: PluginTag = PluginTag("snark")
+        override val type: KClass<out SnarkPlugin> = SnarkPlugin::class
+
+        override fun build(context: Context, meta: Meta): SnarkPlugin = SnarkPlugin()
+    }
+}