Moved to Snark engine

2022-05-04 19:29:29 +03:00 · 2022-05-04 19:29:29 +03:00 · 7de2e02433
commit 7de2e02433
parent 9c75852945
13 changed files with 146 additions and 27 deletions
--- a/build.gradle.kts
+++ b/build.gradle.kts
@ -6,9 +6,9 @@ plugins {
    application
 }

-repositories{
-    mavenLocal()
-}
+//repositories{
+//    mavenLocal()
+//}

 group = "ru.mipt.npm"
 version = "0.0.1-SNAPSHOT"
@ -28,7 +28,7 @@ tasks.withType<org.jetbrains.kotlin.gradle.tasks.KotlinCompile>{
    }
 }

-val dataforgeVersion by extra("0.6.0-dev-4")
+val dataforgeVersion by extra("0.6.0-dev-5")
 val ktorVersion  = KScienceVersions.ktorVersion

 dependencies {
--- a/gradle.properties
+++ b/gradle.properties
@ -2,4 +2,4 @@ kotlin.code.style=official

 toolsVersion=0.11.4-kotlin-1.6.20

-#development=true
+development=true
--- a/src/main/kotlin/ru/mipt/spc/magprog/DataSetPageContext.kt
+++ b/src/main/kotlin/ru/mipt/spc/magprog/DataSetPageContext.kt
@ -1,6 +1,5 @@
 package ru.mipt.spc.magprog

-import io.ktor.server.application.Application
 import kotlinx.coroutines.runBlocking
 import kotlinx.html.div
 import kotlinx.html.unsafe
@ -25,8 +24,9 @@ import kotlin.reflect.KType
 import kotlin.reflect.full.isSubtypeOf
 import kotlin.reflect.typeOf

+internal val Data<*>.published: Boolean get() = meta["published"].string != "false"
+
 class DataSetPageContext(
-    val application: Application,
    override val context: Context,
    val prefix: String,
    val dataSet: DataSet<Any>,
@ -65,12 +65,13 @@ class DataSetPageContext(

    }

-    private val Data<*>.published: Boolean get() = meta["published"].string != "false"
-
    @Suppress("UNCHECKED_CAST")
    @DFInternal
    override fun <T : Any> resolve(type: KType, name: Name): Data<T>? {
        val data: Data<Any> = dataSet[name] ?: return null
+
+        if(data.type == type) return data as Data<T>
+
        return if (type == typeOf<Meta>() && data.type == typeOf<ByteArray>()) {
            data as Data<ByteArray>
            when (data.meta[META_FILE_EXTENSION_KEY].string) {
@ -94,8 +95,8 @@ class DataSetPageContext(
    }

    @DFInternal
-    override fun <T : Any> resolveAll(type: KType, filter: (name: Name, meta: Meta) -> Boolean): DataSet<T> =
-        dataSet.select(type, filter = filter)
+    override fun <T : Any> resolveAll(type: KType, predicate: (name: Name, meta: Meta) -> Boolean): DataSet<T> =
+        dataSet.filterIsInstance(type, predicate = predicate)

    override fun resolveHtml(name: Name): HtmlData? = runBlocking {
        resolve<ByteArray>(name)?.takeIf { it.published }?.toHtmlBlock()
--- a/src/main/kotlin/ru/mipt/spc/magprog/PageContext.kt
+++ b/src/main/kotlin/ru/mipt/spc/magprog/PageContext.kt
@ -24,7 +24,7 @@ interface PageContext: ContextAware {
    fun <T: Any> resolve(type: KType, name: Name): Data<T>?

    @DFInternal
-    fun <T: Any> resolveAll(type: KType, filter: (name: Name, meta: Meta) -> Boolean): DataSet<T>
+    fun <T: Any> resolveAll(type: KType, predicate: (name: Name, meta: Meta) -> Boolean): DataSet<T>

    /**
     * Resolve a Html builder by its full name
--- a/src/main/kotlin/ru/mipt/spc/magprog/SnarkPageContext.kt
+++ b/src/main/kotlin/ru/mipt/spc/magprog/SnarkPageContext.kt
@ -0,0 +1,43 @@
+package ru.mipt.spc.magprog
+
+import space.kscience.dataforge.actions.invoke
+import space.kscience.dataforge.context.Context
+import space.kscience.dataforge.data.Data
+import space.kscience.dataforge.data.DataSet
+import space.kscience.dataforge.data.filterIsInstance
+import space.kscience.dataforge.data.selectOne
+import space.kscience.dataforge.meta.Meta
+import space.kscience.dataforge.misc.DFInternal
+import space.kscience.dataforge.names.Name
+import space.kscience.snark.DirectoryDataTree
+import space.kscience.snark.HtmlData
+import space.kscience.snark.HtmlFragment
+import space.kscience.snark.SnarkPlugin
+import java.nio.file.Path
+import kotlin.reflect.KType
+
+class SnarkPageContext(
+    val snarkPlugin: SnarkPlugin,
+    val path: Path,
+    val prefix: String,
+) : PageContext {
+    override val context: Context get() = snarkPlugin.context
+
+    override fun resolveRef(name: String): String = "$prefix/$name"
+
+    private val directoryDataTree by lazy { DirectoryDataTree(snarkPlugin.io, path) }
+
+    private val parsedData: DataSet<Any> by lazy { snarkPlugin.parseAction(directoryDataTree) }
+
+    @DFInternal
+    override fun <T : Any> resolve(type: KType, name: Name): Data<T>? = parsedData.selectOne(type, name)
+
+    @DFInternal
+    override fun <T : Any> resolveAll(type: KType, predicate: (name: Name, meta: Meta) -> Boolean): DataSet<T> =
+        parsedData.filterIsInstance(type, predicate)
+
+    override fun resolveHtml(name: Name): HtmlData? = resolve(name)
+
+    override fun resolveAllHtml(filter: (name: Name, meta: Meta) -> Boolean): Map<Name, HtmlData> =
+        resolveAll<HtmlFragment>(filter).dataSequence().filter { it.published }.associate { it.name to it.data }
+}
--- a/src/main/kotlin/ru/mipt/spc/magprog/magProgPage.kt
+++ b/src/main/kotlin/ru/mipt/spc/magprog/magProgPage.kt
@ -11,8 +11,8 @@ import io.ktor.server.routing.routing
 import kotlinx.coroutines.runBlocking
 import kotlinx.html.*
 import space.kscience.dataforge.context.Context
+import space.kscience.dataforge.context.fetch
 import space.kscience.dataforge.data.await
-import space.kscience.dataforge.io.io
 import space.kscience.dataforge.meta.Meta
 import space.kscience.dataforge.meta.get
 import space.kscience.dataforge.meta.getIndexed
@ -266,15 +266,18 @@ context(PageContext) internal fun BODY.magProgFooter() {
 internal val Person.mentorPageId get() = "mentor-${id}"

 internal fun Application.magProgPage(context: Context, rootPath: Path, prefix: String = "/magprog") {
-    val io = context.io
-    val content = DirectoryDataTree(io, rootPath.resolve("content"))
+//    val io = context.io
+//    val content = DirectoryDataTree(io, rootPath.resolve("content"))
+//
+//    val magprogPageContext: PageContext = DataSetPageContext(context, prefix, content)

+    val snark = context.fetch(SnarkPlugin)

-    val magprogPageContext: PageContext = DataSetPageContext(this, context, prefix, content)
+    val magProgPageContext = SnarkPageContext(snark, rootPath.resolve("content"), prefix)

    routing {
        route(prefix) {
-            with(magprogPageContext) {
+            with(magProgPageContext) {
                static {
                    files(rootPath.resolve("assets").toFile())
                }
--- a/src/main/kotlin/space/kscience/snark/SnarkJsonParser.kt
+++ b/src/main/kotlin/space/kscience/snark/SnarkJsonParser.kt
@ -0,0 +1,17 @@
+package space.kscience.snark
+
+import io.ktor.http.ContentType
+import kotlinx.serialization.json.Json
+import space.kscience.dataforge.meta.Meta
+import space.kscience.dataforge.meta.toMeta
+import kotlin.reflect.KType
+import kotlin.reflect.typeOf
+
+object SnarkJsonParser: SnarkParser<Meta> {
+    override val contentType: ContentType = ContentType.Application.Json
+    override val fileExtensions: Set<String> = setOf("json")
+    override val resultType: KType= typeOf<Meta>()
+
+    override suspend fun parse(bytes: ByteArray, meta: Meta): Meta =
+        Json.parseToJsonElement(bytes.decodeToString()).toMeta()
+}
--- a/src/main/kotlin/space/kscience/snark/SnarkMarkdownParser.kt
+++ b/src/main/kotlin/space/kscience/snark/SnarkMarkdownParser.kt
@ -0,0 +1,31 @@
+package space.kscience.snark
+
+import io.ktor.http.ContentType
+import kotlinx.html.div
+import kotlinx.html.unsafe
+import org.intellij.markdown.flavours.commonmark.CommonMarkFlavourDescriptor
+import org.intellij.markdown.html.HtmlGenerator
+import org.intellij.markdown.parser.MarkdownParser
+import space.kscience.dataforge.meta.Meta
+import kotlin.reflect.KType
+import kotlin.reflect.typeOf
+
+object SnarkMarkdownParser:SnarkParser<HtmlFragment> {
+    override val contentType: ContentType = ContentType.Text.Html
+    override val fileExtensions: Set<String> = setOf("markdown", "mdown", "mkdn", "mkd", "md")
+    override val resultType: KType = typeOf<HtmlFragment>()
+
+    private val markdownFlavor = CommonMarkFlavourDescriptor()
+    private val markdownParser = MarkdownParser(markdownFlavor)
+
+    override suspend fun parse(bytes: ByteArray, meta: Meta): HtmlFragment = {
+        val src = bytes.decodeToString()
+        div{
+            val parsedTree = markdownParser.buildMarkdownTreeFromString(src)
+
+            unsafe {
+                +HtmlGenerator(src, parsedTree, markdownFlavor).generateHtml()
+            }
+        }
+    }
+}
--- a/src/main/kotlin/space/kscience/snark/SnarkPlugin.kt
+++ b/src/main/kotlin/space/kscience/snark/SnarkPlugin.kt
@ -36,14 +36,14 @@ interface SnarkParser<R : Any> {

@OptIn(DFExperimental::class)
 class SnarkPlugin : AbstractPlugin() {
-    val yaml by require(YamlPlugin)
+    private val yaml by require(YamlPlugin)
    val io get() = yaml.io

    override val tag: PluginTag get() = Companion.tag

    private val parsers: Map<Name, SnarkParser<*>> by lazy { context.gather(SnarkParser.TYPE, true) }

-    private val parseAction = Action.map<ByteArray, Any> {
+    val parseAction = Action.map<ByteArray, Any> {
        val parser: SnarkParser<*>? = parsers.values.filter { parser ->
            parser.contentType.toString() == meta["contentType"].string ||
                    meta[DirectoryDataTree.META_FILE_EXTENSION_KEY].string in parser.fileExtensions
@ -65,7 +65,10 @@ class SnarkPlugin : AbstractPlugin() {
    override fun content(target: String): Map<Name, Any> {
        return when(target){
            SnarkParser.TYPE -> mapOf(
-                "html".asName() to SnarkHtmlParser
+                "html".asName() to SnarkHtmlParser,
+                "markdown".asName() to SnarkMarkdownParser,
+                "json".asName() to SnarkJsonParser,
+                "yaml".asName() to SnarkYamlParser
            )
            else ->super.content(target)
        }
--- a/src/main/kotlin/space/kscience/snark/SnarkYamlParser.kt
+++ b/src/main/kotlin/space/kscience/snark/SnarkYamlParser.kt
@ -0,0 +1,18 @@
+package space.kscience.snark
+
+import io.ktor.http.ContentType
+import space.kscience.dataforge.io.asBinary
+import space.kscience.dataforge.io.readObject
+import space.kscience.dataforge.io.yaml.YamlMetaFormat
+import space.kscience.dataforge.meta.Meta
+import kotlin.reflect.KType
+import kotlin.reflect.typeOf
+
+object SnarkYamlParser : SnarkParser<Meta> {
+    override val contentType: ContentType = ContentType.Application.Json
+    override val fileExtensions: Set<String> = setOf("yaml", "yml")
+    override val resultType: KType = typeOf<Meta>()
+
+    override suspend fun parse(bytes: ByteArray, meta: Meta): Meta =
+        YamlMetaFormat.readObject(bytes.asBinary())
+}
--- a/src/main/resources/magprog/content/contacts.md
+++ b/src/main/resources/magprog/content/contacts.md
@ -5,10 +5,6 @@ section_title: Контакты
 language: ru
 ---

-Сайт лаборатории: [https://npm.mipt.ru](/)
-
-Страница направления в JetBrains Research: [https://research.jetbrains.org/groups/npm/](https://research.jetbrains.org/groups/npm/).
-
 Все вопросы можно задать в телеграм-канале лаборатории: [https://t.me/mipt_npm](https://t.me/mipt_npm).

 Также можно писать на электронную почту: <a href='mailto&#58;&#110;p&#109;&#64;m%&#54;&#57;%70&#116;&#46;ru'>npm&#64;mip&#116;&#46;ru</a>.
--- a/src/main/resources/magprog/content/intro.md
+++ b/src/main/resources/magprog/content/intro.md
@ -1,11 +1,11 @@
 ---
 content_type: magprog
 magprog_section: intro
-section_title: Магистерская программа
+section_title: О программе
 language: ru
 ---

-Магистерская программа МФТИ **&laquo;Разработка и применение программного обеспечения в физических исследованиях&raquo;** создана на базе [лаборатории методов ядерно-физических экспериментов (ЛМЯФЭ)](/) при поддержке двух школ МФТИ: Физтех-школы физики и исследований им. Ландау ([ЛФИ](https://mipt.ru/education/departments/lpr/)) и Физтех-школы прикладной математики и информатики ([ФПМИ](https://mipt.ru/education/departments/fpmi/)) и ряда академических и промышленных партнеров. В ее основе лежит взаимодействие студента и [научного руководителя](#mentors). 
+Магистерская программа МФТИ **"Научное программное обеспечение"** (старое название: **"Разработка и применение программного обеспечения в физических исследованиях"**) создана при поддержке двух школ МФТИ: Физтех-школы физики и исследований им. Ландау ([ЛФИ](https://mipt.ru/education/departments/lpr/)) и Физтех-школы прикладной математики и информатики ([ФПМИ](https://mipt.ru/education/departments/fpmi/)) и ряда академических и промышленных партнеров. В ее основе лежит взаимодействие студента и [научного руководителя](#mentors). 

 Цель создания программы &mdash; объединение усилий ученых и программистов для разработки лучших компьютерных решений и применения этих решений в области фундаментальной и прикладной науки и инженерии.

--- a/src/test/kotlin/ru/mipt/ApplicationTest.kt
+++ b/src/test/kotlin/ru/mipt/ApplicationTest.kt
@ -3,6 +3,9 @@ package ru.mipt
 import io.ktor.client.request.get
 import io.ktor.http.HttpStatusCode
 import io.ktor.server.testing.testApplication
+import ru.mipt.spc.magprog.magProgPage
+import space.kscience.dataforge.context.Context
+import space.kscience.snark.SnarkPlugin
 import java.nio.file.Path
 import kotlin.test.Test
 import kotlin.test.assertEquals
@ -10,8 +13,12 @@ import kotlin.test.assertEquals
 class ApplicationTest {
    @Test
    fun testRoot() = testApplication {
+        val context = Context("spc-site") {
+            plugin(SnarkPlugin)
+        }
+
        application {
-            magProgPage(rootPath = Path.of(javaClass.getResource("/magprog")!!.toURI()))
+            magProgPage(context, rootPath = Path.of(javaClass.getResource("/magprog")!!.toURI()))
        }
        client.get("/magprog").apply {
            assertEquals(HttpStatusCode.OK, status)