From 7de2e02433419727dec390f780a3c7d2aca2d5eb Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Wed, 4 May 2022 19:29:29 +0300 Subject: [PATCH] Moved to Snark engine --- build.gradle.kts | 8 ++-- gradle.properties | 2 +- .../ru/mipt/spc/magprog/DataSetPageContext.kt | 13 +++--- .../kotlin/ru/mipt/spc/magprog/PageContext.kt | 2 +- .../ru/mipt/spc/magprog/SnarkPageContext.kt | 43 +++++++++++++++++++ .../kotlin/ru/mipt/spc/magprog/magProgPage.kt | 13 +++--- .../space/kscience/snark/SnarkJsonParser.kt | 17 ++++++++ .../kscience/snark/SnarkMarkdownParser.kt | 31 +++++++++++++ .../space/kscience/snark/SnarkPlugin.kt | 9 ++-- .../space/kscience/snark/SnarkYamlParser.kt | 18 ++++++++ .../resources/magprog/content/contacts.md | 4 -- src/main/resources/magprog/content/intro.md | 4 +- src/test/kotlin/ru/mipt/ApplicationTest.kt | 9 +++- 13 files changed, 146 insertions(+), 27 deletions(-) create mode 100644 src/main/kotlin/ru/mipt/spc/magprog/SnarkPageContext.kt create mode 100644 src/main/kotlin/space/kscience/snark/SnarkJsonParser.kt create mode 100644 src/main/kotlin/space/kscience/snark/SnarkMarkdownParser.kt create mode 100644 src/main/kotlin/space/kscience/snark/SnarkYamlParser.kt diff --git a/build.gradle.kts b/build.gradle.kts index f956679..26cdd8b 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -6,9 +6,9 @@ plugins { application } -repositories{ - mavenLocal() -} +//repositories{ +// mavenLocal() +//} group = "ru.mipt.npm" version = "0.0.1-SNAPSHOT" @@ -28,7 +28,7 @@ tasks.withType{ } } -val dataforgeVersion by extra("0.6.0-dev-4") +val dataforgeVersion by extra("0.6.0-dev-5") val ktorVersion = KScienceVersions.ktorVersion dependencies { diff --git a/gradle.properties b/gradle.properties index a902a04..04320d7 100644 --- a/gradle.properties +++ b/gradle.properties @@ -2,4 +2,4 @@ kotlin.code.style=official toolsVersion=0.11.4-kotlin-1.6.20 -#development=true +development=true diff --git a/src/main/kotlin/ru/mipt/spc/magprog/DataSetPageContext.kt b/src/main/kotlin/ru/mipt/spc/magprog/DataSetPageContext.kt index 1a13110..afa45a9 100644 --- a/src/main/kotlin/ru/mipt/spc/magprog/DataSetPageContext.kt +++ b/src/main/kotlin/ru/mipt/spc/magprog/DataSetPageContext.kt @@ -1,6 +1,5 @@ package ru.mipt.spc.magprog -import io.ktor.server.application.Application import kotlinx.coroutines.runBlocking import kotlinx.html.div import kotlinx.html.unsafe @@ -25,8 +24,9 @@ import kotlin.reflect.KType import kotlin.reflect.full.isSubtypeOf import kotlin.reflect.typeOf +internal val Data<*>.published: Boolean get() = meta["published"].string != "false" + class DataSetPageContext( - val application: Application, override val context: Context, val prefix: String, val dataSet: DataSet, @@ -65,12 +65,13 @@ class DataSetPageContext( } - private val Data<*>.published: Boolean get() = meta["published"].string != "false" - @Suppress("UNCHECKED_CAST") @DFInternal override fun resolve(type: KType, name: Name): Data? { val data: Data = dataSet[name] ?: return null + + if(data.type == type) return data as Data + return if (type == typeOf() && data.type == typeOf()) { data as Data when (data.meta[META_FILE_EXTENSION_KEY].string) { @@ -94,8 +95,8 @@ class DataSetPageContext( } @DFInternal - override fun resolveAll(type: KType, filter: (name: Name, meta: Meta) -> Boolean): DataSet = - dataSet.select(type, filter = filter) + override fun resolveAll(type: KType, predicate: (name: Name, meta: Meta) -> Boolean): DataSet = + dataSet.filterIsInstance(type, predicate = predicate) override fun resolveHtml(name: Name): HtmlData? = runBlocking { resolve(name)?.takeIf { it.published }?.toHtmlBlock() diff --git a/src/main/kotlin/ru/mipt/spc/magprog/PageContext.kt b/src/main/kotlin/ru/mipt/spc/magprog/PageContext.kt index 4e1e215..e230a71 100644 --- a/src/main/kotlin/ru/mipt/spc/magprog/PageContext.kt +++ b/src/main/kotlin/ru/mipt/spc/magprog/PageContext.kt @@ -24,7 +24,7 @@ interface PageContext: ContextAware { fun resolve(type: KType, name: Name): Data? @DFInternal - fun resolveAll(type: KType, filter: (name: Name, meta: Meta) -> Boolean): DataSet + fun resolveAll(type: KType, predicate: (name: Name, meta: Meta) -> Boolean): DataSet /** * Resolve a Html builder by its full name diff --git a/src/main/kotlin/ru/mipt/spc/magprog/SnarkPageContext.kt b/src/main/kotlin/ru/mipt/spc/magprog/SnarkPageContext.kt new file mode 100644 index 0000000..8c817a1 --- /dev/null +++ b/src/main/kotlin/ru/mipt/spc/magprog/SnarkPageContext.kt @@ -0,0 +1,43 @@ +package ru.mipt.spc.magprog + +import space.kscience.dataforge.actions.invoke +import space.kscience.dataforge.context.Context +import space.kscience.dataforge.data.Data +import space.kscience.dataforge.data.DataSet +import space.kscience.dataforge.data.filterIsInstance +import space.kscience.dataforge.data.selectOne +import space.kscience.dataforge.meta.Meta +import space.kscience.dataforge.misc.DFInternal +import space.kscience.dataforge.names.Name +import space.kscience.snark.DirectoryDataTree +import space.kscience.snark.HtmlData +import space.kscience.snark.HtmlFragment +import space.kscience.snark.SnarkPlugin +import java.nio.file.Path +import kotlin.reflect.KType + +class SnarkPageContext( + val snarkPlugin: SnarkPlugin, + val path: Path, + val prefix: String, +) : PageContext { + override val context: Context get() = snarkPlugin.context + + override fun resolveRef(name: String): String = "$prefix/$name" + + private val directoryDataTree by lazy { DirectoryDataTree(snarkPlugin.io, path) } + + private val parsedData: DataSet by lazy { snarkPlugin.parseAction(directoryDataTree) } + + @DFInternal + override fun resolve(type: KType, name: Name): Data? = parsedData.selectOne(type, name) + + @DFInternal + override fun resolveAll(type: KType, predicate: (name: Name, meta: Meta) -> Boolean): DataSet = + parsedData.filterIsInstance(type, predicate) + + override fun resolveHtml(name: Name): HtmlData? = resolve(name) + + override fun resolveAllHtml(filter: (name: Name, meta: Meta) -> Boolean): Map = + resolveAll(filter).dataSequence().filter { it.published }.associate { it.name to it.data } +} \ No newline at end of file diff --git a/src/main/kotlin/ru/mipt/spc/magprog/magProgPage.kt b/src/main/kotlin/ru/mipt/spc/magprog/magProgPage.kt index 52c89bf..7c411e7 100644 --- a/src/main/kotlin/ru/mipt/spc/magprog/magProgPage.kt +++ b/src/main/kotlin/ru/mipt/spc/magprog/magProgPage.kt @@ -11,8 +11,8 @@ import io.ktor.server.routing.routing import kotlinx.coroutines.runBlocking import kotlinx.html.* import space.kscience.dataforge.context.Context +import space.kscience.dataforge.context.fetch import space.kscience.dataforge.data.await -import space.kscience.dataforge.io.io import space.kscience.dataforge.meta.Meta import space.kscience.dataforge.meta.get import space.kscience.dataforge.meta.getIndexed @@ -266,15 +266,18 @@ context(PageContext) internal fun BODY.magProgFooter() { internal val Person.mentorPageId get() = "mentor-${id}" internal fun Application.magProgPage(context: Context, rootPath: Path, prefix: String = "/magprog") { - val io = context.io - val content = DirectoryDataTree(io, rootPath.resolve("content")) +// val io = context.io +// val content = DirectoryDataTree(io, rootPath.resolve("content")) +// +// val magprogPageContext: PageContext = DataSetPageContext(context, prefix, content) + val snark = context.fetch(SnarkPlugin) - val magprogPageContext: PageContext = DataSetPageContext(this, context, prefix, content) + val magProgPageContext = SnarkPageContext(snark, rootPath.resolve("content"), prefix) routing { route(prefix) { - with(magprogPageContext) { + with(magProgPageContext) { static { files(rootPath.resolve("assets").toFile()) } diff --git a/src/main/kotlin/space/kscience/snark/SnarkJsonParser.kt b/src/main/kotlin/space/kscience/snark/SnarkJsonParser.kt new file mode 100644 index 0000000..d0f84fc --- /dev/null +++ b/src/main/kotlin/space/kscience/snark/SnarkJsonParser.kt @@ -0,0 +1,17 @@ +package space.kscience.snark + +import io.ktor.http.ContentType +import kotlinx.serialization.json.Json +import space.kscience.dataforge.meta.Meta +import space.kscience.dataforge.meta.toMeta +import kotlin.reflect.KType +import kotlin.reflect.typeOf + +object SnarkJsonParser: SnarkParser { + override val contentType: ContentType = ContentType.Application.Json + override val fileExtensions: Set = setOf("json") + override val resultType: KType= typeOf() + + override suspend fun parse(bytes: ByteArray, meta: Meta): Meta = + Json.parseToJsonElement(bytes.decodeToString()).toMeta() +} \ No newline at end of file diff --git a/src/main/kotlin/space/kscience/snark/SnarkMarkdownParser.kt b/src/main/kotlin/space/kscience/snark/SnarkMarkdownParser.kt new file mode 100644 index 0000000..279dd07 --- /dev/null +++ b/src/main/kotlin/space/kscience/snark/SnarkMarkdownParser.kt @@ -0,0 +1,31 @@ +package space.kscience.snark + +import io.ktor.http.ContentType +import kotlinx.html.div +import kotlinx.html.unsafe +import org.intellij.markdown.flavours.commonmark.CommonMarkFlavourDescriptor +import org.intellij.markdown.html.HtmlGenerator +import org.intellij.markdown.parser.MarkdownParser +import space.kscience.dataforge.meta.Meta +import kotlin.reflect.KType +import kotlin.reflect.typeOf + +object SnarkMarkdownParser:SnarkParser { + override val contentType: ContentType = ContentType.Text.Html + override val fileExtensions: Set = setOf("markdown", "mdown", "mkdn", "mkd", "md") + override val resultType: KType = typeOf() + + private val markdownFlavor = CommonMarkFlavourDescriptor() + private val markdownParser = MarkdownParser(markdownFlavor) + + override suspend fun parse(bytes: ByteArray, meta: Meta): HtmlFragment = { + val src = bytes.decodeToString() + div{ + val parsedTree = markdownParser.buildMarkdownTreeFromString(src) + + unsafe { + +HtmlGenerator(src, parsedTree, markdownFlavor).generateHtml() + } + } + } +} \ No newline at end of file diff --git a/src/main/kotlin/space/kscience/snark/SnarkPlugin.kt b/src/main/kotlin/space/kscience/snark/SnarkPlugin.kt index 05de48a..21b6436 100644 --- a/src/main/kotlin/space/kscience/snark/SnarkPlugin.kt +++ b/src/main/kotlin/space/kscience/snark/SnarkPlugin.kt @@ -36,14 +36,14 @@ interface SnarkParser { @OptIn(DFExperimental::class) class SnarkPlugin : AbstractPlugin() { - val yaml by require(YamlPlugin) + private val yaml by require(YamlPlugin) val io get() = yaml.io override val tag: PluginTag get() = Companion.tag private val parsers: Map> by lazy { context.gather(SnarkParser.TYPE, true) } - private val parseAction = Action.map { + val parseAction = Action.map { val parser: SnarkParser<*>? = parsers.values.filter { parser -> parser.contentType.toString() == meta["contentType"].string || meta[DirectoryDataTree.META_FILE_EXTENSION_KEY].string in parser.fileExtensions @@ -65,7 +65,10 @@ class SnarkPlugin : AbstractPlugin() { override fun content(target: String): Map { return when(target){ SnarkParser.TYPE -> mapOf( - "html".asName() to SnarkHtmlParser + "html".asName() to SnarkHtmlParser, + "markdown".asName() to SnarkMarkdownParser, + "json".asName() to SnarkJsonParser, + "yaml".asName() to SnarkYamlParser ) else ->super.content(target) } diff --git a/src/main/kotlin/space/kscience/snark/SnarkYamlParser.kt b/src/main/kotlin/space/kscience/snark/SnarkYamlParser.kt new file mode 100644 index 0000000..8d2beeb --- /dev/null +++ b/src/main/kotlin/space/kscience/snark/SnarkYamlParser.kt @@ -0,0 +1,18 @@ +package space.kscience.snark + +import io.ktor.http.ContentType +import space.kscience.dataforge.io.asBinary +import space.kscience.dataforge.io.readObject +import space.kscience.dataforge.io.yaml.YamlMetaFormat +import space.kscience.dataforge.meta.Meta +import kotlin.reflect.KType +import kotlin.reflect.typeOf + +object SnarkYamlParser : SnarkParser { + override val contentType: ContentType = ContentType.Application.Json + override val fileExtensions: Set = setOf("yaml", "yml") + override val resultType: KType = typeOf() + + override suspend fun parse(bytes: ByteArray, meta: Meta): Meta = + YamlMetaFormat.readObject(bytes.asBinary()) +} \ No newline at end of file diff --git a/src/main/resources/magprog/content/contacts.md b/src/main/resources/magprog/content/contacts.md index 80951f6..4c64af9 100644 --- a/src/main/resources/magprog/content/contacts.md +++ b/src/main/resources/magprog/content/contacts.md @@ -5,10 +5,6 @@ section_title: Контакты language: ru --- -Сайт лаборатории: [https://npm.mipt.ru](/) - -Страница направления в JetBrains Research: [https://research.jetbrains.org/groups/npm/](https://research.jetbrains.org/groups/npm/). - Все вопросы можно задать в телеграм-канале лаборатории: [https://t.me/mipt_npm](https://t.me/mipt_npm). Также можно писать на электронную почту: npm@mipt.ru. diff --git a/src/main/resources/magprog/content/intro.md b/src/main/resources/magprog/content/intro.md index 429b97c..ac775e8 100644 --- a/src/main/resources/magprog/content/intro.md +++ b/src/main/resources/magprog/content/intro.md @@ -1,11 +1,11 @@ --- content_type: magprog magprog_section: intro -section_title: Магистерская программа +section_title: О программе language: ru --- -Магистерская программа МФТИ **«Разработка и применение программного обеспечения в физических исследованиях»** создана на базе [лаборатории методов ядерно-физических экспериментов (ЛМЯФЭ)](/) при поддержке двух школ МФТИ: Физтех-школы физики и исследований им. Ландау ([ЛФИ](https://mipt.ru/education/departments/lpr/)) и Физтех-школы прикладной математики и информатики ([ФПМИ](https://mipt.ru/education/departments/fpmi/)) и ряда академических и промышленных партнеров. В ее основе лежит взаимодействие студента и [научного руководителя](#mentors). +Магистерская программа МФТИ **"Научное программное обеспечение"** (старое название: **"Разработка и применение программного обеспечения в физических исследованиях"**) создана при поддержке двух школ МФТИ: Физтех-школы физики и исследований им. Ландау ([ЛФИ](https://mipt.ru/education/departments/lpr/)) и Физтех-школы прикладной математики и информатики ([ФПМИ](https://mipt.ru/education/departments/fpmi/)) и ряда академических и промышленных партнеров. В ее основе лежит взаимодействие студента и [научного руководителя](#mentors). Цель создания программы — объединение усилий ученых и программистов для разработки лучших компьютерных решений и применения этих решений в области фундаментальной и прикладной науки и инженерии. diff --git a/src/test/kotlin/ru/mipt/ApplicationTest.kt b/src/test/kotlin/ru/mipt/ApplicationTest.kt index 741df0d..0c730b3 100644 --- a/src/test/kotlin/ru/mipt/ApplicationTest.kt +++ b/src/test/kotlin/ru/mipt/ApplicationTest.kt @@ -3,6 +3,9 @@ package ru.mipt import io.ktor.client.request.get import io.ktor.http.HttpStatusCode import io.ktor.server.testing.testApplication +import ru.mipt.spc.magprog.magProgPage +import space.kscience.dataforge.context.Context +import space.kscience.snark.SnarkPlugin import java.nio.file.Path import kotlin.test.Test import kotlin.test.assertEquals @@ -10,8 +13,12 @@ import kotlin.test.assertEquals class ApplicationTest { @Test fun testRoot() = testApplication { + val context = Context("spc-site") { + plugin(SnarkPlugin) + } + application { - magProgPage(rootPath = Path.of(javaClass.getResource("/magprog")!!.toURI())) + magProgPage(context, rootPath = Path.of(javaClass.getResource("/magprog")!!.toURI())) } client.get("/magprog").apply { assertEquals(HttpStatusCode.OK, status)