From 3b318c3a8be11736b5af75233ffec7d28bbe49c9 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Fri, 19 Jan 2024 22:25:26 +0300 Subject: [PATCH] [WIP] refactor in progress --- build.gradle.kts | 2 +- .../space/kscience/snark/ReWrapAction.kt | 60 +++++++++++++ .../space/kscience/snark/TextProcessor.kt | 1 + .../space/kscience/snark/snarkWorkspace.kt | 4 - .../space/kscience/snark/html/HtmlPage.kt | 4 +- .../space/kscience/snark/html/SnarkHtml.kt | 90 +++++++++++-------- snark-ktor/build.gradle.kts | 20 +++-- .../kscience/snark/ktor/KtorSiteContext.kt | 0 .../space/kscience/snark/ktor/extractData.kt | 0 9 files changed, 130 insertions(+), 51 deletions(-) create mode 100644 snark-core/src/commonMain/kotlin/space/kscience/snark/ReWrapAction.kt rename snark-ktor/src/{main => jvmMain}/kotlin/space/kscience/snark/ktor/KtorSiteContext.kt (100%) rename snark-ktor/src/{main => jvmMain}/kotlin/space/kscience/snark/ktor/extractData.kt (100%) diff --git a/build.gradle.kts b/build.gradle.kts index a98f86b..5dd48e2 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -14,7 +14,7 @@ allprojects { } } -val dataforgeVersion by extra("0.7.1") +val dataforgeVersion by extra("0.8.0-dev-1") ksciencePublish { pom("https://github.com/SciProgCentre/snark") { diff --git a/snark-core/src/commonMain/kotlin/space/kscience/snark/ReWrapAction.kt b/snark-core/src/commonMain/kotlin/space/kscience/snark/ReWrapAction.kt new file mode 100644 index 0000000..7529f63 --- /dev/null +++ b/snark-core/src/commonMain/kotlin/space/kscience/snark/ReWrapAction.kt @@ -0,0 +1,60 @@ +package space.kscience.snark + +import space.kscience.dataforge.actions.AbstractAction +import space.kscience.dataforge.data.* +import space.kscience.dataforge.meta.Meta +import space.kscience.dataforge.meta.MutableMeta +import space.kscience.dataforge.meta.copy +import space.kscience.dataforge.names.Name +import space.kscience.dataforge.names.NameToken +import space.kscience.dataforge.names.replaceLast +import kotlin.reflect.KType +import kotlin.reflect.typeOf + +/** + * An action to change header (name and meta) without changing the data itself or its computation state + */ +public class ReWrapAction( + type: KType, + private val newMeta: MutableMeta.(name: Name) -> Unit = {}, + private val newName: (name: Name, meta: Meta?) -> Name, +) : AbstractAction(type) { + override fun DataSetBuilder.generate(data: DataSet, meta: Meta) { + data.forEach { namedData -> + data( + newName(namedData.name, namedData.meta), + namedData.data.withMeta(namedData.meta.copy { newMeta(namedData.name) }) + ) + } + } + + override fun DataSourceBuilder.update(dataSet: DataSet, meta: Meta, updateKey: Name) { + val datum = dataSet[updateKey] + data( + newName(updateKey, datum?.meta), + datum?.withMeta(datum.meta.copy { newMeta(updateKey) }) + ) + } + + public companion object { + public inline fun removeExtensions( + vararg bypassExtensions: String, + noinline newMeta: MutableMeta.(name: Name) -> Unit = {}, + ): ReWrapAction = ReWrapAction(typeOf(), newMeta = newMeta) { name, _ -> + name.replaceLast { token -> + val extension = token.body.substringAfterLast('.') + if (extension in bypassExtensions) { + NameToken(token.body.removeSuffix(".$extension")) + } else { + token + } + } + } + } +} + +public inline fun ReWrapAction( + noinline newMeta: MutableMeta.(name: Name) -> Unit = {}, + noinline newName: (Name, Meta?) -> Name +): ReWrapAction = ReWrapAction(typeOf(), newMeta, newName) + diff --git a/snark-core/src/commonMain/kotlin/space/kscience/snark/TextProcessor.kt b/snark-core/src/commonMain/kotlin/space/kscience/snark/TextProcessor.kt index 26a01d4..5808a21 100644 --- a/snark-core/src/commonMain/kotlin/space/kscience/snark/TextProcessor.kt +++ b/snark-core/src/commonMain/kotlin/space/kscience/snark/TextProcessor.kt @@ -14,6 +14,7 @@ public fun interface TextProcessor { public companion object { public const val DF_TYPE: String = "snark.textTransformation" public val TEXT_TRANSFORMATION_KEY: NameToken = NameToken("transformation") + public val TEXT_PREPROCESSOR_KEY: NameToken = NameToken("preprocessor") } } diff --git a/snark-core/src/jvmMain/kotlin/space/kscience/snark/snarkWorkspace.kt b/snark-core/src/jvmMain/kotlin/space/kscience/snark/snarkWorkspace.kt index 2b66345..9a7a9a9 100644 --- a/snark-core/src/jvmMain/kotlin/space/kscience/snark/snarkWorkspace.kt +++ b/snark-core/src/jvmMain/kotlin/space/kscience/snark/snarkWorkspace.kt @@ -2,14 +2,12 @@ package space.kscience.snark -import space.kscience.dataforge.data.DataSet import space.kscience.dataforge.data.DataTree import space.kscience.dataforge.data.node import space.kscience.dataforge.io.Binary import space.kscience.dataforge.io.IOPlugin import space.kscience.dataforge.meta.* import space.kscience.dataforge.misc.DFExperimental -import space.kscience.dataforge.names.Name import space.kscience.dataforge.workspace.Workspace import space.kscience.dataforge.workspace.WorkspaceBuilder import space.kscience.dataforge.workspace.readRawDirectory @@ -39,13 +37,11 @@ private fun IOPlugin.readResources( public fun Snark.workspace( meta: Meta, - customData: DataSet<*> = DataSet.EMPTY, workspaceBuilder: WorkspaceBuilder.() -> Unit = {}, ): Workspace = Workspace { data { - node(Name.EMPTY, customData) meta.getIndexed("directory").forEach { (index, directoryMeta) -> val dataDirectory = directoryMeta["path"].string ?: error("Directory path not defined") val nodeName = directoryMeta["name"].string ?: directoryMeta.string ?: index ?: "" diff --git a/snark-html/src/jvmMain/kotlin/space/kscience/snark/html/HtmlPage.kt b/snark-html/src/jvmMain/kotlin/space/kscience/snark/html/HtmlPage.kt index 3f865fc..20acdad 100644 --- a/snark-html/src/jvmMain/kotlin/space/kscience/snark/html/HtmlPage.kt +++ b/snark-html/src/jvmMain/kotlin/space/kscience/snark/html/HtmlPage.kt @@ -11,8 +11,8 @@ import space.kscience.dataforge.names.Name public fun interface HtmlPage { - context(PageContextWithData) - public fun HTML.renderPage() + context(PageContextWithData, HTML) + public fun renderPage() public companion object { public fun createHtmlString( diff --git a/snark-html/src/jvmMain/kotlin/space/kscience/snark/html/SnarkHtml.kt b/snark-html/src/jvmMain/kotlin/space/kscience/snark/html/SnarkHtml.kt index 775f45f..23ebbc1 100644 --- a/snark-html/src/jvmMain/kotlin/space/kscience/snark/html/SnarkHtml.kt +++ b/snark-html/src/jvmMain/kotlin/space/kscience/snark/html/SnarkHtml.kt @@ -3,8 +3,10 @@ package space.kscience.snark.html import io.ktor.http.ContentType +import kotlinx.coroutines.CoroutineScope import kotlinx.io.readByteArray import space.kscience.dataforge.actions.Action +import space.kscience.dataforge.actions.mapping import space.kscience.dataforge.context.Context import space.kscience.dataforge.context.PluginFactory import space.kscience.dataforge.context.PluginTag @@ -13,27 +15,27 @@ import space.kscience.dataforge.io.* import space.kscience.dataforge.io.yaml.YamlMetaFormat import space.kscience.dataforge.io.yaml.YamlPlugin import space.kscience.dataforge.meta.Meta -import space.kscience.dataforge.meta.copy import space.kscience.dataforge.meta.get +import space.kscience.dataforge.meta.set import space.kscience.dataforge.meta.string import space.kscience.dataforge.misc.DFExperimental import space.kscience.dataforge.names.Name import space.kscience.dataforge.names.asName -import space.kscience.dataforge.provider.dfId +import space.kscience.dataforge.provider.dfType import space.kscience.dataforge.workspace.* +import space.kscience.snark.ReWrapAction import space.kscience.snark.Snark import space.kscience.snark.SnarkReader import space.kscience.snark.TextProcessor import java.net.URLConnection import kotlin.io.path.Path import kotlin.io.path.extension -import kotlin.reflect.typeOf public fun DataSet.transform(action: Action, meta: Meta = Meta.EMPTY): DataSet = action.execute(this, meta) -public fun TaskResultBuilder.fill(dataSet: DataSet) { +public fun DataSetBuilder.fill(dataSet: DataSet) { node(Name.EMPTY, dataSet) } @@ -48,7 +50,7 @@ public class SnarkHtml : WorkspacePlugin() { override val tag: PluginTag get() = Companion.tag override fun content(target: String): Map = when (target) { - SnarkReader::class.dfId -> mapOf( + SnarkReader::class.dfType -> mapOf( "html".asName() to HtmlReader, "markdown".asName() to MarkdownReader, "json".asName() to SnarkReader(JsonMetaFormat, ContentType.Application.Json.toString()), @@ -64,52 +66,55 @@ public class SnarkHtml : WorkspacePlugin() { } - public val parse: TaskReference by task { - from(allData).forEach { (dataName, data) -> - val contentType = getContentType(dataName, data.meta) - val parser = snark.readers.values.filter { parser -> - contentType in parser.types - }.maxByOrNull { - it.priority - } ?: return@forEach //ignore data for which parser is not found + public val prepareHeaderAction: Action = ReWrapAction.removeExtensions("html", "md") { name -> + val contentType = getContentType(name, this) + set(CONTENT_TYPE_KEY, contentType) + } - val preprocessor = meta[TextProcessor.TEXT_TRANSFORMATION_KEY]?.let { snark.preprocessor(it) } + public val parseAction: Action = Action.mapping { + val contentType = getContentType(name, meta) - val newMeta = data.meta.copy { - CONTENT_TYPE_KEY put contentType - } + val parser = snark.readers.values.filter { parser -> + contentType in parser.types + }.maxByOrNull { + it.priority + } + //pass data for which parser is not found + if (parser == null) { + result { it } + } else { - when (data.type) { - typeOf() -> { - data(dataName, data.map { content -> - val string = content as String + val preprocessor = meta[TextProcessor.TEXT_PREPROCESSOR_KEY]?.let { snark.preprocessor(it) } + result { + + when (it) { + is CharSequence -> { + val string = it.toString() val preprocessed = preprocessor?.process(string) ?: string parser.readFrom(preprocessed) - }) - } + } - typeOf() -> { - data(dataName, data.map(meta = newMeta) { content -> - val binary = content as Binary + is Binary -> { if (preprocessor == null) { - parser.readFrom(binary) + parser.readFrom(it) } else { //TODO provide encoding - val string = binary.toByteArray().decodeToString() + val string = it.toByteArray().decodeToString() parser.readFrom(preprocessor.process(string)) } - }) + } + + // bypass for non textual-data + else -> it } - // bypass for non textual-data - else -> data(dataName, data.withMeta(newMeta)) } } } - - public val site: TaskReference by task { - fill(from(allData)) - fill(from(parse)) + public val parse: TaskReference by task({ + description = "Parse all data for which reader is resolved" + }) { + fill(from(allData).transform(parseAction, taskMeta)) } @@ -128,3 +133,18 @@ public class SnarkHtml : WorkspacePlugin() { } } + + +public fun SnarkHtml.readSiteData( + binaries: DataSource, + meta: Meta = Meta.EMPTY, +): DataSet = binaries.transform(parseAction, meta) + + +public fun SnarkHtml.readSiteData( + coroutineScope: CoroutineScope, + meta: Meta = Meta.EMPTY, + builder: context(IOPlugin) DataSourceBuilder.() -> Unit, +): DataSet = DataSource(coroutineScope) { builder(io, this) } + .transform(prepareHeaderAction, meta) + .transform(parseAction, meta) \ No newline at end of file diff --git a/snark-ktor/build.gradle.kts b/snark-ktor/build.gradle.kts index 7969548..3105f2d 100644 --- a/snark-ktor/build.gradle.kts +++ b/snark-ktor/build.gradle.kts @@ -1,5 +1,5 @@ plugins { - id("space.kscience.gradle.jvm") + id("space.kscience.gradle.mpp") `maven-publish` } @@ -7,15 +7,17 @@ val dataforgeVersion: String by rootProject.extra val ktorVersion = space.kscience.gradle.KScienceVersions.ktorVersion kscience{ + jvm() useContextReceivers() -} -dependencies { - api(projects.snarkHtml) + jvmMain{ + api(projects.snarkHtml) - api("io.ktor:ktor-server-core:$ktorVersion") - api("io.ktor:ktor-server-html-builder:$ktorVersion") - api("io.ktor:ktor-server-host-common:$ktorVersion") - - testApi("io.ktor:ktor-server-tests:$ktorVersion") + api("io.ktor:ktor-server-core:$ktorVersion") + api("io.ktor:ktor-server-html-builder:$ktorVersion") + api("io.ktor:ktor-server-host-common:$ktorVersion") + } + jvmTest{ + api("io.ktor:ktor-server-tests:$ktorVersion") + } } \ No newline at end of file diff --git a/snark-ktor/src/main/kotlin/space/kscience/snark/ktor/KtorSiteContext.kt b/snark-ktor/src/jvmMain/kotlin/space/kscience/snark/ktor/KtorSiteContext.kt similarity index 100% rename from snark-ktor/src/main/kotlin/space/kscience/snark/ktor/KtorSiteContext.kt rename to snark-ktor/src/jvmMain/kotlin/space/kscience/snark/ktor/KtorSiteContext.kt diff --git a/snark-ktor/src/main/kotlin/space/kscience/snark/ktor/extractData.kt b/snark-ktor/src/jvmMain/kotlin/space/kscience/snark/ktor/extractData.kt similarity index 100% rename from snark-ktor/src/main/kotlin/space/kscience/snark/ktor/extractData.kt rename to snark-ktor/src/jvmMain/kotlin/space/kscience/snark/ktor/extractData.kt