diff --git a/build.gradle.kts b/build.gradle.kts index 5dd48e2..157cdae 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -1,4 +1,5 @@ -import space.kscience.gradle.* +import space.kscience.gradle.useApache2Licence +import space.kscience.gradle.useSPCTeam plugins { id("space.kscience.gradle.project") @@ -14,7 +15,7 @@ allprojects { } } -val dataforgeVersion by extra("0.8.0-dev-1") +val dataforgeVersion by extra("0.8.0") ksciencePublish { pom("https://github.com/SciProgCentre/snark") { diff --git a/gradle.properties b/gradle.properties index b10d32e..a1c9725 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,3 +1,3 @@ kotlin.code.style=official -toolsVersion=0.15.2-kotlin-1.9.21 \ No newline at end of file +toolsVersion=0.15.2-kotlin-1.9.22 \ No newline at end of file diff --git a/snark-core/src/commonMain/kotlin/space/kscience/snark/ReWrapAction.kt b/snark-core/src/commonMain/kotlin/space/kscience/snark/ReWrapAction.kt index 7529f63..e8fe2b9 100644 --- a/snark-core/src/commonMain/kotlin/space/kscience/snark/ReWrapAction.kt +++ b/snark-core/src/commonMain/kotlin/space/kscience/snark/ReWrapAction.kt @@ -5,9 +5,7 @@ import space.kscience.dataforge.data.* import space.kscience.dataforge.meta.Meta import space.kscience.dataforge.meta.MutableMeta import space.kscience.dataforge.meta.copy -import space.kscience.dataforge.names.Name -import space.kscience.dataforge.names.NameToken -import space.kscience.dataforge.names.replaceLast +import space.kscience.dataforge.names.* import kotlin.reflect.KType import kotlin.reflect.typeOf @@ -19,20 +17,19 @@ public class ReWrapAction( private val newMeta: MutableMeta.(name: Name) -> Unit = {}, private val newName: (name: Name, meta: Meta?) -> Name, ) : AbstractAction(type) { - override fun DataSetBuilder.generate(data: DataSet, meta: Meta) { + override fun DataSink.generate(data: DataTree, meta: Meta) { data.forEach { namedData -> - data( + put( newName(namedData.name, namedData.meta), namedData.data.withMeta(namedData.meta.copy { newMeta(namedData.name) }) ) } } - override fun DataSourceBuilder.update(dataSet: DataSet, meta: Meta, updateKey: Name) { - val datum = dataSet[updateKey] - data( - newName(updateKey, datum?.meta), - datum?.withMeta(datum.meta.copy { newMeta(updateKey) }) + override fun DataSink.update(source: DataTree, meta: Meta, namedData: NamedData) { + put( + newName(namedData.name, namedData.meta), + namedData.withMeta(namedData.meta.copy { newMeta(namedData.name) }) ) } @@ -50,11 +47,15 @@ public class ReWrapAction( } } } + + public inline fun removeIndex(): ReWrapAction = ReWrapAction(typeOf()) { name, _ -> + if (name.endsWith("index")) name.cutLast() else name + } } } public inline fun ReWrapAction( noinline newMeta: MutableMeta.(name: Name) -> Unit = {}, - noinline newName: (Name, Meta?) -> Name -): ReWrapAction = ReWrapAction(typeOf(), newMeta, newName) + noinline newName: (Name, Meta?) -> Name, +): ReWrapAction = ReWrapAction(typeOf(), newMeta, newName) diff --git a/snark-core/src/jvmMain/kotlin/space/kscience/snark/ImageIOReader.kt b/snark-core/src/jvmMain/kotlin/space/kscience/snark/ImageIOReader.kt index 59d4eb6..bc32f3a 100644 --- a/snark-core/src/jvmMain/kotlin/space/kscience/snark/ImageIOReader.kt +++ b/snark-core/src/jvmMain/kotlin/space/kscience/snark/ImageIOReader.kt @@ -5,8 +5,6 @@ import kotlinx.io.asInputStream import space.kscience.dataforge.io.IOReader import java.awt.image.BufferedImage import javax.imageio.ImageIO -import kotlin.reflect.KType -import kotlin.reflect.typeOf /** * The ImageIOReader class is an implementation of the IOReader interface specifically for reading images using the ImageIO library. @@ -15,7 +13,5 @@ import kotlin.reflect.typeOf * @property type The KType of the data to be read by the ImageIOReader. */ public object ImageIOReader : IOReader { - override val type: KType get() = typeOf() - override fun readFrom(source: Source): BufferedImage = ImageIO.read(source.asInputStream()) } \ No newline at end of file diff --git a/snark-core/src/jvmMain/kotlin/space/kscience/snark/extensions.kt b/snark-core/src/jvmMain/kotlin/space/kscience/snark/extensions.kt index 24976bc..865519f 100644 --- a/snark-core/src/jvmMain/kotlin/space/kscience/snark/extensions.kt +++ b/snark-core/src/jvmMain/kotlin/space/kscience/snark/extensions.kt @@ -1,16 +1,12 @@ package space.kscience.snark -import space.kscience.dataforge.data.DataSet + +import space.kscience.dataforge.data.DataSource +import space.kscience.dataforge.data.DataTree import space.kscience.dataforge.data.filterByType import space.kscience.dataforge.names.Name -import space.kscience.dataforge.names.parseAsName import space.kscience.dataforge.names.startsWith -import kotlin.reflect.typeOf -public inline fun DataSet<*>.branch( +public inline fun DataTree<*>.branch( branchName: Name, -): DataSet = filterByType(typeOf()) { name, _ -> name.startsWith(branchName) } - -public inline fun DataSet<*>.branch( - branchName: String, -): DataSet = filterByType(typeOf()) { name, _ -> name.startsWith(branchName.parseAsName()) } \ No newline at end of file +): DataSource = filterByType { name, _, _ -> name.startsWith(branchName) } diff --git a/snark-core/src/jvmMain/kotlin/space/kscience/snark/snarkWorkspace.kt b/snark-core/src/jvmMain/kotlin/space/kscience/snark/snarkWorkspace.kt index 9a7a9a9..06224b6 100644 --- a/snark-core/src/jvmMain/kotlin/space/kscience/snark/snarkWorkspace.kt +++ b/snark-core/src/jvmMain/kotlin/space/kscience/snark/snarkWorkspace.kt @@ -2,38 +2,36 @@ package space.kscience.snark -import space.kscience.dataforge.data.DataTree -import space.kscience.dataforge.data.node -import space.kscience.dataforge.io.Binary -import space.kscience.dataforge.io.IOPlugin +import space.kscience.dataforge.data.branch import space.kscience.dataforge.meta.* import space.kscience.dataforge.misc.DFExperimental +import space.kscience.dataforge.names.parseAsName import space.kscience.dataforge.workspace.Workspace import space.kscience.dataforge.workspace.WorkspaceBuilder -import space.kscience.dataforge.workspace.readRawDirectory +import space.kscience.dataforge.workspace.directory +import space.kscience.dataforge.workspace.resources import kotlin.io.path.Path -import kotlin.io.path.toPath - -/** - * Reads the specified resources and returns a [DataTree] containing the data. - * - * @param resources The names of the resources to read. - * @param classLoader The class loader to use for loading the resources. By default, it uses the current thread's context class loader. - * @return A DataTree containing the data read from the resources. - */ -private fun IOPlugin.readResources( - vararg resources: String, - classLoader: ClassLoader = Thread.currentThread().contextClassLoader, -): DataTree = DataTree { - // require(resource.isNotBlank()) {"Can't mount root resource tree as data root"} - resources.forEach { resource -> - val path = classLoader.getResource(resource)?.toURI()?.toPath() ?: error( - "Resource with name $resource is not resolved" - ) - node(resource, readRawDirectory(path)) - } -} +// +///** +// * Reads the specified resources and returns a [DataTree] containing the data. +// * +// * @param resources The names of the resources to read. +// * @param classLoader The class loader to use for loading the resources. By default, it uses the current thread's context class loader. +// * @return A DataTree containing the data read from the resources. +// */ +//private fun IOPlugin.readResources( +// vararg resources: String, +// classLoader: ClassLoader = Thread.currentThread().contextClassLoader, +//): DataTree = DataTree { +// // require(resource.isNotBlank()) {"Can't mount root resource tree as data root"} +// resources.forEach { resource -> +// val path = classLoader.getResource(resource)?.toURI()?.toPath() ?: error( +// "Resource with name $resource is not resolved" +// ) +// node(resource, readRawDirectory(path)) +// } +//} public fun Snark.workspace( meta: Meta, @@ -45,14 +43,14 @@ public fun Snark.workspace( meta.getIndexed("directory").forEach { (index, directoryMeta) -> val dataDirectory = directoryMeta["path"].string ?: error("Directory path not defined") val nodeName = directoryMeta["name"].string ?: directoryMeta.string ?: index ?: "" - val data = io.readRawDirectory(Path(dataDirectory)) - node(nodeName, data) + directory(io, nodeName.parseAsName(), Path((dataDirectory))) } meta.getIndexed("resource").forEach { (index, resourceMeta) -> val resource = resourceMeta["path"]?.stringList ?: listOf("/") val nodeName = resourceMeta["name"].string ?: resourceMeta.string ?: index ?: "" - val data: DataTree = io.readResources(*resource.toTypedArray()) - node(nodeName, data) + branch(nodeName) { + resources(io, *resource.toTypedArray()) + } } } diff --git a/snark-html/src/jvmMain/kotlin/space/kscience/snark/html/HtmlPage.kt b/snark-html/src/jvmMain/kotlin/space/kscience/snark/html/HtmlPage.kt index 20acdad..e192dbb 100644 --- a/snark-html/src/jvmMain/kotlin/space/kscience/snark/html/HtmlPage.kt +++ b/snark-html/src/jvmMain/kotlin/space/kscience/snark/html/HtmlPage.kt @@ -3,9 +3,9 @@ package space.kscience.snark.html import kotlinx.html.HTML import kotlinx.html.stream.createHTML import kotlinx.html.visitTagAndFinalize -import space.kscience.dataforge.data.DataSet -import space.kscience.dataforge.data.DataSetBuilder -import space.kscience.dataforge.data.static +import space.kscience.dataforge.data.DataSink +import space.kscience.dataforge.data.DataTree +import space.kscience.dataforge.data.wrap import space.kscience.dataforge.meta.Meta import space.kscience.dataforge.names.Name @@ -17,11 +17,11 @@ public fun interface HtmlPage { public companion object { public fun createHtmlString( pageContext: PageContext, - dataSet: DataSet<*>, + dataSet: DataTree<*>?, page: HtmlPage, ): String = createHTML().run { HTML(kotlinx.html.emptyMap, this, null).visitTagAndFinalize(this) { - with(PageContextWithData(pageContext, dataSet)) { + with(PageContextWithData(pageContext, dataSet ?: DataTree.EMPTY)) { with(page) { renderPage() } @@ -34,13 +34,13 @@ public fun interface HtmlPage { // data builders -public fun DataSetBuilder.page( +public fun DataSink.page( name: Name, pageMeta: Meta = Meta.EMPTY, block: context(PageContextWithData) HTML.() -> Unit, ) { val page = HtmlPage(block) - static(name, page, pageMeta) + wrap(name, page, pageMeta) } diff --git a/snark-html/src/jvmMain/kotlin/space/kscience/snark/html/HtmlSite.kt b/snark-html/src/jvmMain/kotlin/space/kscience/snark/html/HtmlSite.kt index 1f908a7..466ea6a 100644 --- a/snark-html/src/jvmMain/kotlin/space/kscience/snark/html/HtmlSite.kt +++ b/snark-html/src/jvmMain/kotlin/space/kscience/snark/html/HtmlSite.kt @@ -1,8 +1,8 @@ package space.kscience.snark.html -import space.kscience.dataforge.data.DataSet -import space.kscience.dataforge.data.DataSetBuilder -import space.kscience.dataforge.data.static +import space.kscience.dataforge.data.DataSink +import space.kscience.dataforge.data.DataTree +import space.kscience.dataforge.data.wrap import space.kscience.dataforge.meta.Meta import space.kscience.dataforge.meta.getIndexed import space.kscience.dataforge.meta.string @@ -15,23 +15,23 @@ public fun interface HtmlSite { public fun renderSite() } -public fun DataSetBuilder.site( +public fun DataSink.site( name: Name, siteMeta: Meta, - block: (siteContext: SiteContext, data: DataSet) -> Unit, + block: (siteContext: SiteContext, data: DataTree<*>?) -> Unit, ) { - static(name, HtmlSite { block(site, siteData) }, siteMeta) + wrap(name, HtmlSite { block(site, siteData) }, siteMeta) } //public fun DataSetBuilder.site(name: Name, block: DataSetBuilder.() -> Unit) { // node(name, block) //} -internal fun DataSetBuilder.assetsFrom(rootMeta: Meta) { +internal fun DataSink.assetsFrom(rootMeta: Meta) { rootMeta.getIndexed("file".asName()).forEach { (_, meta) -> val webName: String? by meta.string() val name by meta.string { error("File path is not provided") } val fileName = name.parseAsName() - static(fileName, webName?.parseAsName() ?: fileName) + wrap(fileName, webName?.parseAsName() ?: fileName) } } \ No newline at end of file diff --git a/snark-html/src/jvmMain/kotlin/space/kscience/snark/html/Language.kt b/snark-html/src/jvmMain/kotlin/space/kscience/snark/html/Language.kt index 9b16f2f..17adc70 100644 --- a/snark-html/src/jvmMain/kotlin/space/kscience/snark/html/Language.kt +++ b/snark-html/src/jvmMain/kotlin/space/kscience/snark/html/Language.kt @@ -1,15 +1,14 @@ package space.kscience.snark.html -import space.kscience.dataforge.data.DataSet -import space.kscience.dataforge.data.branch +import space.kscience.dataforge.actions.AbstractAction +import space.kscience.dataforge.actions.transform +import space.kscience.dataforge.data.* import space.kscience.dataforge.meta.* -import space.kscience.dataforge.names.Name -import space.kscience.dataforge.names.asName -import space.kscience.dataforge.names.parseAsName -import space.kscience.dataforge.names.plus +import space.kscience.dataforge.names.* import space.kscience.snark.SnarkBuilder import space.kscience.snark.html.Language.Companion.SITE_LANGUAGES_KEY import space.kscience.snark.html.Language.Companion.SITE_LANGUAGE_KEY +import kotlin.reflect.typeOf public class Language : Scheme() { @@ -81,9 +80,37 @@ public val SiteContext.languages: Map public val SiteContext.language: String get() = siteMeta[SITE_LANGUAGE_KEY].string ?: Language.DEFAULT_LANGUAGE +// +//public val SiteContext.languagePrefix: Name +// get() = languages[language]?.let { it[Language::prefix.name].string ?: language }?.parseAsName() ?: Name.EMPTY -public val SiteContext.languagePrefix: Name - get() = languages[language]?.let { it[Language::prefix.name].string ?: language }?.parseAsName() ?: Name.EMPTY + +private class LanguageBranchAction(val prefix: Name) : AbstractAction(typeOf()) { + override fun DataSink.generate(data: DataTree, meta: Meta) { + data.forEach { (name, item) -> + val nameWithoutPrefix = name.removeFirstOrNull(prefix) + if (nameWithoutPrefix != null) { + // put language item as is + put(nameWithoutPrefix, item) + } else if (data[name + prefix] == null) { + // put if language item is missing + put(name, item) + } + } + } + + override fun DataSink.update(source: DataTree, meta: Meta, namedData: NamedData) { + val name = namedData.name + val nameWithoutPrefix = name.removeFirstOrNull(prefix) + if (nameWithoutPrefix != null) { + // put language item as is + put(nameWithoutPrefix, namedData.data) + } else if (source[name + prefix] == null) { + // put if language item is missing + put(name, namedData.data) + } + } +} /** * Create a multiple sites for different languages. All sites use the same [content], but rely on different data @@ -92,7 +119,7 @@ public val SiteContext.languagePrefix: Name */ @SnarkBuilder public fun SiteContext.multiLanguageSite( - data: DataSet<*>, + data: DataTree<*>, languageMap: Map, content: HtmlSite, ) { @@ -108,7 +135,7 @@ public fun SiteContext.multiLanguageSite( } site( prefix.parseAsName(), - data.branch(language.dataPath ?: prefix), + data.filterByType().transform(LanguageBranchAction(Name.parse(language.dataPath ?: prefix))), siteMeta = Laminate(languageSiteMeta, siteMeta), content ) diff --git a/snark-html/src/jvmMain/kotlin/space/kscience/snark/html/PageContext.kt b/snark-html/src/jvmMain/kotlin/space/kscience/snark/html/PageContext.kt index 32ea5a1..ea8f124 100644 --- a/snark-html/src/jvmMain/kotlin/space/kscience/snark/html/PageContext.kt +++ b/snark-html/src/jvmMain/kotlin/space/kscience/snark/html/PageContext.kt @@ -1,6 +1,6 @@ package space.kscience.snark.html -import space.kscience.dataforge.data.DataSet +import space.kscience.dataforge.data.DataTree import space.kscience.dataforge.meta.Meta import space.kscience.dataforge.meta.get import space.kscience.dataforge.meta.string @@ -23,7 +23,7 @@ public fun Name.toWebPath(): String = tokens.joinToString(separator = "/") { * A context for building a single page */ @SnarkBuilder -public interface PageContext : SnarkContext { +public interface PageContext : SnarkContext { public val site: SiteContext @@ -57,4 +57,7 @@ public val PageContext.homeRef: String get() = resolvePageRef(SiteContext.INDEX_ public val PageContext.name: Name? get() = pageMeta["name"].string?.parseAsName() -public class PageContextWithData(private val pageContext: PageContext, public val data: DataSet<*>): PageContext by pageContext \ No newline at end of file +public class PageContextWithData( + private val pageContext: PageContext, + public val data: DataTree<*>, +) : PageContext by pageContext \ No newline at end of file diff --git a/snark-html/src/jvmMain/kotlin/space/kscience/snark/html/PageFragment.kt b/snark-html/src/jvmMain/kotlin/space/kscience/snark/html/PageFragment.kt index eabed33..0d37bc3 100644 --- a/snark-html/src/jvmMain/kotlin/space/kscience/snark/html/PageFragment.kt +++ b/snark-html/src/jvmMain/kotlin/space/kscience/snark/html/PageFragment.kt @@ -15,20 +15,19 @@ import space.kscience.snark.SnarkContext public fun interface PageFragment { - context(PageContextWithData) - public fun FlowContent.renderFragment() + context(PageContextWithData, FlowContent) public fun renderFragment() } -context(PageContextWithData) -public fun FlowContent.fragment(fragment: PageFragment): Unit{ +context(PageContextWithData, FlowContent) +public fun fragment(fragment: PageFragment): Unit { with(fragment) { renderFragment() } } -context(PageContextWithData) -public fun FlowContent.fragment(data: Data): Unit = runBlocking { +context(PageContextWithData, FlowContent) +public fun fragment(data: Data): Unit = runBlocking { fragment(data.await()) } @@ -54,7 +53,7 @@ public val Data<*>.published: Boolean * Resolve a Html builder by its full name */ context(SnarkContext) -public fun DataSet<*>.resolveHtmlOrNull(name: Name): Data? { +public fun DataTree<*>.resolveHtmlOrNull(name: Name): Data? { val resolved = (getByType(name) ?: getByType(name + SiteContext.INDEX_PAGE_TOKEN)) return resolved?.takeIf { @@ -63,26 +62,26 @@ public fun DataSet<*>.resolveHtmlOrNull(name: Name): Data? { } context(SnarkContext) -public fun DataSet<*>.resolveHtmlOrNull(name: String): Data? = resolveHtmlOrNull(name.parseAsName()) +public fun DataTree<*>.resolveHtmlOrNull(name: String): Data? = resolveHtmlOrNull(name.parseAsName()) context(SnarkContext) -public fun DataSet<*>.resolveHtml(name: String): Data = resolveHtmlOrNull(name) +public fun DataTree<*>.resolveHtml(name: String): Data = resolveHtmlOrNull(name) ?: error("Html fragment with name $name is not resolved") /** * Find all Html blocks using given name/meta filter */ context(SnarkContext) -public fun DataSet<*>.resolveAllHtml( +public fun DataTree<*>.resolveAllHtml( predicate: (name: Name, meta: Meta) -> Boolean, -): Map> = filterByType { name, meta -> +): Map> = filterByType { name, meta, _ -> predicate(name, meta) && meta["published"].string != "false" //TODO add language confirmation }.asSequence().associate { it.name to it.data } context(SnarkContext) -public fun DataSet<*>.findHtmlByContentType( +public fun DataTree<*>.findHtmlByContentType( contentType: String, baseName: Name = Name.EMPTY, ): Map> = resolveAllHtml { name, meta -> diff --git a/snark-html/src/jvmMain/kotlin/space/kscience/snark/html/ParseAction.kt b/snark-html/src/jvmMain/kotlin/space/kscience/snark/html/ParseAction.kt new file mode 100644 index 0000000..1b662de --- /dev/null +++ b/snark-html/src/jvmMain/kotlin/space/kscience/snark/html/ParseAction.kt @@ -0,0 +1,50 @@ +package space.kscience.snark.html + +import space.kscience.dataforge.actions.AbstractAction +import space.kscience.dataforge.data.* +import space.kscience.dataforge.io.Binary +import space.kscience.dataforge.io.toByteArray +import space.kscience.dataforge.meta.Meta +import space.kscience.dataforge.meta.get +import space.kscience.snark.TextProcessor +import kotlin.reflect.typeOf + +public class ParseAction(private val snarkHtml: SnarkHtml) : + AbstractAction(typeOf()) { + + private fun parseOne(data: NamedData, actionMeta: Meta): NamedData? = with(snarkHtml) { + val contentType = getContentType(data.name, data.meta) + + val parser = snark.readers.values.filterIsInstance().filter { parser -> + contentType in parser.types + }.maxByOrNull { + it.priority + } + + //ignore data for which parser is not found + if (parser != null) { + val preprocessor = meta[TextProcessor.TEXT_PREPROCESSOR_KEY]?.let { snark.preprocessor(it) } + data.transform { + if (preprocessor == null) { + parser.readFrom(it) + } else { + //TODO provide encoding + val string = it.toByteArray().decodeToString() + parser.readFrom(preprocessor.process(string)) + } + }.named(data.name) + } else { + null + } + } + + override fun DataSink.generate(data: DataTree, meta: Meta) { + data.forEach { + parseOne(it, meta)?.let { put(it) } + } + } + + override fun DataSink.update(source: DataTree, meta: Meta, namedData: NamedData) { + parseOne(namedData,meta)?.let { put(it) } + } +} \ No newline at end of file diff --git a/snark-html/src/jvmMain/kotlin/space/kscience/snark/html/SiteContext.kt b/snark-html/src/jvmMain/kotlin/space/kscience/snark/html/SiteContext.kt index f3e7a88..96d92c1 100644 --- a/snark-html/src/jvmMain/kotlin/space/kscience/snark/html/SiteContext.kt +++ b/snark-html/src/jvmMain/kotlin/space/kscience/snark/html/SiteContext.kt @@ -8,8 +8,6 @@ import space.kscience.dataforge.meta.string import space.kscience.dataforge.names.* import space.kscience.snark.SnarkBuilder import space.kscience.snark.SnarkContext -import kotlin.reflect.full.isSubtypeOf -import kotlin.reflect.typeOf /** @@ -45,7 +43,7 @@ public interface SiteContext : SnarkContext { @SnarkBuilder public fun page( route: Name, - data: DataSet<*>, + data: DataTree<*>?, pageMeta: Meta = Meta.EMPTY, content: HtmlPage, ) @@ -56,7 +54,7 @@ public interface SiteContext : SnarkContext { @SnarkBuilder public fun route( route: Name, - data: DataSet<*>, + data: DataTree<*>?, siteMeta: Meta = Meta.EMPTY, content: HtmlSite, ) @@ -68,7 +66,7 @@ public interface SiteContext : SnarkContext { @SnarkBuilder public fun site( route: Name, - data: DataSet<*>, + data: DataTree<*>?, siteMeta: Meta = Meta.EMPTY, content: HtmlSite, ) @@ -81,27 +79,21 @@ public interface SiteContext : SnarkContext { } } -public fun SiteContext.static(dataSet: DataSet, prefix: Name = Name.EMPTY) { +public fun SiteContext.static(dataSet: DataTree, prefix: Name = Name.EMPTY) { dataSet.forEach { (name, data) -> static(prefix + name, data) } } - -public fun SiteContext.static(dataSet: DataSet<*>, branch: String, prefix: String = branch) { +public fun SiteContext.static(dataSet: DataTree<*>, branch: String, prefix: String = branch) { val branchName = branch.parseAsName() val prefixName = prefix.parseAsName() - val binaryType = typeOf() - dataSet.forEach { (name, data) -> - @Suppress("UNCHECKED_CAST") - if (name.startsWith(branchName) && data.type.isSubtypeOf(binaryType)) { - static(prefixName + name, data as Data) - } + dataSet.branch(branchName)?.filterByType()?.forEach { + static(prefixName + it.name, it.data) } } - context(SiteContext) public val site: SiteContext get() = this@SiteContext @@ -110,7 +102,7 @@ public val site: SiteContext /** * A wrapper for site context that allows convenient site building experience */ -public class SiteContextWithData(private val site: SiteContext, public val siteData: DataSet<*>) : SiteContext by site +public class SiteContextWithData(private val site: SiteContext, public val siteData: DataTree<*>) : SiteContext by site @SnarkBuilder @@ -127,23 +119,23 @@ public fun SiteContextWithData.page( @SnarkBuilder public fun SiteContextWithData.route( route: String, - data: DataSet<*> = siteData, + data: DataTree<*>? = siteData, siteMeta: Meta = Meta.EMPTY, content: HtmlSite, -): Unit = route(route.parseAsName(), data, siteMeta,content) +): Unit = route(route.parseAsName(), data, siteMeta, content) @SnarkBuilder public fun SiteContextWithData.site( route: String, - data: DataSet<*> = siteData, + data: DataTree<*>? = siteData, siteMeta: Meta = Meta.EMPTY, content: HtmlSite, -): Unit = site(route.parseAsName(), data, siteMeta,content) +): Unit = site(route.parseAsName(), data, siteMeta, content) /** * Render all pages and sites found in the data */ -public suspend fun SiteContext.renderPages(data: DataSet<*>): Unit { +public suspend fun SiteContext.renderPages(data: DataTree<*>): Unit { // Render all sub-sites data.filterByType().forEach { siteData: NamedData -> @@ -151,7 +143,7 @@ public suspend fun SiteContext.renderPages(data: DataSet<*>): Unit { val dataPrefix = siteData.meta["site.dataPath"].string?.asName() ?: Name.EMPTY site( route = siteData.meta["site.route"].string?.asName() ?: siteData.name, - data.branch(dataPrefix), + data.branch(dataPrefix) ?: DataTree.EMPTY, siteMeta = siteData.meta, siteData.await() ) @@ -162,7 +154,7 @@ public suspend fun SiteContext.renderPages(data: DataSet<*>): Unit { val dataPrefix = pageData.meta["page.dataPath"].string?.asName() ?: Name.EMPTY page( route = pageData.meta["page.route"].string?.asName() ?: pageData.name, - data.branch(dataPrefix), + data.branch(dataPrefix) ?: DataTree.EMPTY, pageMeta = pageData.meta, pageData.await() ) diff --git a/snark-html/src/jvmMain/kotlin/space/kscience/snark/html/SnarkHtml.kt b/snark-html/src/jvmMain/kotlin/space/kscience/snark/html/SnarkHtml.kt index 23ebbc1..5ab561a 100644 --- a/snark-html/src/jvmMain/kotlin/space/kscience/snark/html/SnarkHtml.kt +++ b/snark-html/src/jvmMain/kotlin/space/kscience/snark/html/SnarkHtml.kt @@ -6,16 +6,18 @@ import io.ktor.http.ContentType import kotlinx.coroutines.CoroutineScope import kotlinx.io.readByteArray import space.kscience.dataforge.actions.Action -import space.kscience.dataforge.actions.mapping +import space.kscience.dataforge.actions.transform import space.kscience.dataforge.context.Context import space.kscience.dataforge.context.PluginFactory import space.kscience.dataforge.context.PluginTag import space.kscience.dataforge.data.* -import space.kscience.dataforge.io.* +import space.kscience.dataforge.io.Binary +import space.kscience.dataforge.io.IOPlugin +import space.kscience.dataforge.io.IOReader +import space.kscience.dataforge.io.JsonMetaFormat import space.kscience.dataforge.io.yaml.YamlMetaFormat import space.kscience.dataforge.io.yaml.YamlPlugin import space.kscience.dataforge.meta.Meta -import space.kscience.dataforge.meta.get import space.kscience.dataforge.meta.set import space.kscience.dataforge.meta.string import space.kscience.dataforge.misc.DFExperimental @@ -26,24 +28,19 @@ import space.kscience.dataforge.workspace.* import space.kscience.snark.ReWrapAction import space.kscience.snark.Snark import space.kscience.snark.SnarkReader -import space.kscience.snark.TextProcessor import java.net.URLConnection import kotlin.io.path.Path import kotlin.io.path.extension -public fun DataSet.transform(action: Action, meta: Meta = Meta.EMPTY): DataSet = +public fun DataTree.transform(action: Action, meta: Meta = Meta.EMPTY): DataTree = action.execute(this, meta) -public fun DataSetBuilder.fill(dataSet: DataSet) { - node(Name.EMPTY, dataSet) -} - /** * A plugin used for rendering a [DataTree] as HTML */ public class SnarkHtml : WorkspacePlugin() { - private val snark by require(Snark) + public val snark: Snark by require(Snark) private val yaml by require(YamlPlugin) public val io: IOPlugin get() = snark.io @@ -60,61 +57,30 @@ public class SnarkHtml : WorkspacePlugin() { else -> super.content(target) } - private fun getContentType(name: Name, meta: Meta): String = meta[CONTENT_TYPE_KEY].string ?: run { + internal fun getContentType(name: Name, meta: Meta): String = meta[CONTENT_TYPE_KEY].string ?: run { val filePath = meta[FileData.FILE_PATH_KEY]?.string ?: name.toString() URLConnection.guessContentTypeFromName(filePath) ?: Path(filePath).extension } - public val prepareHeaderAction: Action = ReWrapAction.removeExtensions("html", "md") { name -> + public val prepareHeaderAction: ReWrapAction = ReWrapAction.removeExtensions("html", "md") { name -> val contentType = getContentType(name, this) set(CONTENT_TYPE_KEY, contentType) } - public val parseAction: Action = Action.mapping { - val contentType = getContentType(name, meta) + public val removeIndexAction: ReWrapAction = ReWrapAction.removeIndex() - val parser = snark.readers.values.filter { parser -> - contentType in parser.types - }.maxByOrNull { - it.priority - } - //pass data for which parser is not found - if (parser == null) { - result { it } - } else { - - val preprocessor = meta[TextProcessor.TEXT_PREPROCESSOR_KEY]?.let { snark.preprocessor(it) } - result { - - when (it) { - is CharSequence -> { - val string = it.toString() - val preprocessed = preprocessor?.process(string) ?: string - parser.readFrom(preprocessed) - } - - is Binary -> { - if (preprocessor == null) { - parser.readFrom(it) - } else { - //TODO provide encoding - val string = it.toByteArray().decodeToString() - parser.readFrom(preprocessor.process(string)) - } - } - - // bypass for non textual-data - else -> it - } - } - } - } + public val parseAction: Action = ParseAction(this) + private val allDataNotNull: DataSelector + get() = DataSelector { workspace, _ -> workspace.data.filterByType() } public val parse: TaskReference by task({ description = "Parse all data for which reader is resolved" }) { - fill(from(allData).transform(parseAction, taskMeta)) + //put all data + putAll(from(allDataNotNull)) + //override parsed data + putAll(from(allDataNotNull).filterByType().transform(parseAction)) } @@ -136,15 +102,18 @@ public class SnarkHtml : WorkspacePlugin() { public fun SnarkHtml.readSiteData( - binaries: DataSource, + binaries: DataTree, meta: Meta = Meta.EMPTY, -): DataSet = binaries.transform(parseAction, meta) +): DataTree = ObservableDataTree(context) { + //put all binaries + putAll(binaries) + //override ones which could be parsed + putAll(binaries.transform(parseAction, meta)) +}.transform(prepareHeaderAction, meta).transform(removeIndexAction, meta) public fun SnarkHtml.readSiteData( coroutineScope: CoroutineScope, meta: Meta = Meta.EMPTY, - builder: context(IOPlugin) DataSourceBuilder.() -> Unit, -): DataSet = DataSource(coroutineScope) { builder(io, this) } - .transform(prepareHeaderAction, meta) - .transform(parseAction, meta) \ No newline at end of file + builder: DataSink.() -> Unit, +): DataTree = readSiteData(ObservableDataTree(coroutineScope) { builder() }, meta) diff --git a/snark-html/src/jvmMain/kotlin/space/kscience/snark/html/readers.kt b/snark-html/src/jvmMain/kotlin/space/kscience/snark/html/readers.kt index d9a9b4d..940d9d9 100644 --- a/snark-html/src/jvmMain/kotlin/space/kscience/snark/html/readers.kt +++ b/snark-html/src/jvmMain/kotlin/space/kscience/snark/html/readers.kt @@ -8,10 +8,11 @@ import org.intellij.markdown.flavours.commonmark.CommonMarkFlavourDescriptor import org.intellij.markdown.html.HtmlGenerator import org.intellij.markdown.parser.MarkdownParser import space.kscience.snark.SnarkReader -import kotlin.reflect.KType -import kotlin.reflect.typeOf -public object HtmlReader : SnarkReader { + +public interface SnarkHtmlReader: SnarkReader + +public object HtmlReader : SnarkHtmlReader { override val types: Set = setOf("html") override fun readFrom(source: String): PageFragment = PageFragment { @@ -21,12 +22,9 @@ public object HtmlReader : SnarkReader { } override fun readFrom(source: Source): PageFragment = readFrom(source.readString()) - override val type: KType = typeOf() } -public object MarkdownReader : SnarkReader { - override val type: KType = typeOf() - +public object MarkdownReader : SnarkHtmlReader { override val types: Set = setOf("text/markdown", "md", "markdown") override fun readFrom(source: String): PageFragment = PageFragment { diff --git a/snark-html/src/jvmMain/kotlin/space/kscience/snark/html/static/StaticSiteContext.kt b/snark-html/src/jvmMain/kotlin/space/kscience/snark/html/static/StaticSiteContext.kt index 1fbfc61..a001393 100644 --- a/snark-html/src/jvmMain/kotlin/space/kscience/snark/html/static/StaticSiteContext.kt +++ b/snark-html/src/jvmMain/kotlin/space/kscience/snark/html/static/StaticSiteContext.kt @@ -100,7 +100,7 @@ internal class StaticSiteContext( ) } - override fun page(route: Name, data: DataSet, pageMeta: Meta, content: HtmlPage) { + override fun page(route: Name, data: DataTree<*>?, pageMeta: Meta, content: HtmlPage) { val modifiedPageMeta = pageMeta.toMutableMeta().apply { @@ -119,7 +119,7 @@ internal class StaticSiteContext( newPath.writeText(HtmlPage.createHtmlString(pageContext, data, content)) } - override fun route(route: Name, data: DataSet<*>, siteMeta: Meta, content: HtmlSite) { + override fun route(route: Name, data: DataTree<*>?, siteMeta: Meta, content: HtmlSite) { val siteContextWithData = SiteContextWithData( StaticSiteContext( siteMeta = Laminate(siteMeta, this@StaticSiteContext.siteMeta), @@ -127,7 +127,7 @@ internal class StaticSiteContext( route = route, outputPath = outputPath.resolve(route.toWebPath()) ), - data + data ?: DataTree.EMPTY ) with(content) { with(siteContextWithData) { @@ -136,7 +136,7 @@ internal class StaticSiteContext( } } - override fun site(route: Name, data: DataSet, siteMeta: Meta, content: HtmlSite) { + override fun site(route: Name, data: DataTree<*>?, siteMeta: Meta, content: HtmlSite) { val siteContextWithData = SiteContextWithData( StaticSiteContext( siteMeta = Laminate(siteMeta, this@StaticSiteContext.siteMeta), @@ -144,7 +144,7 @@ internal class StaticSiteContext( route = Name.EMPTY, outputPath = outputPath.resolve(route.toWebPath()) ), - data + data ?: DataTree.EMPTY ) with(content) { with(siteContextWithData) { @@ -162,17 +162,17 @@ internal class StaticSiteContext( */ @Suppress("UnusedReceiverParameter") public suspend fun SnarkHtml.staticSite( - data: DataSet<*>, + data: DataTree<*>?, outputPath: Path, siteUrl: String = outputPath.absolutePathString().replace("\\", "/"), - siteMeta: Meta = data.meta, + siteMeta: Meta = data?.meta ?: Meta.EMPTY, content: HtmlSite, ) { val siteContextWithData = SiteContextWithData( StaticSiteContext(siteMeta, siteUrl, Name.EMPTY, outputPath), - data + data ?: DataTree.EMPTY ) - with(content){ + with(content) { with(siteContextWithData) { renderSite() } diff --git a/snark-ktor/src/jvmMain/kotlin/space/kscience/snark/ktor/KtorSiteContext.kt b/snark-ktor/src/jvmMain/kotlin/space/kscience/snark/ktor/KtorSiteContext.kt index 4917e1e..c0f8d4d 100644 --- a/snark-ktor/src/jvmMain/kotlin/space/kscience/snark/ktor/KtorSiteContext.kt +++ b/snark-ktor/src/jvmMain/kotlin/space/kscience/snark/ktor/KtorSiteContext.kt @@ -15,8 +15,9 @@ import space.kscience.dataforge.context.ContextAware import space.kscience.dataforge.context.error import space.kscience.dataforge.context.logger import space.kscience.dataforge.data.Data -import space.kscience.dataforge.data.DataSet +import space.kscience.dataforge.data.DataTree import space.kscience.dataforge.data.await +import space.kscience.dataforge.data.meta import space.kscience.dataforge.io.Binary import space.kscience.dataforge.io.toByteArray import space.kscience.dataforge.meta.Laminate @@ -106,7 +107,7 @@ public class KtorSiteContext( } } - override fun page(route: Name, data: DataSet<*>, pageMeta: Meta, content: HtmlPage) { + override fun page(route: Name, data: DataTree<*>?, pageMeta: Meta, content: HtmlPage) { ktorRoute.get(route.toWebPath()) { val request = call.request //substitute host for url for backwards calls @@ -129,7 +130,7 @@ public class KtorSiteContext( } } - override fun route(route: Name, data: DataSet<*>, siteMeta: Meta, content: HtmlSite) { + override fun route(route: Name, data: DataTree<*>?, siteMeta: Meta, content: HtmlSite) { val siteContext = SiteContextWithData( KtorSiteContext( context, @@ -138,7 +139,7 @@ public class KtorSiteContext( route = route, ktorRoute = ktorRoute.createRouteFromPath(route.toWebPath()) ), - data + data ?: DataTree.EMPTY ) with(content) { with(siteContext) { @@ -147,7 +148,7 @@ public class KtorSiteContext( } } - override fun site(route: Name, data: DataSet<*>, siteMeta: Meta, content: HtmlSite) { + override fun site(route: Name, data: DataTree<*>?, siteMeta: Meta, content: HtmlSite) { val siteContext = SiteContextWithData( KtorSiteContext( context, @@ -156,7 +157,7 @@ public class KtorSiteContext( route = Name.EMPTY, ktorRoute = ktorRoute.createRouteFromPath(route.toWebPath()) ), - data + data?: DataTree.EMPTY ) with(content) { with(siteContext) { @@ -169,14 +170,14 @@ public class KtorSiteContext( public fun Route.site( context: Context, - data: DataSet<*>, + data: DataTree<*>?, baseUrl: String = "", - siteMeta: Meta = data.meta, + siteMeta: Meta = data?.meta ?: Meta.EMPTY, content: HtmlSite, ) { val siteContext = SiteContextWithData( KtorSiteContext(context, siteMeta, baseUrl, route = Name.EMPTY, this@Route), - data + data?: DataTree.EMPTY ) with(content) { with(siteContext) {