Update to DataForge 0.8

This commit is contained in:
Alexander Nozik 2024-02-19 19:47:40 +03:00
parent 3b318c3a8b
commit b66c6b4fe6
17 changed files with 234 additions and 203 deletions

View File

@ -1,4 +1,5 @@
import space.kscience.gradle.* import space.kscience.gradle.useApache2Licence
import space.kscience.gradle.useSPCTeam
plugins { plugins {
id("space.kscience.gradle.project") id("space.kscience.gradle.project")
@ -14,7 +15,7 @@ allprojects {
} }
} }
val dataforgeVersion by extra("0.8.0-dev-1") val dataforgeVersion by extra("0.8.0")
ksciencePublish { ksciencePublish {
pom("https://github.com/SciProgCentre/snark") { pom("https://github.com/SciProgCentre/snark") {

View File

@ -1,3 +1,3 @@
kotlin.code.style=official kotlin.code.style=official
toolsVersion=0.15.2-kotlin-1.9.21 toolsVersion=0.15.2-kotlin-1.9.22

View File

@ -5,9 +5,7 @@ import space.kscience.dataforge.data.*
import space.kscience.dataforge.meta.Meta import space.kscience.dataforge.meta.Meta
import space.kscience.dataforge.meta.MutableMeta import space.kscience.dataforge.meta.MutableMeta
import space.kscience.dataforge.meta.copy import space.kscience.dataforge.meta.copy
import space.kscience.dataforge.names.Name import space.kscience.dataforge.names.*
import space.kscience.dataforge.names.NameToken
import space.kscience.dataforge.names.replaceLast
import kotlin.reflect.KType import kotlin.reflect.KType
import kotlin.reflect.typeOf import kotlin.reflect.typeOf
@ -19,20 +17,19 @@ public class ReWrapAction<R : Any>(
private val newMeta: MutableMeta.(name: Name) -> Unit = {}, private val newMeta: MutableMeta.(name: Name) -> Unit = {},
private val newName: (name: Name, meta: Meta?) -> Name, private val newName: (name: Name, meta: Meta?) -> Name,
) : AbstractAction<R, R>(type) { ) : AbstractAction<R, R>(type) {
override fun DataSetBuilder<R>.generate(data: DataSet<R>, meta: Meta) { override fun DataSink<R>.generate(data: DataTree<R>, meta: Meta) {
data.forEach { namedData -> data.forEach { namedData ->
data( put(
newName(namedData.name, namedData.meta), newName(namedData.name, namedData.meta),
namedData.data.withMeta(namedData.meta.copy { newMeta(namedData.name) }) namedData.data.withMeta(namedData.meta.copy { newMeta(namedData.name) })
) )
} }
} }
override fun DataSourceBuilder<R>.update(dataSet: DataSet<R>, meta: Meta, updateKey: Name) { override fun DataSink<R>.update(source: DataTree<R>, meta: Meta, namedData: NamedData<R>) {
val datum = dataSet[updateKey] put(
data( newName(namedData.name, namedData.meta),
newName(updateKey, datum?.meta), namedData.withMeta(namedData.meta.copy { newMeta(namedData.name) })
datum?.withMeta(datum.meta.copy { newMeta(updateKey) })
) )
} }
@ -50,11 +47,15 @@ public class ReWrapAction<R : Any>(
} }
} }
} }
public inline fun <reified R : Any> removeIndex(): ReWrapAction<R> = ReWrapAction<R>(typeOf<R>()) { name, _ ->
if (name.endsWith("index")) name.cutLast() else name
}
} }
} }
public inline fun <reified R : Any> ReWrapAction( public inline fun <reified R : Any> ReWrapAction(
noinline newMeta: MutableMeta.(name: Name) -> Unit = {}, noinline newMeta: MutableMeta.(name: Name) -> Unit = {},
noinline newName: (Name, Meta?) -> Name noinline newName: (Name, Meta?) -> Name,
): ReWrapAction<R> = ReWrapAction(typeOf<R>(), newMeta, newName) ): ReWrapAction<R> = ReWrapAction(typeOf<R>(), newMeta, newName)

View File

@ -5,8 +5,6 @@ import kotlinx.io.asInputStream
import space.kscience.dataforge.io.IOReader import space.kscience.dataforge.io.IOReader
import java.awt.image.BufferedImage import java.awt.image.BufferedImage
import javax.imageio.ImageIO import javax.imageio.ImageIO
import kotlin.reflect.KType
import kotlin.reflect.typeOf
/** /**
* The ImageIOReader class is an implementation of the IOReader interface specifically for reading images using the ImageIO library. * The ImageIOReader class is an implementation of the IOReader interface specifically for reading images using the ImageIO library.
@ -15,7 +13,5 @@ import kotlin.reflect.typeOf
* @property type The KType of the data to be read by the ImageIOReader. * @property type The KType of the data to be read by the ImageIOReader.
*/ */
public object ImageIOReader : IOReader<BufferedImage> { public object ImageIOReader : IOReader<BufferedImage> {
override val type: KType get() = typeOf<BufferedImage>()
override fun readFrom(source: Source): BufferedImage = ImageIO.read(source.asInputStream()) override fun readFrom(source: Source): BufferedImage = ImageIO.read(source.asInputStream())
} }

View File

@ -1,16 +1,12 @@
package space.kscience.snark package space.kscience.snark
import space.kscience.dataforge.data.DataSet
import space.kscience.dataforge.data.DataSource
import space.kscience.dataforge.data.DataTree
import space.kscience.dataforge.data.filterByType import space.kscience.dataforge.data.filterByType
import space.kscience.dataforge.names.Name import space.kscience.dataforge.names.Name
import space.kscience.dataforge.names.parseAsName
import space.kscience.dataforge.names.startsWith import space.kscience.dataforge.names.startsWith
import kotlin.reflect.typeOf
public inline fun <reified R : Any> DataSet<*>.branch( public inline fun <reified R : Any> DataTree<*>.branch(
branchName: Name, branchName: Name,
): DataSet<R> = filterByType(typeOf<R>()) { name, _ -> name.startsWith(branchName) } ): DataSource<R> = filterByType<R> { name, _, _ -> name.startsWith(branchName) }
public inline fun <reified R : Any> DataSet<*>.branch(
branchName: String,
): DataSet<R> = filterByType(typeOf<R>()) { name, _ -> name.startsWith(branchName.parseAsName()) }

View File

@ -2,38 +2,36 @@
package space.kscience.snark package space.kscience.snark
import space.kscience.dataforge.data.DataTree import space.kscience.dataforge.data.branch
import space.kscience.dataforge.data.node
import space.kscience.dataforge.io.Binary
import space.kscience.dataforge.io.IOPlugin
import space.kscience.dataforge.meta.* import space.kscience.dataforge.meta.*
import space.kscience.dataforge.misc.DFExperimental import space.kscience.dataforge.misc.DFExperimental
import space.kscience.dataforge.names.parseAsName
import space.kscience.dataforge.workspace.Workspace import space.kscience.dataforge.workspace.Workspace
import space.kscience.dataforge.workspace.WorkspaceBuilder import space.kscience.dataforge.workspace.WorkspaceBuilder
import space.kscience.dataforge.workspace.readRawDirectory import space.kscience.dataforge.workspace.directory
import space.kscience.dataforge.workspace.resources
import kotlin.io.path.Path import kotlin.io.path.Path
import kotlin.io.path.toPath
//
/** ///**
* Reads the specified resources and returns a [DataTree] containing the data. // * Reads the specified resources and returns a [DataTree] containing the data.
* // *
* @param resources The names of the resources to read. // * @param resources The names of the resources to read.
* @param classLoader The class loader to use for loading the resources. By default, it uses the current thread's context class loader. // * @param classLoader The class loader to use for loading the resources. By default, it uses the current thread's context class loader.
* @return A DataTree containing the data read from the resources. // * @return A DataTree containing the data read from the resources.
*/ // */
private fun IOPlugin.readResources( //private fun IOPlugin.readResources(
vararg resources: String, // vararg resources: String,
classLoader: ClassLoader = Thread.currentThread().contextClassLoader, // classLoader: ClassLoader = Thread.currentThread().contextClassLoader,
): DataTree<Binary> = DataTree { //): DataTree<Binary> = DataTree {
// require(resource.isNotBlank()) {"Can't mount root resource tree as data root"} // // require(resource.isNotBlank()) {"Can't mount root resource tree as data root"}
resources.forEach { resource -> // resources.forEach { resource ->
val path = classLoader.getResource(resource)?.toURI()?.toPath() ?: error( // val path = classLoader.getResource(resource)?.toURI()?.toPath() ?: error(
"Resource with name $resource is not resolved" // "Resource with name $resource is not resolved"
) // )
node(resource, readRawDirectory(path)) // node(resource, readRawDirectory(path))
} // }
} //}
public fun Snark.workspace( public fun Snark.workspace(
meta: Meta, meta: Meta,
@ -45,14 +43,14 @@ public fun Snark.workspace(
meta.getIndexed("directory").forEach { (index, directoryMeta) -> meta.getIndexed("directory").forEach { (index, directoryMeta) ->
val dataDirectory = directoryMeta["path"].string ?: error("Directory path not defined") val dataDirectory = directoryMeta["path"].string ?: error("Directory path not defined")
val nodeName = directoryMeta["name"].string ?: directoryMeta.string ?: index ?: "" val nodeName = directoryMeta["name"].string ?: directoryMeta.string ?: index ?: ""
val data = io.readRawDirectory(Path(dataDirectory)) directory(io, nodeName.parseAsName(), Path((dataDirectory)))
node(nodeName, data)
} }
meta.getIndexed("resource").forEach { (index, resourceMeta) -> meta.getIndexed("resource").forEach { (index, resourceMeta) ->
val resource = resourceMeta["path"]?.stringList ?: listOf("/") val resource = resourceMeta["path"]?.stringList ?: listOf("/")
val nodeName = resourceMeta["name"].string ?: resourceMeta.string ?: index ?: "" val nodeName = resourceMeta["name"].string ?: resourceMeta.string ?: index ?: ""
val data: DataTree<Binary> = io.readResources(*resource.toTypedArray()) branch(nodeName) {
node(nodeName, data) resources(io, *resource.toTypedArray())
}
} }
} }

View File

@ -3,9 +3,9 @@ package space.kscience.snark.html
import kotlinx.html.HTML import kotlinx.html.HTML
import kotlinx.html.stream.createHTML import kotlinx.html.stream.createHTML
import kotlinx.html.visitTagAndFinalize import kotlinx.html.visitTagAndFinalize
import space.kscience.dataforge.data.DataSet import space.kscience.dataforge.data.DataSink
import space.kscience.dataforge.data.DataSetBuilder import space.kscience.dataforge.data.DataTree
import space.kscience.dataforge.data.static import space.kscience.dataforge.data.wrap
import space.kscience.dataforge.meta.Meta import space.kscience.dataforge.meta.Meta
import space.kscience.dataforge.names.Name import space.kscience.dataforge.names.Name
@ -17,11 +17,11 @@ public fun interface HtmlPage {
public companion object { public companion object {
public fun createHtmlString( public fun createHtmlString(
pageContext: PageContext, pageContext: PageContext,
dataSet: DataSet<*>, dataSet: DataTree<*>?,
page: HtmlPage, page: HtmlPage,
): String = createHTML().run { ): String = createHTML().run {
HTML(kotlinx.html.emptyMap, this, null).visitTagAndFinalize(this) { HTML(kotlinx.html.emptyMap, this, null).visitTagAndFinalize(this) {
with(PageContextWithData(pageContext, dataSet)) { with(PageContextWithData(pageContext, dataSet ?: DataTree.EMPTY)) {
with(page) { with(page) {
renderPage() renderPage()
} }
@ -34,13 +34,13 @@ public fun interface HtmlPage {
// data builders // data builders
public fun DataSetBuilder<Any>.page( public fun DataSink<Any>.page(
name: Name, name: Name,
pageMeta: Meta = Meta.EMPTY, pageMeta: Meta = Meta.EMPTY,
block: context(PageContextWithData) HTML.() -> Unit, block: context(PageContextWithData) HTML.() -> Unit,
) { ) {
val page = HtmlPage(block) val page = HtmlPage(block)
static<HtmlPage>(name, page, pageMeta) wrap<HtmlPage>(name, page, pageMeta)
} }

View File

@ -1,8 +1,8 @@
package space.kscience.snark.html package space.kscience.snark.html
import space.kscience.dataforge.data.DataSet import space.kscience.dataforge.data.DataSink
import space.kscience.dataforge.data.DataSetBuilder import space.kscience.dataforge.data.DataTree
import space.kscience.dataforge.data.static import space.kscience.dataforge.data.wrap
import space.kscience.dataforge.meta.Meta import space.kscience.dataforge.meta.Meta
import space.kscience.dataforge.meta.getIndexed import space.kscience.dataforge.meta.getIndexed
import space.kscience.dataforge.meta.string import space.kscience.dataforge.meta.string
@ -15,23 +15,23 @@ public fun interface HtmlSite {
public fun renderSite() public fun renderSite()
} }
public fun DataSetBuilder<Any>.site( public fun DataSink<Any>.site(
name: Name, name: Name,
siteMeta: Meta, siteMeta: Meta,
block: (siteContext: SiteContext, data: DataSet<Any>) -> Unit, block: (siteContext: SiteContext, data: DataTree<*>?) -> Unit,
) { ) {
static(name, HtmlSite { block(site, siteData) }, siteMeta) wrap(name, HtmlSite { block(site, siteData) }, siteMeta)
} }
//public fun DataSetBuilder<Any>.site(name: Name, block: DataSetBuilder<Any>.() -> Unit) { //public fun DataSetBuilder<Any>.site(name: Name, block: DataSetBuilder<Any>.() -> Unit) {
// node(name, block) // node(name, block)
//} //}
internal fun DataSetBuilder<Any>.assetsFrom(rootMeta: Meta) { internal fun DataSink<Any>.assetsFrom(rootMeta: Meta) {
rootMeta.getIndexed("file".asName()).forEach { (_, meta) -> rootMeta.getIndexed("file".asName()).forEach { (_, meta) ->
val webName: String? by meta.string() val webName: String? by meta.string()
val name by meta.string { error("File path is not provided") } val name by meta.string { error("File path is not provided") }
val fileName = name.parseAsName() val fileName = name.parseAsName()
static(fileName, webName?.parseAsName() ?: fileName) wrap(fileName, webName?.parseAsName() ?: fileName)
} }
} }

View File

@ -1,15 +1,14 @@
package space.kscience.snark.html package space.kscience.snark.html
import space.kscience.dataforge.data.DataSet import space.kscience.dataforge.actions.AbstractAction
import space.kscience.dataforge.data.branch import space.kscience.dataforge.actions.transform
import space.kscience.dataforge.data.*
import space.kscience.dataforge.meta.* import space.kscience.dataforge.meta.*
import space.kscience.dataforge.names.Name import space.kscience.dataforge.names.*
import space.kscience.dataforge.names.asName
import space.kscience.dataforge.names.parseAsName
import space.kscience.dataforge.names.plus
import space.kscience.snark.SnarkBuilder import space.kscience.snark.SnarkBuilder
import space.kscience.snark.html.Language.Companion.SITE_LANGUAGES_KEY import space.kscience.snark.html.Language.Companion.SITE_LANGUAGES_KEY
import space.kscience.snark.html.Language.Companion.SITE_LANGUAGE_KEY import space.kscience.snark.html.Language.Companion.SITE_LANGUAGE_KEY
import kotlin.reflect.typeOf
public class Language : Scheme() { public class Language : Scheme() {
@ -81,9 +80,37 @@ public val SiteContext.languages: Map<String, Meta>
public val SiteContext.language: String public val SiteContext.language: String
get() = siteMeta[SITE_LANGUAGE_KEY].string ?: Language.DEFAULT_LANGUAGE get() = siteMeta[SITE_LANGUAGE_KEY].string ?: Language.DEFAULT_LANGUAGE
//
//public val SiteContext.languagePrefix: Name
// get() = languages[language]?.let { it[Language::prefix.name].string ?: language }?.parseAsName() ?: Name.EMPTY
public val SiteContext.languagePrefix: Name
get() = languages[language]?.let { it[Language::prefix.name].string ?: language }?.parseAsName() ?: Name.EMPTY private class LanguageBranchAction(val prefix: Name) : AbstractAction<Any, Any>(typeOf<Any>()) {
override fun DataSink<Any>.generate(data: DataTree<Any>, meta: Meta) {
data.forEach { (name, item) ->
val nameWithoutPrefix = name.removeFirstOrNull(prefix)
if (nameWithoutPrefix != null) {
// put language item as is
put(nameWithoutPrefix, item)
} else if (data[name + prefix] == null) {
// put if language item is missing
put(name, item)
}
}
}
override fun DataSink<Any>.update(source: DataTree<Any>, meta: Meta, namedData: NamedData<Any>) {
val name = namedData.name
val nameWithoutPrefix = name.removeFirstOrNull(prefix)
if (nameWithoutPrefix != null) {
// put language item as is
put(nameWithoutPrefix, namedData.data)
} else if (source[name + prefix] == null) {
// put if language item is missing
put(name, namedData.data)
}
}
}
/** /**
* Create a multiple sites for different languages. All sites use the same [content], but rely on different data * Create a multiple sites for different languages. All sites use the same [content], but rely on different data
@ -92,7 +119,7 @@ public val SiteContext.languagePrefix: Name
*/ */
@SnarkBuilder @SnarkBuilder
public fun SiteContext.multiLanguageSite( public fun SiteContext.multiLanguageSite(
data: DataSet<*>, data: DataTree<*>,
languageMap: Map<String, Language>, languageMap: Map<String, Language>,
content: HtmlSite, content: HtmlSite,
) { ) {
@ -108,7 +135,7 @@ public fun SiteContext.multiLanguageSite(
} }
site( site(
prefix.parseAsName(), prefix.parseAsName(),
data.branch(language.dataPath ?: prefix), data.filterByType<Any>().transform(LanguageBranchAction(Name.parse(language.dataPath ?: prefix))),
siteMeta = Laminate(languageSiteMeta, siteMeta), siteMeta = Laminate(languageSiteMeta, siteMeta),
content content
) )

View File

@ -1,6 +1,6 @@
package space.kscience.snark.html package space.kscience.snark.html
import space.kscience.dataforge.data.DataSet import space.kscience.dataforge.data.DataTree
import space.kscience.dataforge.meta.Meta import space.kscience.dataforge.meta.Meta
import space.kscience.dataforge.meta.get import space.kscience.dataforge.meta.get
import space.kscience.dataforge.meta.string import space.kscience.dataforge.meta.string
@ -57,4 +57,7 @@ public val PageContext.homeRef: String get() = resolvePageRef(SiteContext.INDEX_
public val PageContext.name: Name? get() = pageMeta["name"].string?.parseAsName() public val PageContext.name: Name? get() = pageMeta["name"].string?.parseAsName()
public class PageContextWithData(private val pageContext: PageContext, public val data: DataSet<*>): PageContext by pageContext public class PageContextWithData(
private val pageContext: PageContext,
public val data: DataTree<*>,
) : PageContext by pageContext

View File

@ -15,20 +15,19 @@ import space.kscience.snark.SnarkContext
public fun interface PageFragment { public fun interface PageFragment {
context(PageContextWithData) context(PageContextWithData, FlowContent) public fun renderFragment()
public fun FlowContent.renderFragment()
} }
context(PageContextWithData) context(PageContextWithData, FlowContent)
public fun FlowContent.fragment(fragment: PageFragment): Unit{ public fun fragment(fragment: PageFragment): Unit {
with(fragment) { with(fragment) {
renderFragment() renderFragment()
} }
} }
context(PageContextWithData) context(PageContextWithData, FlowContent)
public fun FlowContent.fragment(data: Data<PageFragment>): Unit = runBlocking { public fun fragment(data: Data<PageFragment>): Unit = runBlocking {
fragment(data.await()) fragment(data.await())
} }
@ -54,7 +53,7 @@ public val Data<*>.published: Boolean
* Resolve a Html builder by its full name * Resolve a Html builder by its full name
*/ */
context(SnarkContext) context(SnarkContext)
public fun DataSet<*>.resolveHtmlOrNull(name: Name): Data<PageFragment>? { public fun DataTree<*>.resolveHtmlOrNull(name: Name): Data<PageFragment>? {
val resolved = (getByType<PageFragment>(name) ?: getByType<PageFragment>(name + SiteContext.INDEX_PAGE_TOKEN)) val resolved = (getByType<PageFragment>(name) ?: getByType<PageFragment>(name + SiteContext.INDEX_PAGE_TOKEN))
return resolved?.takeIf { return resolved?.takeIf {
@ -63,26 +62,26 @@ public fun DataSet<*>.resolveHtmlOrNull(name: Name): Data<PageFragment>? {
} }
context(SnarkContext) context(SnarkContext)
public fun DataSet<*>.resolveHtmlOrNull(name: String): Data<PageFragment>? = resolveHtmlOrNull(name.parseAsName()) public fun DataTree<*>.resolveHtmlOrNull(name: String): Data<PageFragment>? = resolveHtmlOrNull(name.parseAsName())
context(SnarkContext) context(SnarkContext)
public fun DataSet<*>.resolveHtml(name: String): Data<PageFragment> = resolveHtmlOrNull(name) public fun DataTree<*>.resolveHtml(name: String): Data<PageFragment> = resolveHtmlOrNull(name)
?: error("Html fragment with name $name is not resolved") ?: error("Html fragment with name $name is not resolved")
/** /**
* Find all Html blocks using given name/meta filter * Find all Html blocks using given name/meta filter
*/ */
context(SnarkContext) context(SnarkContext)
public fun DataSet<*>.resolveAllHtml( public fun DataTree<*>.resolveAllHtml(
predicate: (name: Name, meta: Meta) -> Boolean, predicate: (name: Name, meta: Meta) -> Boolean,
): Map<Name, Data<PageFragment>> = filterByType<PageFragment> { name, meta -> ): Map<Name, Data<PageFragment>> = filterByType<PageFragment> { name, meta, _ ->
predicate(name, meta) predicate(name, meta)
&& meta["published"].string != "false" && meta["published"].string != "false"
//TODO add language confirmation //TODO add language confirmation
}.asSequence().associate { it.name to it.data } }.asSequence().associate { it.name to it.data }
context(SnarkContext) context(SnarkContext)
public fun DataSet<*>.findHtmlByContentType( public fun DataTree<*>.findHtmlByContentType(
contentType: String, contentType: String,
baseName: Name = Name.EMPTY, baseName: Name = Name.EMPTY,
): Map<Name, Data<PageFragment>> = resolveAllHtml { name, meta -> ): Map<Name, Data<PageFragment>> = resolveAllHtml { name, meta ->

View File

@ -0,0 +1,50 @@
package space.kscience.snark.html
import space.kscience.dataforge.actions.AbstractAction
import space.kscience.dataforge.data.*
import space.kscience.dataforge.io.Binary
import space.kscience.dataforge.io.toByteArray
import space.kscience.dataforge.meta.Meta
import space.kscience.dataforge.meta.get
import space.kscience.snark.TextProcessor
import kotlin.reflect.typeOf
public class ParseAction(private val snarkHtml: SnarkHtml) :
AbstractAction<Binary, PageFragment>(typeOf<PageFragment>()) {
private fun parseOne(data: NamedData<Binary>, actionMeta: Meta): NamedData<PageFragment>? = with(snarkHtml) {
val contentType = getContentType(data.name, data.meta)
val parser = snark.readers.values.filterIsInstance<SnarkHtmlReader>().filter { parser ->
contentType in parser.types
}.maxByOrNull {
it.priority
}
//ignore data for which parser is not found
if (parser != null) {
val preprocessor = meta[TextProcessor.TEXT_PREPROCESSOR_KEY]?.let { snark.preprocessor(it) }
data.transform {
if (preprocessor == null) {
parser.readFrom(it)
} else {
//TODO provide encoding
val string = it.toByteArray().decodeToString()
parser.readFrom(preprocessor.process(string))
}
}.named(data.name)
} else {
null
}
}
override fun DataSink<PageFragment>.generate(data: DataTree<Binary>, meta: Meta) {
data.forEach {
parseOne(it, meta)?.let { put(it) }
}
}
override fun DataSink<PageFragment>.update(source: DataTree<Binary>, meta: Meta, namedData: NamedData<Binary>) {
parseOne(namedData,meta)?.let { put(it) }
}
}

View File

@ -8,8 +8,6 @@ import space.kscience.dataforge.meta.string
import space.kscience.dataforge.names.* import space.kscience.dataforge.names.*
import space.kscience.snark.SnarkBuilder import space.kscience.snark.SnarkBuilder
import space.kscience.snark.SnarkContext import space.kscience.snark.SnarkContext
import kotlin.reflect.full.isSubtypeOf
import kotlin.reflect.typeOf
/** /**
@ -45,7 +43,7 @@ public interface SiteContext : SnarkContext {
@SnarkBuilder @SnarkBuilder
public fun page( public fun page(
route: Name, route: Name,
data: DataSet<*>, data: DataTree<*>?,
pageMeta: Meta = Meta.EMPTY, pageMeta: Meta = Meta.EMPTY,
content: HtmlPage, content: HtmlPage,
) )
@ -56,7 +54,7 @@ public interface SiteContext : SnarkContext {
@SnarkBuilder @SnarkBuilder
public fun route( public fun route(
route: Name, route: Name,
data: DataSet<*>, data: DataTree<*>?,
siteMeta: Meta = Meta.EMPTY, siteMeta: Meta = Meta.EMPTY,
content: HtmlSite, content: HtmlSite,
) )
@ -68,7 +66,7 @@ public interface SiteContext : SnarkContext {
@SnarkBuilder @SnarkBuilder
public fun site( public fun site(
route: Name, route: Name,
data: DataSet<*>, data: DataTree<*>?,
siteMeta: Meta = Meta.EMPTY, siteMeta: Meta = Meta.EMPTY,
content: HtmlSite, content: HtmlSite,
) )
@ -81,25 +79,19 @@ public interface SiteContext : SnarkContext {
} }
} }
public fun SiteContext.static(dataSet: DataSet<Binary>, prefix: Name = Name.EMPTY) { public fun SiteContext.static(dataSet: DataTree<Binary>, prefix: Name = Name.EMPTY) {
dataSet.forEach { (name, data) -> dataSet.forEach { (name, data) ->
static(prefix + name, data) static(prefix + name, data)
} }
} }
public fun SiteContext.static(dataSet: DataTree<*>, branch: String, prefix: String = branch) {
public fun SiteContext.static(dataSet: DataSet<*>, branch: String, prefix: String = branch) {
val branchName = branch.parseAsName() val branchName = branch.parseAsName()
val prefixName = prefix.parseAsName() val prefixName = prefix.parseAsName()
val binaryType = typeOf<Binary>() dataSet.branch(branchName)?.filterByType<Binary>()?.forEach {
dataSet.forEach { (name, data) -> static(prefixName + it.name, it.data)
@Suppress("UNCHECKED_CAST")
if (name.startsWith(branchName) && data.type.isSubtypeOf(binaryType)) {
static(prefixName + name, data as Data<Binary>)
} }
} }
}
context(SiteContext) context(SiteContext)
@ -110,7 +102,7 @@ public val site: SiteContext
/** /**
* A wrapper for site context that allows convenient site building experience * A wrapper for site context that allows convenient site building experience
*/ */
public class SiteContextWithData(private val site: SiteContext, public val siteData: DataSet<*>) : SiteContext by site public class SiteContextWithData(private val site: SiteContext, public val siteData: DataTree<*>) : SiteContext by site
@SnarkBuilder @SnarkBuilder
@ -127,7 +119,7 @@ public fun SiteContextWithData.page(
@SnarkBuilder @SnarkBuilder
public fun SiteContextWithData.route( public fun SiteContextWithData.route(
route: String, route: String,
data: DataSet<*> = siteData, data: DataTree<*>? = siteData,
siteMeta: Meta = Meta.EMPTY, siteMeta: Meta = Meta.EMPTY,
content: HtmlSite, content: HtmlSite,
): Unit = route(route.parseAsName(), data, siteMeta, content) ): Unit = route(route.parseAsName(), data, siteMeta, content)
@ -135,7 +127,7 @@ public fun SiteContextWithData.route(
@SnarkBuilder @SnarkBuilder
public fun SiteContextWithData.site( public fun SiteContextWithData.site(
route: String, route: String,
data: DataSet<*> = siteData, data: DataTree<*>? = siteData,
siteMeta: Meta = Meta.EMPTY, siteMeta: Meta = Meta.EMPTY,
content: HtmlSite, content: HtmlSite,
): Unit = site(route.parseAsName(), data, siteMeta, content) ): Unit = site(route.parseAsName(), data, siteMeta, content)
@ -143,7 +135,7 @@ public fun SiteContextWithData.site(
/** /**
* Render all pages and sites found in the data * Render all pages and sites found in the data
*/ */
public suspend fun SiteContext.renderPages(data: DataSet<*>): Unit { public suspend fun SiteContext.renderPages(data: DataTree<*>): Unit {
// Render all sub-sites // Render all sub-sites
data.filterByType<HtmlSite>().forEach { siteData: NamedData<HtmlSite> -> data.filterByType<HtmlSite>().forEach { siteData: NamedData<HtmlSite> ->
@ -151,7 +143,7 @@ public suspend fun SiteContext.renderPages(data: DataSet<*>): Unit {
val dataPrefix = siteData.meta["site.dataPath"].string?.asName() ?: Name.EMPTY val dataPrefix = siteData.meta["site.dataPath"].string?.asName() ?: Name.EMPTY
site( site(
route = siteData.meta["site.route"].string?.asName() ?: siteData.name, route = siteData.meta["site.route"].string?.asName() ?: siteData.name,
data.branch(dataPrefix), data.branch(dataPrefix) ?: DataTree.EMPTY,
siteMeta = siteData.meta, siteMeta = siteData.meta,
siteData.await() siteData.await()
) )
@ -162,7 +154,7 @@ public suspend fun SiteContext.renderPages(data: DataSet<*>): Unit {
val dataPrefix = pageData.meta["page.dataPath"].string?.asName() ?: Name.EMPTY val dataPrefix = pageData.meta["page.dataPath"].string?.asName() ?: Name.EMPTY
page( page(
route = pageData.meta["page.route"].string?.asName() ?: pageData.name, route = pageData.meta["page.route"].string?.asName() ?: pageData.name,
data.branch(dataPrefix), data.branch(dataPrefix) ?: DataTree.EMPTY,
pageMeta = pageData.meta, pageMeta = pageData.meta,
pageData.await() pageData.await()
) )

View File

@ -6,16 +6,18 @@ import io.ktor.http.ContentType
import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.CoroutineScope
import kotlinx.io.readByteArray import kotlinx.io.readByteArray
import space.kscience.dataforge.actions.Action import space.kscience.dataforge.actions.Action
import space.kscience.dataforge.actions.mapping import space.kscience.dataforge.actions.transform
import space.kscience.dataforge.context.Context import space.kscience.dataforge.context.Context
import space.kscience.dataforge.context.PluginFactory import space.kscience.dataforge.context.PluginFactory
import space.kscience.dataforge.context.PluginTag import space.kscience.dataforge.context.PluginTag
import space.kscience.dataforge.data.* import space.kscience.dataforge.data.*
import space.kscience.dataforge.io.* import space.kscience.dataforge.io.Binary
import space.kscience.dataforge.io.IOPlugin
import space.kscience.dataforge.io.IOReader
import space.kscience.dataforge.io.JsonMetaFormat
import space.kscience.dataforge.io.yaml.YamlMetaFormat import space.kscience.dataforge.io.yaml.YamlMetaFormat
import space.kscience.dataforge.io.yaml.YamlPlugin import space.kscience.dataforge.io.yaml.YamlPlugin
import space.kscience.dataforge.meta.Meta import space.kscience.dataforge.meta.Meta
import space.kscience.dataforge.meta.get
import space.kscience.dataforge.meta.set import space.kscience.dataforge.meta.set
import space.kscience.dataforge.meta.string import space.kscience.dataforge.meta.string
import space.kscience.dataforge.misc.DFExperimental import space.kscience.dataforge.misc.DFExperimental
@ -26,24 +28,19 @@ import space.kscience.dataforge.workspace.*
import space.kscience.snark.ReWrapAction import space.kscience.snark.ReWrapAction
import space.kscience.snark.Snark import space.kscience.snark.Snark
import space.kscience.snark.SnarkReader import space.kscience.snark.SnarkReader
import space.kscience.snark.TextProcessor
import java.net.URLConnection import java.net.URLConnection
import kotlin.io.path.Path import kotlin.io.path.Path
import kotlin.io.path.extension import kotlin.io.path.extension
public fun <T : Any, R : Any> DataSet<T>.transform(action: Action<T, R>, meta: Meta = Meta.EMPTY): DataSet<R> = public fun <T : Any, R : Any> DataTree<T>.transform(action: Action<T, R>, meta: Meta = Meta.EMPTY): DataTree<R> =
action.execute(this, meta) action.execute(this, meta)
public fun <T : Any> DataSetBuilder<T>.fill(dataSet: DataSet<T>) {
node(Name.EMPTY, dataSet)
}
/** /**
* A plugin used for rendering a [DataTree] as HTML * A plugin used for rendering a [DataTree] as HTML
*/ */
public class SnarkHtml : WorkspacePlugin() { public class SnarkHtml : WorkspacePlugin() {
private val snark by require(Snark) public val snark: Snark by require(Snark)
private val yaml by require(YamlPlugin) private val yaml by require(YamlPlugin)
public val io: IOPlugin get() = snark.io public val io: IOPlugin get() = snark.io
@ -60,61 +57,30 @@ public class SnarkHtml : WorkspacePlugin() {
else -> super.content(target) else -> super.content(target)
} }
private fun getContentType(name: Name, meta: Meta): String = meta[CONTENT_TYPE_KEY].string ?: run { internal fun getContentType(name: Name, meta: Meta): String = meta[CONTENT_TYPE_KEY].string ?: run {
val filePath = meta[FileData.FILE_PATH_KEY]?.string ?: name.toString() val filePath = meta[FileData.FILE_PATH_KEY]?.string ?: name.toString()
URLConnection.guessContentTypeFromName(filePath) ?: Path(filePath).extension URLConnection.guessContentTypeFromName(filePath) ?: Path(filePath).extension
} }
public val prepareHeaderAction: Action<Any, Any> = ReWrapAction.removeExtensions("html", "md") { name -> public val prepareHeaderAction: ReWrapAction<Any> = ReWrapAction.removeExtensions<Any>("html", "md") { name ->
val contentType = getContentType(name, this) val contentType = getContentType(name, this)
set(CONTENT_TYPE_KEY, contentType) set(CONTENT_TYPE_KEY, contentType)
} }
public val parseAction: Action<Any, Any> = Action.mapping { public val removeIndexAction: ReWrapAction<Any> = ReWrapAction.removeIndex<Any>()
val contentType = getContentType(name, meta)
val parser = snark.readers.values.filter { parser -> public val parseAction: Action<Binary, PageFragment> = ParseAction(this)
contentType in parser.types private val allDataNotNull: DataSelector<Any>
}.maxByOrNull { get() = DataSelector { workspace, _ -> workspace.data.filterByType() }
it.priority
}
//pass data for which parser is not found
if (parser == null) {
result { it }
} else {
val preprocessor = meta[TextProcessor.TEXT_PREPROCESSOR_KEY]?.let { snark.preprocessor(it) }
result {
when (it) {
is CharSequence -> {
val string = it.toString()
val preprocessed = preprocessor?.process(string) ?: string
parser.readFrom(preprocessed)
}
is Binary -> {
if (preprocessor == null) {
parser.readFrom(it)
} else {
//TODO provide encoding
val string = it.toByteArray().decodeToString()
parser.readFrom(preprocessor.process(string))
}
}
// bypass for non textual-data
else -> it
}
}
}
}
public val parse: TaskReference<Any> by task<Any>({ public val parse: TaskReference<Any> by task<Any>({
description = "Parse all data for which reader is resolved" description = "Parse all data for which reader is resolved"
}) { }) {
fill(from(allData).transform(parseAction, taskMeta)) //put all data
putAll(from(allDataNotNull))
//override parsed data
putAll(from(allDataNotNull).filterByType<Binary>().transform(parseAction))
} }
@ -136,15 +102,18 @@ public class SnarkHtml : WorkspacePlugin() {
public fun SnarkHtml.readSiteData( public fun SnarkHtml.readSiteData(
binaries: DataSource<Binary>, binaries: DataTree<Binary>,
meta: Meta = Meta.EMPTY, meta: Meta = Meta.EMPTY,
): DataSet<Any> = binaries.transform(parseAction, meta) ): DataTree<Any> = ObservableDataTree(context) {
//put all binaries
putAll(binaries)
//override ones which could be parsed
putAll(binaries.transform(parseAction, meta))
}.transform(prepareHeaderAction, meta).transform(removeIndexAction, meta)
public fun SnarkHtml.readSiteData( public fun SnarkHtml.readSiteData(
coroutineScope: CoroutineScope, coroutineScope: CoroutineScope,
meta: Meta = Meta.EMPTY, meta: Meta = Meta.EMPTY,
builder: context(IOPlugin) DataSourceBuilder<Any>.() -> Unit, builder: DataSink<Binary>.() -> Unit,
): DataSet<Any> = DataSource(coroutineScope) { builder(io, this) } ): DataTree<Any> = readSiteData(ObservableDataTree(coroutineScope) { builder() }, meta)
.transform(prepareHeaderAction, meta)
.transform(parseAction, meta)

View File

@ -8,10 +8,11 @@ import org.intellij.markdown.flavours.commonmark.CommonMarkFlavourDescriptor
import org.intellij.markdown.html.HtmlGenerator import org.intellij.markdown.html.HtmlGenerator
import org.intellij.markdown.parser.MarkdownParser import org.intellij.markdown.parser.MarkdownParser
import space.kscience.snark.SnarkReader import space.kscience.snark.SnarkReader
import kotlin.reflect.KType
import kotlin.reflect.typeOf
public object HtmlReader : SnarkReader<PageFragment> {
public interface SnarkHtmlReader: SnarkReader<PageFragment>
public object HtmlReader : SnarkHtmlReader {
override val types: Set<String> = setOf("html") override val types: Set<String> = setOf("html")
override fun readFrom(source: String): PageFragment = PageFragment { override fun readFrom(source: String): PageFragment = PageFragment {
@ -21,12 +22,9 @@ public object HtmlReader : SnarkReader<PageFragment> {
} }
override fun readFrom(source: Source): PageFragment = readFrom(source.readString()) override fun readFrom(source: Source): PageFragment = readFrom(source.readString())
override val type: KType = typeOf<PageFragment>()
} }
public object MarkdownReader : SnarkReader<PageFragment> { public object MarkdownReader : SnarkHtmlReader {
override val type: KType = typeOf<PageFragment>()
override val types: Set<String> = setOf("text/markdown", "md", "markdown") override val types: Set<String> = setOf("text/markdown", "md", "markdown")
override fun readFrom(source: String): PageFragment = PageFragment { override fun readFrom(source: String): PageFragment = PageFragment {

View File

@ -100,7 +100,7 @@ internal class StaticSiteContext(
) )
} }
override fun page(route: Name, data: DataSet<Any>, pageMeta: Meta, content: HtmlPage) { override fun page(route: Name, data: DataTree<*>?, pageMeta: Meta, content: HtmlPage) {
val modifiedPageMeta = pageMeta.toMutableMeta().apply { val modifiedPageMeta = pageMeta.toMutableMeta().apply {
@ -119,7 +119,7 @@ internal class StaticSiteContext(
newPath.writeText(HtmlPage.createHtmlString(pageContext, data, content)) newPath.writeText(HtmlPage.createHtmlString(pageContext, data, content))
} }
override fun route(route: Name, data: DataSet<*>, siteMeta: Meta, content: HtmlSite) { override fun route(route: Name, data: DataTree<*>?, siteMeta: Meta, content: HtmlSite) {
val siteContextWithData = SiteContextWithData( val siteContextWithData = SiteContextWithData(
StaticSiteContext( StaticSiteContext(
siteMeta = Laminate(siteMeta, this@StaticSiteContext.siteMeta), siteMeta = Laminate(siteMeta, this@StaticSiteContext.siteMeta),
@ -127,7 +127,7 @@ internal class StaticSiteContext(
route = route, route = route,
outputPath = outputPath.resolve(route.toWebPath()) outputPath = outputPath.resolve(route.toWebPath())
), ),
data data ?: DataTree.EMPTY
) )
with(content) { with(content) {
with(siteContextWithData) { with(siteContextWithData) {
@ -136,7 +136,7 @@ internal class StaticSiteContext(
} }
} }
override fun site(route: Name, data: DataSet<Any>, siteMeta: Meta, content: HtmlSite) { override fun site(route: Name, data: DataTree<*>?, siteMeta: Meta, content: HtmlSite) {
val siteContextWithData = SiteContextWithData( val siteContextWithData = SiteContextWithData(
StaticSiteContext( StaticSiteContext(
siteMeta = Laminate(siteMeta, this@StaticSiteContext.siteMeta), siteMeta = Laminate(siteMeta, this@StaticSiteContext.siteMeta),
@ -144,7 +144,7 @@ internal class StaticSiteContext(
route = Name.EMPTY, route = Name.EMPTY,
outputPath = outputPath.resolve(route.toWebPath()) outputPath = outputPath.resolve(route.toWebPath())
), ),
data data ?: DataTree.EMPTY
) )
with(content) { with(content) {
with(siteContextWithData) { with(siteContextWithData) {
@ -162,15 +162,15 @@ internal class StaticSiteContext(
*/ */
@Suppress("UnusedReceiverParameter") @Suppress("UnusedReceiverParameter")
public suspend fun SnarkHtml.staticSite( public suspend fun SnarkHtml.staticSite(
data: DataSet<*>, data: DataTree<*>?,
outputPath: Path, outputPath: Path,
siteUrl: String = outputPath.absolutePathString().replace("\\", "/"), siteUrl: String = outputPath.absolutePathString().replace("\\", "/"),
siteMeta: Meta = data.meta, siteMeta: Meta = data?.meta ?: Meta.EMPTY,
content: HtmlSite, content: HtmlSite,
) { ) {
val siteContextWithData = SiteContextWithData( val siteContextWithData = SiteContextWithData(
StaticSiteContext(siteMeta, siteUrl, Name.EMPTY, outputPath), StaticSiteContext(siteMeta, siteUrl, Name.EMPTY, outputPath),
data data ?: DataTree.EMPTY
) )
with(content) { with(content) {
with(siteContextWithData) { with(siteContextWithData) {

View File

@ -15,8 +15,9 @@ import space.kscience.dataforge.context.ContextAware
import space.kscience.dataforge.context.error import space.kscience.dataforge.context.error
import space.kscience.dataforge.context.logger import space.kscience.dataforge.context.logger
import space.kscience.dataforge.data.Data import space.kscience.dataforge.data.Data
import space.kscience.dataforge.data.DataSet import space.kscience.dataforge.data.DataTree
import space.kscience.dataforge.data.await import space.kscience.dataforge.data.await
import space.kscience.dataforge.data.meta
import space.kscience.dataforge.io.Binary import space.kscience.dataforge.io.Binary
import space.kscience.dataforge.io.toByteArray import space.kscience.dataforge.io.toByteArray
import space.kscience.dataforge.meta.Laminate import space.kscience.dataforge.meta.Laminate
@ -106,7 +107,7 @@ public class KtorSiteContext(
} }
} }
override fun page(route: Name, data: DataSet<*>, pageMeta: Meta, content: HtmlPage) { override fun page(route: Name, data: DataTree<*>?, pageMeta: Meta, content: HtmlPage) {
ktorRoute.get(route.toWebPath()) { ktorRoute.get(route.toWebPath()) {
val request = call.request val request = call.request
//substitute host for url for backwards calls //substitute host for url for backwards calls
@ -129,7 +130,7 @@ public class KtorSiteContext(
} }
} }
override fun route(route: Name, data: DataSet<*>, siteMeta: Meta, content: HtmlSite) { override fun route(route: Name, data: DataTree<*>?, siteMeta: Meta, content: HtmlSite) {
val siteContext = SiteContextWithData( val siteContext = SiteContextWithData(
KtorSiteContext( KtorSiteContext(
context, context,
@ -138,7 +139,7 @@ public class KtorSiteContext(
route = route, route = route,
ktorRoute = ktorRoute.createRouteFromPath(route.toWebPath()) ktorRoute = ktorRoute.createRouteFromPath(route.toWebPath())
), ),
data data ?: DataTree.EMPTY
) )
with(content) { with(content) {
with(siteContext) { with(siteContext) {
@ -147,7 +148,7 @@ public class KtorSiteContext(
} }
} }
override fun site(route: Name, data: DataSet<*>, siteMeta: Meta, content: HtmlSite) { override fun site(route: Name, data: DataTree<*>?, siteMeta: Meta, content: HtmlSite) {
val siteContext = SiteContextWithData( val siteContext = SiteContextWithData(
KtorSiteContext( KtorSiteContext(
context, context,
@ -156,7 +157,7 @@ public class KtorSiteContext(
route = Name.EMPTY, route = Name.EMPTY,
ktorRoute = ktorRoute.createRouteFromPath(route.toWebPath()) ktorRoute = ktorRoute.createRouteFromPath(route.toWebPath())
), ),
data data?: DataTree.EMPTY
) )
with(content) { with(content) {
with(siteContext) { with(siteContext) {
@ -169,14 +170,14 @@ public class KtorSiteContext(
public fun Route.site( public fun Route.site(
context: Context, context: Context,
data: DataSet<*>, data: DataTree<*>?,
baseUrl: String = "", baseUrl: String = "",
siteMeta: Meta = data.meta, siteMeta: Meta = data?.meta ?: Meta.EMPTY,
content: HtmlSite, content: HtmlSite,
) { ) {
val siteContext = SiteContextWithData( val siteContext = SiteContextWithData(
KtorSiteContext(context, siteMeta, baseUrl, route = Name.EMPTY, this@Route), KtorSiteContext(context, siteMeta, baseUrl, route = Name.EMPTY, this@Route),
data data?: DataTree.EMPTY
) )
with(content) { with(content) {
with(siteContext) { with(siteContext) {