SPC/snark
6
1
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Update to DataForge 0.8

Browse Source
This commit is contained in:
Alexander Nozik 2024-02-19 19:47:40 +03:00
parent 3b318c3a8b
commit b66c6b4fe6
17 changed files with 234 additions and 203 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
build.gradle.kts
View File

@ -1,4 +1,5 @@
import space.kscience.gradle.*
import space.kscience.gradle.useApache2Licence
import space.kscience.gradle.useSPCTeam
plugins {
id("space.kscience.gradle.project")
@ -14,7 +15,7 @@ allprojects {
}
}
val dataforgeVersion by extra("0.8.0-dev-1")
val dataforgeVersion by extra("0.8.0")
ksciencePublish {
pom("https://github.com/SciProgCentre/snark") {

2
gradle.properties
View File

@ -1,3 +1,3 @@
kotlin.code.style=official
toolsVersion=0.15.2-kotlin-1.9.21
toolsVersion=0.15.2-kotlin-1.9.22

25
snark-core/src/commonMain/kotlin/space/kscience/snark/ReWrapAction.kt
View File

@ -5,9 +5,7 @@ import space.kscience.dataforge.data.*
import space.kscience.dataforge.meta.Meta
import space.kscience.dataforge.meta.MutableMeta
import space.kscience.dataforge.meta.copy
import space.kscience.dataforge.names.Name
import space.kscience.dataforge.names.NameToken
import space.kscience.dataforge.names.replaceLast
import space.kscience.dataforge.names.*
import kotlin.reflect.KType
import kotlin.reflect.typeOf
@ -19,20 +17,19 @@ public class ReWrapAction<R : Any>(
private val newMeta: MutableMeta.(name: Name) -> Unit = {},
private val newName: (name: Name, meta: Meta?) -> Name,
) : AbstractAction<R, R>(type) {
override fun DataSetBuilder<R>.generate(data: DataSet<R>, meta: Meta) {
override fun DataSink<R>.generate(data: DataTree<R>, meta: Meta) {
data.forEach { namedData ->
data(
put(
newName(namedData.name, namedData.meta),
namedData.data.withMeta(namedData.meta.copy { newMeta(namedData.name) })
)
}
}
override fun DataSourceBuilder<R>.update(dataSet: DataSet<R>, meta: Meta, updateKey: Name) {
val datum = dataSet[updateKey]
data(
newName(updateKey, datum?.meta),
datum?.withMeta(datum.meta.copy { newMeta(updateKey) })
override fun DataSink<R>.update(source: DataTree<R>, meta: Meta, namedData: NamedData<R>) {
put(
newName(namedData.name, namedData.meta),
namedData.withMeta(namedData.meta.copy { newMeta(namedData.name) })
)
}
@ -50,11 +47,15 @@ public class ReWrapAction<R : Any>(
}
}
}
public inline fun <reified R : Any> removeIndex(): ReWrapAction<R> = ReWrapAction<R>(typeOf<R>()) { name, _ ->
if (name.endsWith("index")) name.cutLast() else name
}
}
}
public inline fun <reified R : Any> ReWrapAction(
noinline newMeta: MutableMeta.(name: Name) -> Unit = {},
noinline newName: (Name, Meta?) -> Name
): ReWrapAction<R> = ReWrapAction(typeOf<R>(), newMeta, newName)
noinline newName: (Name, Meta?) -> Name,
): ReWrapAction<R> = ReWrapAction(typeOf<R>(), newMeta, newName)

4
snark-core/src/jvmMain/kotlin/space/kscience/snark/ImageIOReader.kt
View File

@ -5,8 +5,6 @@ import kotlinx.io.asInputStream
import space.kscience.dataforge.io.IOReader
import java.awt.image.BufferedImage
import javax.imageio.ImageIO
import kotlin.reflect.KType
import kotlin.reflect.typeOf
/**
* The ImageIOReader class is an implementation of the IOReader interface specifically for reading images using the ImageIO library.
@ -15,7 +13,5 @@ import kotlin.reflect.typeOf
* @property type The KType of the data to be read by the ImageIOReader.
*/
public object ImageIOReader : IOReader<BufferedImage> {
override val type: KType get() = typeOf<BufferedImage>()
override fun readFrom(source: Source): BufferedImage = ImageIO.read(source.asInputStream())
}

14
snark-core/src/jvmMain/kotlin/space/kscience/snark/extensions.kt
View File

@ -1,16 +1,12 @@
package space.kscience.snark
import space.kscience.dataforge.data.DataSet
import space.kscience.dataforge.data.DataSource
import space.kscience.dataforge.data.DataTree
import space.kscience.dataforge.data.filterByType
import space.kscience.dataforge.names.Name
import space.kscience.dataforge.names.parseAsName
import space.kscience.dataforge.names.startsWith
import kotlin.reflect.typeOf
public inline fun <reified R : Any> DataSet<*>.branch(
public inline fun <reified R : Any> DataTree<*>.branch(
branchName: Name,
): DataSet<R> = filterByType(typeOf<R>()) { name, _ -> name.startsWith(branchName) }
public inline fun <reified R : Any> DataSet<*>.branch(
branchName: String,
): DataSet<R> = filterByType(typeOf<R>()) { name, _ -> name.startsWith(branchName.parseAsName()) }
): DataSource<R> = filterByType<R> { name, _, _ -> name.startsWith(branchName) }

58
snark-core/src/jvmMain/kotlin/space/kscience/snark/snarkWorkspace.kt
View File

@ -2,38 +2,36 @@
package space.kscience.snark
import space.kscience.dataforge.data.DataTree
import space.kscience.dataforge.data.node
import space.kscience.dataforge.io.Binary
import space.kscience.dataforge.io.IOPlugin
import space.kscience.dataforge.data.branch
import space.kscience.dataforge.meta.*
import space.kscience.dataforge.misc.DFExperimental
import space.kscience.dataforge.names.parseAsName
import space.kscience.dataforge.workspace.Workspace
import space.kscience.dataforge.workspace.WorkspaceBuilder
import space.kscience.dataforge.workspace.readRawDirectory
import space.kscience.dataforge.workspace.directory
import space.kscience.dataforge.workspace.resources
import kotlin.io.path.Path
import kotlin.io.path.toPath
/**
* Reads the specified resources and returns a [DataTree] containing the data.
*
* @param resources The names of the resources to read.
* @param classLoader The class loader to use for loading the resources. By default, it uses the current thread's context class loader.
* @return A DataTree containing the data read from the resources.
*/
private fun IOPlugin.readResources(
vararg resources: String,
classLoader: ClassLoader = Thread.currentThread().contextClassLoader,
): DataTree<Binary> = DataTree {
// require(resource.isNotBlank()) {"Can't mount root resource tree as data root"}
resources.forEach { resource ->
val path = classLoader.getResource(resource)?.toURI()?.toPath() ?: error(
"Resource with name $resource is not resolved"
)
node(resource, readRawDirectory(path))
}
}
//
///**
// * Reads the specified resources and returns a [DataTree] containing the data.
// *
// * @param resources The names of the resources to read.
// * @param classLoader The class loader to use for loading the resources. By default, it uses the current thread's context class loader.
// * @return A DataTree containing the data read from the resources.
// */
//private fun IOPlugin.readResources(
// vararg resources: String,
// classLoader: ClassLoader = Thread.currentThread().contextClassLoader,
//): DataTree<Binary> = DataTree {
// // require(resource.isNotBlank()) {"Can't mount root resource tree as data root"}
// resources.forEach { resource ->
// val path = classLoader.getResource(resource)?.toURI()?.toPath() ?: error(
// "Resource with name $resource is not resolved"
// )
// node(resource, readRawDirectory(path))
// }
//}
public fun Snark.workspace(
meta: Meta,
@ -45,14 +43,14 @@ public fun Snark.workspace(
meta.getIndexed("directory").forEach { (index, directoryMeta) ->
val dataDirectory = directoryMeta["path"].string ?: error("Directory path not defined")
val nodeName = directoryMeta["name"].string ?: directoryMeta.string ?: index ?: ""
val data = io.readRawDirectory(Path(dataDirectory))
node(nodeName, data)
directory(io, nodeName.parseAsName(), Path((dataDirectory)))
}
meta.getIndexed("resource").forEach { (index, resourceMeta) ->
val resource = resourceMeta["path"]?.stringList ?: listOf("/")
val nodeName = resourceMeta["name"].string ?: resourceMeta.string ?: index ?: ""
val data: DataTree<Binary> = io.readResources(*resource.toTypedArray())
node(nodeName, data)
branch(nodeName) {
resources(io, *resource.toTypedArray())
}
}
}

14
snark-html/src/jvmMain/kotlin/space/kscience/snark/html/HtmlPage.kt
View File

@ -3,9 +3,9 @@ package space.kscience.snark.html
import kotlinx.html.HTML
import kotlinx.html.stream.createHTML
import kotlinx.html.visitTagAndFinalize
import space.kscience.dataforge.data.DataSet
import space.kscience.dataforge.data.DataSetBuilder
import space.kscience.dataforge.data.static
import space.kscience.dataforge.data.DataSink
import space.kscience.dataforge.data.DataTree
import space.kscience.dataforge.data.wrap
import space.kscience.dataforge.meta.Meta
import space.kscience.dataforge.names.Name
@ -17,11 +17,11 @@ public fun interface HtmlPage {
public companion object {
public fun createHtmlString(
pageContext: PageContext,
dataSet: DataSet<*>,
dataSet: DataTree<*>?,
page: HtmlPage,
): String = createHTML().run {
HTML(kotlinx.html.emptyMap, this, null).visitTagAndFinalize(this) {
with(PageContextWithData(pageContext, dataSet)) {
with(PageContextWithData(pageContext, dataSet ?: DataTree.EMPTY)) {
with(page) {
renderPage()
}
@ -34,13 +34,13 @@ public fun interface HtmlPage {
// data builders
public fun DataSetBuilder<Any>.page(
public fun DataSink<Any>.page(
name: Name,
pageMeta: Meta = Meta.EMPTY,
block: context(PageContextWithData) HTML.() -> Unit,
) {
val page = HtmlPage(block)
static<HtmlPage>(name, page, pageMeta)
wrap<HtmlPage>(name, page, pageMeta)
}

16
snark-html/src/jvmMain/kotlin/space/kscience/snark/html/HtmlSite.kt
View File

@ -1,8 +1,8 @@
package space.kscience.snark.html
import space.kscience.dataforge.data.DataSet
import space.kscience.dataforge.data.DataSetBuilder
import space.kscience.dataforge.data.static
import space.kscience.dataforge.data.DataSink
import space.kscience.dataforge.data.DataTree
import space.kscience.dataforge.data.wrap
import space.kscience.dataforge.meta.Meta
import space.kscience.dataforge.meta.getIndexed
import space.kscience.dataforge.meta.string
@ -15,23 +15,23 @@ public fun interface HtmlSite {
public fun renderSite()
}
public fun DataSetBuilder<Any>.site(
public fun DataSink<Any>.site(
name: Name,
siteMeta: Meta,
block: (siteContext: SiteContext, data: DataSet<Any>) -> Unit,
block: (siteContext: SiteContext, data: DataTree<*>?) -> Unit,
) {
static(name, HtmlSite { block(site, siteData) }, siteMeta)
wrap(name, HtmlSite { block(site, siteData) }, siteMeta)
}
//public fun DataSetBuilder<Any>.site(name: Name, block: DataSetBuilder<Any>.() -> Unit) {
// node(name, block)
//}
internal fun DataSetBuilder<Any>.assetsFrom(rootMeta: Meta) {
internal fun DataSink<Any>.assetsFrom(rootMeta: Meta) {
rootMeta.getIndexed("file".asName()).forEach { (_, meta) ->
val webName: String? by meta.string()
val name by meta.string { error("File path is not provided") }
val fileName = name.parseAsName()
static(fileName, webName?.parseAsName() ?: fileName)
wrap(fileName, webName?.parseAsName() ?: fileName)
}
}

47
snark-html/src/jvmMain/kotlin/space/kscience/snark/html/Language.kt
View File

@ -1,15 +1,14 @@
package space.kscience.snark.html
import space.kscience.dataforge.data.DataSet
import space.kscience.dataforge.data.branch
import space.kscience.dataforge.actions.AbstractAction
import space.kscience.dataforge.actions.transform
import space.kscience.dataforge.data.*
import space.kscience.dataforge.meta.*
import space.kscience.dataforge.names.Name
import space.kscience.dataforge.names.asName
import space.kscience.dataforge.names.parseAsName
import space.kscience.dataforge.names.plus
import space.kscience.dataforge.names.*
import space.kscience.snark.SnarkBuilder
import space.kscience.snark.html.Language.Companion.SITE_LANGUAGES_KEY
import space.kscience.snark.html.Language.Companion.SITE_LANGUAGE_KEY
import kotlin.reflect.typeOf
public class Language : Scheme() {
@ -81,9 +80,37 @@ public val SiteContext.languages: Map<String, Meta>
public val SiteContext.language: String
get() = siteMeta[SITE_LANGUAGE_KEY].string ?: Language.DEFAULT_LANGUAGE
//
//public val SiteContext.languagePrefix: Name
// get() = languages[language]?.let { it[Language::prefix.name].string ?: language }?.parseAsName() ?: Name.EMPTY
public val SiteContext.languagePrefix: Name
get() = languages[language]?.let { it[Language::prefix.name].string ?: language }?.parseAsName() ?: Name.EMPTY
private class LanguageBranchAction(val prefix: Name) : AbstractAction<Any, Any>(typeOf<Any>()) {
override fun DataSink<Any>.generate(data: DataTree<Any>, meta: Meta) {
data.forEach { (name, item) ->
val nameWithoutPrefix = name.removeFirstOrNull(prefix)
if (nameWithoutPrefix != null) {
// put language item as is
put(nameWithoutPrefix, item)
} else if (data[name + prefix] == null) {
// put if language item is missing
put(name, item)
}
}
}
override fun DataSink<Any>.update(source: DataTree<Any>, meta: Meta, namedData: NamedData<Any>) {
val name = namedData.name
val nameWithoutPrefix = name.removeFirstOrNull(prefix)
if (nameWithoutPrefix != null) {
// put language item as is
put(nameWithoutPrefix, namedData.data)
} else if (source[name + prefix] == null) {
// put if language item is missing
put(name, namedData.data)
}
}
}
/**
* Create a multiple sites for different languages. All sites use the same [content], but rely on different data
@ -92,7 +119,7 @@ public val SiteContext.languagePrefix: Name
*/
@SnarkBuilder
public fun SiteContext.multiLanguageSite(
data: DataSet<*>,
data: DataTree<*>,
languageMap: Map<String, Language>,
content: HtmlSite,
) {
@ -108,7 +135,7 @@ public fun SiteContext.multiLanguageSite(
}
site(
prefix.parseAsName(),
data.branch(language.dataPath ?: prefix),
data.filterByType<Any>().transform(LanguageBranchAction(Name.parse(language.dataPath ?: prefix))),
siteMeta = Laminate(languageSiteMeta, siteMeta),
content
)

9
snark-html/src/jvmMain/kotlin/space/kscience/snark/html/PageContext.kt
View File

@ -1,6 +1,6 @@
package space.kscience.snark.html
import space.kscience.dataforge.data.DataSet
import space.kscience.dataforge.data.DataTree
import space.kscience.dataforge.meta.Meta
import space.kscience.dataforge.meta.get
import space.kscience.dataforge.meta.string
@ -23,7 +23,7 @@ public fun Name.toWebPath(): String = tokens.joinToString(separator = "/") {
* A context for building a single page
*/
@SnarkBuilder
public interface PageContext : SnarkContext {
public interface PageContext : SnarkContext {
public val site: SiteContext
@ -57,4 +57,7 @@ public val PageContext.homeRef: String get() = resolvePageRef(SiteContext.INDEX_
public val PageContext.name: Name? get() = pageMeta["name"].string?.parseAsName()
public class PageContextWithData(private val pageContext: PageContext, public val data: DataSet<*>): PageContext by pageContext
public class PageContextWithData(
private val pageContext: PageContext,
public val data: DataTree<*>,
) : PageContext by pageContext

23
snark-html/src/jvmMain/kotlin/space/kscience/snark/html/PageFragment.kt
View File

@ -15,20 +15,19 @@ import space.kscience.snark.SnarkContext
public fun interface PageFragment {
context(PageContextWithData)
public fun FlowContent.renderFragment()
context(PageContextWithData, FlowContent) public fun renderFragment()
}
context(PageContextWithData)
public fun FlowContent.fragment(fragment: PageFragment): Unit{
context(PageContextWithData, FlowContent)
public fun fragment(fragment: PageFragment): Unit {
with(fragment) {
renderFragment()
}
}
context(PageContextWithData)
public fun FlowContent.fragment(data: Data<PageFragment>): Unit = runBlocking {
context(PageContextWithData, FlowContent)
public fun fragment(data: Data<PageFragment>): Unit = runBlocking {
fragment(data.await())
}
@ -54,7 +53,7 @@ public val Data<*>.published: Boolean
* Resolve a Html builder by its full name
*/
context(SnarkContext)
public fun DataSet<*>.resolveHtmlOrNull(name: Name): Data<PageFragment>? {
public fun DataTree<*>.resolveHtmlOrNull(name: Name): Data<PageFragment>? {
val resolved = (getByType<PageFragment>(name) ?: getByType<PageFragment>(name + SiteContext.INDEX_PAGE_TOKEN))
return resolved?.takeIf {
@ -63,26 +62,26 @@ public fun DataSet<*>.resolveHtmlOrNull(name: Name): Data<PageFragment>? {
}
context(SnarkContext)
public fun DataSet<*>.resolveHtmlOrNull(name: String): Data<PageFragment>? = resolveHtmlOrNull(name.parseAsName())
public fun DataTree<*>.resolveHtmlOrNull(name: String): Data<PageFragment>? = resolveHtmlOrNull(name.parseAsName())
context(SnarkContext)
public fun DataSet<*>.resolveHtml(name: String): Data<PageFragment> = resolveHtmlOrNull(name)
public fun DataTree<*>.resolveHtml(name: String): Data<PageFragment> = resolveHtmlOrNull(name)
?: error("Html fragment with name $name is not resolved")
/**
* Find all Html blocks using given name/meta filter
*/
context(SnarkContext)
public fun DataSet<*>.resolveAllHtml(
public fun DataTree<*>.resolveAllHtml(
predicate: (name: Name, meta: Meta) -> Boolean,
): Map<Name, Data<PageFragment>> = filterByType<PageFragment> { name, meta ->
): Map<Name, Data<PageFragment>> = filterByType<PageFragment> { name, meta, _ ->
predicate(name, meta)
&& meta["published"].string != "false"
//TODO add language confirmation
}.asSequence().associate { it.name to it.data }
context(SnarkContext)
public fun DataSet<*>.findHtmlByContentType(
public fun DataTree<*>.findHtmlByContentType(
contentType: String,
baseName: Name = Name.EMPTY,
): Map<Name, Data<PageFragment>> = resolveAllHtml { name, meta ->

50
snark-html/src/jvmMain/kotlin/space/kscience/snark/html/ParseAction.kt Normal file
View File

@ -0,0 +1,50 @@
package space.kscience.snark.html
import space.kscience.dataforge.actions.AbstractAction
import space.kscience.dataforge.data.*
import space.kscience.dataforge.io.Binary
import space.kscience.dataforge.io.toByteArray
import space.kscience.dataforge.meta.Meta
import space.kscience.dataforge.meta.get
import space.kscience.snark.TextProcessor
import kotlin.reflect.typeOf
public class ParseAction(private val snarkHtml: SnarkHtml) :
AbstractAction<Binary, PageFragment>(typeOf<PageFragment>()) {
private fun parseOne(data: NamedData<Binary>, actionMeta: Meta): NamedData<PageFragment>? = with(snarkHtml) {
val contentType = getContentType(data.name, data.meta)
val parser = snark.readers.values.filterIsInstance<SnarkHtmlReader>().filter { parser ->
contentType in parser.types
}.maxByOrNull {
it.priority
}
//ignore data for which parser is not found
if (parser != null) {
val preprocessor = meta[TextProcessor.TEXT_PREPROCESSOR_KEY]?.let { snark.preprocessor(it) }
data.transform {
if (preprocessor == null) {
parser.readFrom(it)
} else {
//TODO provide encoding
val string = it.toByteArray().decodeToString()
parser.readFrom(preprocessor.process(string))
}
}.named(data.name)
} else {
null
}
}
override fun DataSink<PageFragment>.generate(data: DataTree<Binary>, meta: Meta) {
data.forEach {
parseOne(it, meta)?.let { put(it) }
}
}
override fun DataSink<PageFragment>.update(source: DataTree<Binary>, meta: Meta, namedData: NamedData<Binary>) {
parseOne(namedData,meta)?.let { put(it) }
}
}

38
snark-html/src/jvmMain/kotlin/space/kscience/snark/html/SiteContext.kt
View File

@ -8,8 +8,6 @@ import space.kscience.dataforge.meta.string
import space.kscience.dataforge.names.*
import space.kscience.snark.SnarkBuilder
import space.kscience.snark.SnarkContext
import kotlin.reflect.full.isSubtypeOf
import kotlin.reflect.typeOf
/**
@ -45,7 +43,7 @@ public interface SiteContext : SnarkContext {
@SnarkBuilder
public fun page(
route: Name,
data: DataSet<*>,
data: DataTree<*>?,
pageMeta: Meta = Meta.EMPTY,
content: HtmlPage,
)
@ -56,7 +54,7 @@ public interface SiteContext : SnarkContext {
@SnarkBuilder
public fun route(
route: Name,
data: DataSet<*>,
data: DataTree<*>?,
siteMeta: Meta = Meta.EMPTY,
content: HtmlSite,
)
@ -68,7 +66,7 @@ public interface SiteContext : SnarkContext {
@SnarkBuilder
public fun site(
route: Name,
data: DataSet<*>,
data: DataTree<*>?,
siteMeta: Meta = Meta.EMPTY,
content: HtmlSite,
)
@ -81,27 +79,21 @@ public interface SiteContext : SnarkContext {
}
}
public fun SiteContext.static(dataSet: DataSet<Binary>, prefix: Name = Name.EMPTY) {
public fun SiteContext.static(dataSet: DataTree<Binary>, prefix: Name = Name.EMPTY) {
dataSet.forEach { (name, data) ->
static(prefix + name, data)
}
}
public fun SiteContext.static(dataSet: DataSet<*>, branch: String, prefix: String = branch) {
public fun SiteContext.static(dataSet: DataTree<*>, branch: String, prefix: String = branch) {
val branchName = branch.parseAsName()
val prefixName = prefix.parseAsName()
val binaryType = typeOf<Binary>()
dataSet.forEach { (name, data) ->
@Suppress("UNCHECKED_CAST")
if (name.startsWith(branchName) && data.type.isSubtypeOf(binaryType)) {
static(prefixName + name, data as Data<Binary>)
}
dataSet.branch(branchName)?.filterByType<Binary>()?.forEach {
static(prefixName + it.name, it.data)
}
}
context(SiteContext)
public val site: SiteContext
get() = this@SiteContext
@ -110,7 +102,7 @@ public val site: SiteContext
/**
* A wrapper for site context that allows convenient site building experience
*/
public class SiteContextWithData(private val site: SiteContext, public val siteData: DataSet<*>) : SiteContext by site
public class SiteContextWithData(private val site: SiteContext, public val siteData: DataTree<*>) : SiteContext by site
@SnarkBuilder
@ -127,23 +119,23 @@ public fun SiteContextWithData.page(
@SnarkBuilder
public fun SiteContextWithData.route(
route: String,
data: DataSet<*> = siteData,
data: DataTree<*>? = siteData,
siteMeta: Meta = Meta.EMPTY,
content: HtmlSite,
): Unit = route(route.parseAsName(), data, siteMeta,content)
): Unit = route(route.parseAsName(), data, siteMeta, content)
@SnarkBuilder
public fun SiteContextWithData.site(
route: String,
data: DataSet<*> = siteData,
data: DataTree<*>? = siteData,
siteMeta: Meta = Meta.EMPTY,
content: HtmlSite,
): Unit = site(route.parseAsName(), data, siteMeta,content)
): Unit = site(route.parseAsName(), data, siteMeta, content)
/**
* Render all pages and sites found in the data
*/
public suspend fun SiteContext.renderPages(data: DataSet<*>): Unit {
public suspend fun SiteContext.renderPages(data: DataTree<*>): Unit {
// Render all sub-sites
data.filterByType<HtmlSite>().forEach { siteData: NamedData<HtmlSite> ->
@ -151,7 +143,7 @@ public suspend fun SiteContext.renderPages(data: DataSet<*>): Unit {
val dataPrefix = siteData.meta["site.dataPath"].string?.asName() ?: Name.EMPTY
site(
route = siteData.meta["site.route"].string?.asName() ?: siteData.name,
data.branch(dataPrefix),
data.branch(dataPrefix) ?: DataTree.EMPTY,
siteMeta = siteData.meta,
siteData.await()
)
@ -162,7 +154,7 @@ public suspend fun SiteContext.renderPages(data: DataSet<*>): Unit {
val dataPrefix = pageData.meta["page.dataPath"].string?.asName() ?: Name.EMPTY
page(
route = pageData.meta["page.route"].string?.asName() ?: pageData.name,
data.branch(dataPrefix),
data.branch(dataPrefix) ?: DataTree.EMPTY,
pageMeta = pageData.meta,
pageData.await()
)

83
snark-html/src/jvmMain/kotlin/space/kscience/snark/html/SnarkHtml.kt
View File

@ -6,16 +6,18 @@ import io.ktor.http.ContentType
import kotlinx.coroutines.CoroutineScope
import kotlinx.io.readByteArray
import space.kscience.dataforge.actions.Action
import space.kscience.dataforge.actions.mapping
import space.kscience.dataforge.actions.transform
import space.kscience.dataforge.context.Context
import space.kscience.dataforge.context.PluginFactory
import space.kscience.dataforge.context.PluginTag
import space.kscience.dataforge.data.*
import space.kscience.dataforge.io.*
import space.kscience.dataforge.io.Binary
import space.kscience.dataforge.io.IOPlugin
import space.kscience.dataforge.io.IOReader
import space.kscience.dataforge.io.JsonMetaFormat
import space.kscience.dataforge.io.yaml.YamlMetaFormat
import space.kscience.dataforge.io.yaml.YamlPlugin
import space.kscience.dataforge.meta.Meta
import space.kscience.dataforge.meta.get
import space.kscience.dataforge.meta.set
import space.kscience.dataforge.meta.string
import space.kscience.dataforge.misc.DFExperimental
@ -26,24 +28,19 @@ import space.kscience.dataforge.workspace.*
import space.kscience.snark.ReWrapAction
import space.kscience.snark.Snark
import space.kscience.snark.SnarkReader
import space.kscience.snark.TextProcessor
import java.net.URLConnection
import kotlin.io.path.Path
import kotlin.io.path.extension
public fun <T : Any, R : Any> DataSet<T>.transform(action: Action<T, R>, meta: Meta = Meta.EMPTY): DataSet<R> =
public fun <T : Any, R : Any> DataTree<T>.transform(action: Action<T, R>, meta: Meta = Meta.EMPTY): DataTree<R> =
action.execute(this, meta)
public fun <T : Any> DataSetBuilder<T>.fill(dataSet: DataSet<T>) {
node(Name.EMPTY, dataSet)
}
/**
* A plugin used for rendering a [DataTree] as HTML
*/
public class SnarkHtml : WorkspacePlugin() {
private val snark by require(Snark)
public val snark: Snark by require(Snark)
private val yaml by require(YamlPlugin)
public val io: IOPlugin get() = snark.io
@ -60,61 +57,30 @@ public class SnarkHtml : WorkspacePlugin() {
else -> super.content(target)
}
private fun getContentType(name: Name, meta: Meta): String = meta[CONTENT_TYPE_KEY].string ?: run {
internal fun getContentType(name: Name, meta: Meta): String = meta[CONTENT_TYPE_KEY].string ?: run {
val filePath = meta[FileData.FILE_PATH_KEY]?.string ?: name.toString()
URLConnection.guessContentTypeFromName(filePath) ?: Path(filePath).extension
}
public val prepareHeaderAction: Action<Any, Any> = ReWrapAction.removeExtensions("html", "md") { name ->
public val prepareHeaderAction: ReWrapAction<Any> = ReWrapAction.removeExtensions<Any>("html", "md") { name ->
val contentType = getContentType(name, this)
set(CONTENT_TYPE_KEY, contentType)
}
public val parseAction: Action<Any, Any> = Action.mapping {
val contentType = getContentType(name, meta)
public val removeIndexAction: ReWrapAction<Any> = ReWrapAction.removeIndex<Any>()
val parser = snark.readers.values.filter { parser ->
contentType in parser.types
}.maxByOrNull {
it.priority
}
//pass data for which parser is not found
if (parser == null) {
result { it }
} else {
val preprocessor = meta[TextProcessor.TEXT_PREPROCESSOR_KEY]?.let { snark.preprocessor(it) }
result {
when (it) {
is CharSequence -> {
val string = it.toString()
val preprocessed = preprocessor?.process(string) ?: string
parser.readFrom(preprocessed)
}
is Binary -> {
if (preprocessor == null) {
parser.readFrom(it)
} else {
//TODO provide encoding
val string = it.toByteArray().decodeToString()
parser.readFrom(preprocessor.process(string))
}
}
// bypass for non textual-data
else -> it
}
}
}
}
public val parseAction: Action<Binary, PageFragment> = ParseAction(this)
private val allDataNotNull: DataSelector<Any>
get() = DataSelector { workspace, _ -> workspace.data.filterByType() }
public val parse: TaskReference<Any> by task<Any>({
description = "Parse all data for which reader is resolved"
}) {
fill(from(allData).transform(parseAction, taskMeta))
//put all data
putAll(from(allDataNotNull))
//override parsed data
putAll(from(allDataNotNull).filterByType<Binary>().transform(parseAction))
}
@ -136,15 +102,18 @@ public class SnarkHtml : WorkspacePlugin() {
public fun SnarkHtml.readSiteData(
binaries: DataSource<Binary>,
binaries: DataTree<Binary>,
meta: Meta = Meta.EMPTY,
): DataSet<Any> = binaries.transform(parseAction, meta)
): DataTree<Any> = ObservableDataTree(context) {
//put all binaries
putAll(binaries)
//override ones which could be parsed
putAll(binaries.transform(parseAction, meta))
}.transform(prepareHeaderAction, meta).transform(removeIndexAction, meta)
public fun SnarkHtml.readSiteData(
coroutineScope: CoroutineScope,
meta: Meta = Meta.EMPTY,
builder: context(IOPlugin) DataSourceBuilder<Any>.() -> Unit,
): DataSet<Any> = DataSource(coroutineScope) { builder(io, this) }
.transform(prepareHeaderAction, meta)
.transform(parseAction, meta)
builder: DataSink<Binary>.() -> Unit,
): DataTree<Any> = readSiteData(ObservableDataTree(coroutineScope) { builder() }, meta)

12
snark-html/src/jvmMain/kotlin/space/kscience/snark/html/readers.kt
View File

@ -8,10 +8,11 @@ import org.intellij.markdown.flavours.commonmark.CommonMarkFlavourDescriptor
import org.intellij.markdown.html.HtmlGenerator
import org.intellij.markdown.parser.MarkdownParser
import space.kscience.snark.SnarkReader
import kotlin.reflect.KType
import kotlin.reflect.typeOf
public object HtmlReader : SnarkReader<PageFragment> {
public interface SnarkHtmlReader: SnarkReader<PageFragment>
public object HtmlReader : SnarkHtmlReader {
override val types: Set<String> = setOf("html")
override fun readFrom(source: String): PageFragment = PageFragment {
@ -21,12 +22,9 @@ public object HtmlReader : SnarkReader<PageFragment> {
}
override fun readFrom(source: Source): PageFragment = readFrom(source.readString())
override val type: KType = typeOf<PageFragment>()
}
public object MarkdownReader : SnarkReader<PageFragment> {
override val type: KType = typeOf<PageFragment>()
public object MarkdownReader : SnarkHtmlReader {
override val types: Set<String> = setOf("text/markdown", "md", "markdown")
override fun readFrom(source: String): PageFragment = PageFragment {

18
snark-html/src/jvmMain/kotlin/space/kscience/snark/html/static/StaticSiteContext.kt
View File

@ -100,7 +100,7 @@ internal class StaticSiteContext(
)
}
override fun page(route: Name, data: DataSet<Any>, pageMeta: Meta, content: HtmlPage) {
override fun page(route: Name, data: DataTree<*>?, pageMeta: Meta, content: HtmlPage) {
val modifiedPageMeta = pageMeta.toMutableMeta().apply {
@ -119,7 +119,7 @@ internal class StaticSiteContext(
newPath.writeText(HtmlPage.createHtmlString(pageContext, data, content))
}
override fun route(route: Name, data: DataSet<*>, siteMeta: Meta, content: HtmlSite) {
override fun route(route: Name, data: DataTree<*>?, siteMeta: Meta, content: HtmlSite) {
val siteContextWithData = SiteContextWithData(
StaticSiteContext(
siteMeta = Laminate(siteMeta, this@StaticSiteContext.siteMeta),
@ -127,7 +127,7 @@ internal class StaticSiteContext(
route = route,
outputPath = outputPath.resolve(route.toWebPath())
),
data
data ?: DataTree.EMPTY
)
with(content) {
with(siteContextWithData) {
@ -136,7 +136,7 @@ internal class StaticSiteContext(
}
}
override fun site(route: Name, data: DataSet<Any>, siteMeta: Meta, content: HtmlSite) {
override fun site(route: Name, data: DataTree<*>?, siteMeta: Meta, content: HtmlSite) {
val siteContextWithData = SiteContextWithData(
StaticSiteContext(
siteMeta = Laminate(siteMeta, this@StaticSiteContext.siteMeta),
@ -144,7 +144,7 @@ internal class StaticSiteContext(
route = Name.EMPTY,
outputPath = outputPath.resolve(route.toWebPath())
),
data
data ?: DataTree.EMPTY
)
with(content) {
with(siteContextWithData) {
@ -162,17 +162,17 @@ internal class StaticSiteContext(
*/
@Suppress("UnusedReceiverParameter")
public suspend fun SnarkHtml.staticSite(
data: DataSet<*>,
data: DataTree<*>?,
outputPath: Path,
siteUrl: String = outputPath.absolutePathString().replace("\\", "/"),
siteMeta: Meta = data.meta,
siteMeta: Meta = data?.meta ?: Meta.EMPTY,
content: HtmlSite,
) {
val siteContextWithData = SiteContextWithData(
StaticSiteContext(siteMeta, siteUrl, Name.EMPTY, outputPath),
data
data ?: DataTree.EMPTY
)
with(content){
with(content) {
with(siteContextWithData) {
renderSite()
}

19
snark-ktor/src/jvmMain/kotlin/space/kscience/snark/ktor/KtorSiteContext.kt
View File

@ -15,8 +15,9 @@ import space.kscience.dataforge.context.ContextAware
import space.kscience.dataforge.context.error
import space.kscience.dataforge.context.logger
import space.kscience.dataforge.data.Data
import space.kscience.dataforge.data.DataSet
import space.kscience.dataforge.data.DataTree
import space.kscience.dataforge.data.await
import space.kscience.dataforge.data.meta
import space.kscience.dataforge.io.Binary
import space.kscience.dataforge.io.toByteArray
import space.kscience.dataforge.meta.Laminate
@ -106,7 +107,7 @@ public class KtorSiteContext(
}
}
override fun page(route: Name, data: DataSet<*>, pageMeta: Meta, content: HtmlPage) {
override fun page(route: Name, data: DataTree<*>?, pageMeta: Meta, content: HtmlPage) {
ktorRoute.get(route.toWebPath()) {
val request = call.request
//substitute host for url for backwards calls
@ -129,7 +130,7 @@ public class KtorSiteContext(
}
}
override fun route(route: Name, data: DataSet<*>, siteMeta: Meta, content: HtmlSite) {
override fun route(route: Name, data: DataTree<*>?, siteMeta: Meta, content: HtmlSite) {
val siteContext = SiteContextWithData(
KtorSiteContext(
context,
@ -138,7 +139,7 @@ public class KtorSiteContext(
route = route,
ktorRoute = ktorRoute.createRouteFromPath(route.toWebPath())
),
data
data ?: DataTree.EMPTY
)
with(content) {
with(siteContext) {
@ -147,7 +148,7 @@ public class KtorSiteContext(
}
}
override fun site(route: Name, data: DataSet<*>, siteMeta: Meta, content: HtmlSite) {
override fun site(route: Name, data: DataTree<*>?, siteMeta: Meta, content: HtmlSite) {
val siteContext = SiteContextWithData(
KtorSiteContext(
context,
@ -156,7 +157,7 @@ public class KtorSiteContext(
route = Name.EMPTY,
ktorRoute = ktorRoute.createRouteFromPath(route.toWebPath())
),
data
data?: DataTree.EMPTY
)
with(content) {
with(siteContext) {
@ -169,14 +170,14 @@ public class KtorSiteContext(
public fun Route.site(
context: Context,
data: DataSet<*>,
data: DataTree<*>?,
baseUrl: String = "",
siteMeta: Meta = data.meta,
siteMeta: Meta = data?.meta ?: Meta.EMPTY,
content: HtmlSite,
) {
val siteContext = SiteContextWithData(
KtorSiteContext(context, siteMeta, baseUrl, route = Name.EMPTY, this@Route),
data
data?: DataTree.EMPTY
)
with(content) {
with(siteContext) {