SPC/snark
5
1
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[WIP] refactor in progress

Browse Source
This commit is contained in:
Alexander Nozik 2024-01-04 11:26:26 +03:00
parent c0f869f6e3
commit 3d44ea9a88
14 changed files with 315 additions and 173 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
gradle.properties
View File

@ -1,3 +1,3 @@
kotlin.code.style=official
toolsVersion=0.15.1-kotlin-1.9.21
toolsVersion=0.15.2-kotlin-1.9.21

2
snark-core/src/commonMain/kotlin/space/kscience/snark/Snark.kt
View File

@ -34,7 +34,7 @@ public class Snark : WorkspacePlugin() {
context.gather(TextProcessor.DF_TYPE, true)
}
public fun textProcessor(transformationMeta: Meta): TextProcessor {
public fun preprocessor(transformationMeta: Meta): TextProcessor {
val transformationName = transformationMeta.string
?: transformationMeta["name"].string ?: error("Transformation name not defined in $transformationMeta")
return textProcessors[transformationName.parseAsName()]

16
snark-core/src/jvmMain/kotlin/space/kscience/snark/snarkWorkspace.kt
View File

@ -27,15 +27,13 @@ import kotlin.io.path.toPath
private fun IOPlugin.readResources(
vararg resources: String,
classLoader: ClassLoader = Thread.currentThread().contextClassLoader,
): DataTree<Binary> {
// require(resource.isNotBlank()) {"Can't mount root resource tree as data root"}
return DataTree {
resources.forEach { resource ->
val path = classLoader.getResource(resource)?.toURI()?.toPath() ?: error(
"Resource with name $resource is not resolved"
)
node(resource, readRawDirectory(path))
}
): DataTree<Binary> = DataTree {
// require(resource.isNotBlank()) {"Can't mount root resource tree as data root"}
resources.forEach { resource ->
val path = classLoader.getResource(resource)?.toURI()?.toPath() ?: error(
"Resource with name $resource is not resolved"
)
node(resource, readRawDirectory(path))
}
}

27
snark-html/src/jvmMain/kotlin/space/kscience/snark/html/HtmlPage.kt
View File

@ -10,18 +10,23 @@ import space.kscience.dataforge.meta.Meta
import space.kscience.dataforge.names.Name
public fun interface HtmlPage {
public suspend fun HTML.renderPage(page: PageContext, data: DataSet<*>)
public companion object{
public suspend fun createHtmlString(pageContext: PageContext, page: HtmlPage, data: DataSet<*>): String{
return createHTML().run {
HTML(kotlinx.html.emptyMap, this, null).visitTagAndFinalize(this) {
context(PageContextWithData)
public fun HTML.renderPage()
public companion object {
public fun createHtmlString(
pageContext: PageContext,
dataSet: DataSet<*>,
page: HtmlPage,
): String = createHTML().run {
HTML(kotlinx.html.emptyMap, this, null).visitTagAndFinalize(this) {
with(PageContextWithData(pageContext, dataSet)) {
with(page) {
renderPage(pageContext, data)
renderPage()
}
}
}
}
}
}
@ -29,14 +34,16 @@ public fun interface HtmlPage {
// data builders
public fun DataSetBuilder<Any>.page(name: Name, pageMeta: Meta = Meta.EMPTY, block: HTML.(pageContext: PageContext, pageData: DataSet<Any>) -> Unit) {
public fun DataSetBuilder<Any>.page(
name: Name,
pageMeta: Meta = Meta.EMPTY,
block: context(PageContextWithData) HTML.() -> Unit,
) {
val page = HtmlPage(block)
static<HtmlPage>(name, page, pageMeta)
}
// if (data.type == typeOf<HtmlData>()) {
// val languageMeta: Meta = Language.forName(name)
//

7
snark-html/src/jvmMain/kotlin/space/kscience/snark/html/HtmlSite.kt
View File

@ -11,15 +11,16 @@ import space.kscience.dataforge.names.asName
import space.kscience.dataforge.names.parseAsName
public fun interface HtmlSite {
public suspend fun SiteContext.renderSite(data: DataSet<Any>)
context(SiteContextWithData)
public fun renderSite()
}
public fun DataSetBuilder<Any>.site(
name: Name,
siteMeta: Meta,
block: (siteContext: SiteContext, siteData: DataSet<Any>) -> Unit,
block: (siteContext: SiteContext, data: DataSet<Any>) -> Unit,
) {
static(name, HtmlSite(block), siteMeta)
static(name, HtmlSite { block(site, siteData) }, siteMeta)
}
//public fun DataSetBuilder<Any>.site(name: Name, block: DataSetBuilder<Any>.() -> Unit) {

51
snark-html/src/jvmMain/kotlin/space/kscience/snark/html/Language.kt
View File

@ -3,13 +3,17 @@ package space.kscience.snark.html
import space.kscience.dataforge.data.DataSet
import space.kscience.dataforge.data.branch
import space.kscience.dataforge.meta.*
import space.kscience.dataforge.names.*
import space.kscience.dataforge.names.Name
import space.kscience.dataforge.names.asName
import space.kscience.dataforge.names.parseAsName
import space.kscience.dataforge.names.plus
import space.kscience.snark.SnarkBuilder
import space.kscience.snark.html.Language.Companion.SITE_LANGUAGES_KEY
import space.kscience.snark.html.Language.Companion.SITE_LANGUAGE_KEY
public class Language : Scheme() {
/**
* Language key override
*/
@ -20,6 +24,11 @@ public class Language : Scheme() {
*/
public var prefix: String? by string()
/**
* An override for data path. By default uses [prefix]
*/
public var dataPath: String? by string()
/**
* Target page name with a given language key
*/
@ -33,21 +42,21 @@ public class Language : Scheme() {
public val LANGUAGE_KEY: Name = "language".asName()
public val LANGUAGES_KEY: Name = "languages".asName()
public val LANGUAGE_MAP_KEY: Name = "languageMap".asName()
public val SITE_LANGUAGE_KEY: Name = SiteContext.SITE_META_KEY + LANGUAGE_KEY
public val SITE_LANGUAGES_KEY: Name = SiteContext.SITE_META_KEY + LANGUAGES_KEY
public val SITE_LANGUAGES_KEY: Name = SiteContext.SITE_META_KEY + LANGUAGE_MAP_KEY
public const val DEFAULT_LANGUAGE: String = "en"
//
// /**
// * Automatically build a language map for a data piece with given [name] based on existence of appropriate data nodes.
// */
// context(SiteContext)
// public fun forName(name: Name): Meta = Meta {
// context(PageContextWithData)
// public fun languageMapFor(name: Name): Meta = Meta {
// val currentLanguagePrefix = languages[language]?.get(Language::prefix.name)?.string ?: language
// val fullName = (route.removeFirstOrNull(currentLanguagePrefix.asName()) ?: route) + name
// val fullName = (site.route.removeFirstOrNull(currentLanguagePrefix.asName()) ?: site.route) + name
// languages.forEach { (key, meta) ->
// val languagePrefix: String = meta[Language::prefix.name].string ?: key
// val nameWithLanguage: Name = if (languagePrefix.isBlank()) {
@ -55,7 +64,7 @@ public class Language : Scheme() {
// } else {
// languagePrefix.asName() + fullName
// }
// if (resolveData.getItem(name) != null) {
// if (data.resolveHtmlOrNull(name) != null) {
// key put meta.asMutableMeta().apply {
// Language::target.name put nameWithLanguage.toString()
// }
@ -65,6 +74,8 @@ public class Language : Scheme() {
}
}
public fun Language(prefix: String): Language = Language { this.prefix = prefix }
public val SiteContext.languages: Map<String, Meta>
get() = siteMeta[SITE_LANGUAGES_KEY]?.items?.mapKeys { it.key.toStringUnescaped() } ?: emptyMap()
@ -74,8 +85,17 @@ public val SiteContext.language: String
public val SiteContext.languagePrefix: Name
get() = languages[language]?.let { it[Language::prefix.name].string ?: language }?.parseAsName() ?: Name.EMPTY
/**
* Create a multiple sites for different languages. All sites use the same [content], but rely on different data
*
* @param data a common data root for all sites
*/
@SnarkBuilder
public suspend fun SiteContext.multiLanguageSite(data: DataSet<Any>, languageMap: Map<String, Language>, site: HtmlSite) {
public fun SiteContext.multiLanguageSite(
data: DataSet<*>,
languageMap: Map<String, Language>,
content: HtmlSite,
) {
languageMap.forEach { (languageKey, language) ->
val prefix = language.prefix ?: languageKey
val languageSiteMeta = Meta {
@ -86,7 +106,12 @@ public suspend fun SiteContext.multiLanguageSite(data: DataSet<Any>, languageMap
}
}
}
site(prefix.parseAsName(), data.branch(prefix), siteMeta = Laminate(languageSiteMeta, siteMeta), site)
site(
prefix.parseAsName(),
data.branch(language.dataPath ?: prefix),
siteMeta = Laminate(languageSiteMeta, siteMeta),
content
)
}
}
@ -99,11 +124,11 @@ public val PageContext.language: String
/**
* Mapping of language keys to other language versions of this page
*/
public val PageContext.languages: Map<String, Meta>
get() = pageMeta[Language.LANGUAGES_KEY]?.items?.mapKeys { it.key.toStringUnescaped() } ?: emptyMap()
public fun PageContext.getLanguageMap(): Map<String, Meta> =
pageMeta[Language.LANGUAGE_MAP_KEY]?.items?.mapKeys { it.key.toStringUnescaped() } ?: emptyMap()
public fun PageContext.localisedPageRef(pageName: Name, relative: Boolean = false): String {
val prefix = languages[language]?.get(Language::prefix.name)?.string?.parseAsName() ?: Name.EMPTY
val prefix = getLanguageMap()[language]?.get(Language::prefix.name)?.string?.parseAsName() ?: Name.EMPTY
return resolvePageRef(prefix + pageName, relative)
}

15
snark-html/src/jvmMain/kotlin/space/kscience/snark/html/PageContext.kt
View File

@ -1,13 +1,13 @@
package space.kscience.snark.html
import kotlinx.html.HTML
import space.kscience.dataforge.context.Context
import space.kscience.dataforge.context.ContextAware
import space.kscience.dataforge.data.*
import space.kscience.dataforge.data.DataSet
import space.kscience.dataforge.meta.Meta
import space.kscience.dataforge.meta.get
import space.kscience.dataforge.meta.string
import space.kscience.dataforge.names.*
import space.kscience.dataforge.names.Name
import space.kscience.dataforge.names.asName
import space.kscience.dataforge.names.hasIndex
import space.kscience.dataforge.names.parseAsName
import space.kscience.snark.SnarkBuilder
import space.kscience.snark.SnarkContext
@ -54,4 +54,7 @@ public fun PageContext.resolvePageRef(pageName: String): String = resolvePageRef
public val PageContext.homeRef: String get() = resolvePageRef(SiteContext.INDEX_PAGE_TOKEN.asName())
public val PageContext.name: Name? get() = pageMeta["name"].string?.parseAsName()
public val PageContext.name: Name? get() = pageMeta["name"].string?.parseAsName()
public class PageContextWithData(private val pageContext: PageContext, public val data: DataSet<*>): PageContext by pageContext

33
snark-html/src/jvmMain/kotlin/space/kscience/snark/html/DataFragment.kt → snark-html/src/jvmMain/kotlin/space/kscience/snark/html/PageFragment.kt
View File

@ -1,6 +1,5 @@
package space.kscience.snark.html
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.runBlocking
import kotlinx.html.FlowContent
import space.kscience.dataforge.data.*
@ -14,16 +13,26 @@ import space.kscience.dataforge.names.plus
import space.kscience.dataforge.names.startsWith
import space.kscience.snark.SnarkContext
public fun interface DataFragment {
public suspend fun FlowContent.renderFragment(page: PageContext, data: DataSet<*>)
public fun interface PageFragment {
context(PageContextWithData)
public fun FlowContent.renderFragment()
}
context(PageContextWithData)
public fun FlowContent.fragment(fragment: PageFragment): Unit{
with(fragment) {
renderFragment()
}
}
context(PageContext)
public fun FlowContent.htmlData(data: DataSet<*>, fragment: Data<DataFragment>): Unit = runBlocking(Dispatchers.IO) {
with(fragment.await()) { renderFragment(page, data) }
context(PageContextWithData)
public fun FlowContent.fragment(data: Data<PageFragment>): Unit = runBlocking {
fragment(data.await())
}
context(SnarkContext)
public val Data<*>.id: String
get() = meta["id"]?.string ?: "block[${hashCode()}]"
@ -45,8 +54,8 @@ public val Data<*>.published: Boolean
* Resolve a Html builder by its full name
*/
context(SnarkContext)
public fun DataSet<*>.resolveHtmlOrNull(name: Name): Data<DataFragment>? {
val resolved = (getByType<DataFragment>(name) ?: getByType<DataFragment>(name + SiteContext.INDEX_PAGE_TOKEN))
public fun DataSet<*>.resolveHtmlOrNull(name: Name): Data<PageFragment>? {
val resolved = (getByType<PageFragment>(name) ?: getByType<PageFragment>(name + SiteContext.INDEX_PAGE_TOKEN))
return resolved?.takeIf {
it.published //TODO add language confirmation
@ -54,10 +63,10 @@ public fun DataSet<*>.resolveHtmlOrNull(name: Name): Data<DataFragment>? {
}
context(SnarkContext)
public fun DataSet<*>.resolveHtmlOrNull(name: String): Data<DataFragment>? = resolveHtmlOrNull(name.parseAsName())
public fun DataSet<*>.resolveHtmlOrNull(name: String): Data<PageFragment>? = resolveHtmlOrNull(name.parseAsName())
context(SnarkContext)
public fun DataSet<*>.resolveHtml(name: String): Data<DataFragment> = resolveHtmlOrNull(name)
public fun DataSet<*>.resolveHtml(name: String): Data<PageFragment> = resolveHtmlOrNull(name)
?: error("Html fragment with name $name is not resolved")
/**
@ -66,7 +75,7 @@ public fun DataSet<*>.resolveHtml(name: String): Data<DataFragment> = resolveHtm
context(SnarkContext)
public fun DataSet<*>.resolveAllHtml(
predicate: (name: Name, meta: Meta) -> Boolean,
): Map<Name, Data<DataFragment>> = filterByType<DataFragment> { name, meta ->
): Map<Name, Data<PageFragment>> = filterByType<PageFragment> { name, meta ->
predicate(name, meta)
&& meta["published"].string != "false"
//TODO add language confirmation
@ -76,6 +85,6 @@ context(SnarkContext)
public fun DataSet<*>.findHtmlByContentType(
contentType: String,
baseName: Name = Name.EMPTY,
): Map<Name, Data<DataFragment>> = resolveAllHtml { name, meta ->
): Map<Name, Data<PageFragment>> = resolveAllHtml { name, meta ->
name.startsWith(baseName) && meta["content_type"].string == contentType
}

14
snark-html/src/jvmMain/kotlin/space/kscience/snark/html/WebPagePostprocessor.kt → snark-html/src/jvmMain/kotlin/space/kscience/snark/html/Postprocessor.kt
View File

@ -42,13 +42,16 @@ public class WebPageTextProcessor(private val page: PageContext) : TextProcessor
}
public class WebPagePostprocessor<out R>(
/**
* A tag consumer wrapper that wraps existing [TagConsumer] and adds postprocessing.
*
*/
public class Postprocessor<out R>(
public val page: PageContext,
private val consumer: TagConsumer<R>,
private val processor: TextProcessor = WebPageTextProcessor(page),
) : TagConsumer<R> by consumer {
private val processor = WebPageTextProcessor(page)
override fun onTagAttributeChange(tag: Tag, attribute: String, value: String?) {
if (tag is A && attribute == "href" && value != null) {
consumer.onTagAttributeChange(tag, attribute, processor.process(value))
@ -73,9 +76,10 @@ public class WebPagePostprocessor<out R>(
}
}
public inline fun FlowContent.withSnarkPage(page: PageContext, block: FlowContent.() -> Unit) {
context(PageContext)
public inline fun FlowContent.postprocess(block: FlowContent.() -> Unit) {
val fc = object : FlowContent by this {
override val consumer: TagConsumer<*> = WebPagePostprocessor(page, this@withSnarkPage.consumer)
override val consumer: TagConsumer<*> = Postprocessor(page, this@postprocess.consumer)
}
fc.block()
}

79
snark-html/src/jvmMain/kotlin/space/kscience/snark/html/SiteContext.kt
View File

@ -1,6 +1,5 @@
package space.kscience.snark.html
import kotlinx.html.HTML
import space.kscience.dataforge.data.*
import space.kscience.dataforge.io.Binary
import space.kscience.dataforge.meta.Meta
@ -35,34 +34,43 @@ public interface SiteContext : SnarkContext {
* @param route The route name of the static file relative to the site root.
* @param data The data object containing the binary data for the static file.
*/
public suspend fun static(route: Name, data: Data<Binary>)
public fun static(route: Name, data: Data<Binary>)
/**
* Create a single page at given [route]. If route is empty, create an index page at current route.
* Create a single page at given [route]. If the route is empty, create an index page the current route.
*
* @param pageMeta additional page meta. [PageContext] will use both it and [siteMeta]
*/
@SnarkBuilder
public suspend fun page(
public fun page(
route: Name,
data: DataSet<Any>,
data: DataSet<*>,
pageMeta: Meta = Meta.EMPTY,
htmlPage: HtmlPage,
content: HtmlPage,
)
/**
* Create a route block with its own data. Does not change base url
*/
@SnarkBuilder
public fun route(
route: Name,
data: DataSet<*>,
siteMeta: Meta = Meta.EMPTY,
content: HtmlSite,
)
/**
* Creates a sub-site and sets it as site base url
* @param route mount site at [rootName]
* @param dataPrefix prefix path for data used in this site
*/
@SnarkBuilder
public suspend fun site(
public fun site(
route: Name,
data: DataSet<Any>,
data: DataSet<*>,
siteMeta: Meta = Meta.EMPTY,
htmlSite: HtmlSite,
content: HtmlSite,
)
@ -73,13 +81,14 @@ public interface SiteContext : SnarkContext {
}
}
public suspend fun SiteContext.static(dataSet: DataSet<Binary>, prefix: Name = Name.EMPTY) {
public fun SiteContext.static(dataSet: DataSet<Binary>, prefix: Name = Name.EMPTY) {
dataSet.forEach { (name, data) ->
static(prefix + name, data)
}
}
public suspend fun SiteContext.static(dataSet: DataSet<*>, branch: String, prefix: String = branch) {
public fun SiteContext.static(dataSet: DataSet<*>, branch: String, prefix: String = branch) {
val branchName = branch.parseAsName()
val prefixName = prefix.parseAsName()
val binaryType = typeOf<Binary>()
@ -91,20 +100,50 @@ public suspend fun SiteContext.static(dataSet: DataSet<*>, branch: String, prefi
}
}
@SnarkBuilder
public suspend fun SiteContext.page(
route: Name,
data: DataSet<Any>,
pageMeta: Meta = Meta.EMPTY,
htmlPage: HTML.(page: PageContext, data: DataSet<Any>) -> Unit,
): Unit = page(route, data, pageMeta, HtmlPage(htmlPage))
context(SiteContext)
public val site: SiteContext
get() = this@SiteContext
public suspend fun SiteContext.renderPages(data: DataSet<Any>): Unit {
/**
* A wrapper for site context that allows convenient site building experience
*/
public class SiteContextWithData(private val site: SiteContext, public val siteData: DataSet<*>) : SiteContext by site
@SnarkBuilder
public fun SiteContextWithData.static(branch: String, prefix: String = branch): Unit = static(siteData, branch, prefix)
@SnarkBuilder
public fun SiteContextWithData.page(
route: Name = Name.EMPTY,
pageMeta: Meta = Meta.EMPTY,
content: HtmlPage,
): Unit = page(route, siteData, pageMeta, content)
@SnarkBuilder
public suspend fun SiteContextWithData.route(
route: String,
data: DataSet<*> = siteData,
siteMeta: Meta = Meta.EMPTY,
content: HtmlSite,
): Unit = route(route.parseAsName(), data, siteMeta,content)
@SnarkBuilder
public suspend fun SiteContextWithData.site(
route: String,
data: DataSet<*> = siteData,
siteMeta: Meta = Meta.EMPTY,
content: HtmlSite,
): Unit = site(route.parseAsName(), data, siteMeta,content)
/**
* Render all pages and sites found in the data
*/
public suspend fun SiteContext.renderPages(data: DataSet<*>): Unit {
// Render all sub-sites
data.filterByType<HtmlSite>().forEach { siteData: NamedData<HtmlSite> ->

74
snark-html/src/jvmMain/kotlin/space/kscience/snark/html/SnarkHtml.kt
View File

@ -4,6 +4,7 @@ package space.kscience.snark.html
import io.ktor.http.ContentType
import kotlinx.io.readByteArray
import space.kscience.dataforge.actions.Action
import space.kscience.dataforge.context.*
import space.kscience.dataforge.data.*
import space.kscience.dataforge.io.IOPlugin
@ -15,10 +16,12 @@ import space.kscience.dataforge.meta.Meta
import space.kscience.dataforge.meta.get
import space.kscience.dataforge.meta.string
import space.kscience.dataforge.misc.DFExperimental
import space.kscience.dataforge.names.*
import space.kscience.dataforge.names.Name
import space.kscience.dataforge.names.NameToken
import space.kscience.dataforge.names.asName
import space.kscience.dataforge.names.replaceLast
import space.kscience.dataforge.provider.dfId
import space.kscience.dataforge.workspace.*
import space.kscience.snark.ImageIOReader
import space.kscience.snark.Snark
import space.kscience.snark.SnarkReader
import space.kscience.snark.TextProcessor
@ -27,6 +30,13 @@ import kotlin.io.path.Path
import kotlin.io.path.extension
public fun <T : Any, R : Any> DataSet<T>.transform(action: Action<T, R>, meta: Meta = Meta.EMPTY): DataSet<R> =
action.execute(this, meta)
public fun <T : Any> TaskResultBuilder<T>.fill(dataSet: DataSet<T>) {
node(Name.EMPTY, dataSet)
}
/**
* A plugin used for rendering a [DataTree] as HTML
*/
@ -43,56 +53,52 @@ public class SnarkHtml : WorkspacePlugin() {
"markdown".asName() to MarkdownReader,
"json".asName() to SnarkReader(JsonMetaFormat, ContentType.Application.Json.toString()),
"yaml".asName() to SnarkReader(YamlMetaFormat, "text/yaml", "yaml"),
"png".asName() to SnarkReader(ImageIOReader, ContentType.Image.PNG.toString()),
"jpg".asName() to SnarkReader(ImageIOReader, ContentType.Image.JPEG.toString()),
"gif".asName() to SnarkReader(ImageIOReader, ContentType.Image.GIF.toString()),
"svg".asName() to SnarkReader(IOReader.binary, ContentType.Image.SVG.toString(), "svg"),
"raw".asName() to SnarkReader(
IOReader.binary,
"css",
"js",
"javascript",
"scss",
"woff",
"woff2",
"ttf",
"eot"
)
// "png".asName() to SnarkReader(ImageIOReader, ContentType.Image.PNG.toString()),
// "jpg".asName() to SnarkReader(ImageIOReader, ContentType.Image.JPEG.toString()),
// "gif".asName() to SnarkReader(ImageIOReader, ContentType.Image.GIF.toString()),
// "svg".asName() to SnarkReader(IOReader.binary, ContentType.Image.SVG.toString(), "svg"),
// "raw".asName() to SnarkReader(
// IOReader.binary,
// "css",
// "js",
// "javascript",
// "scss",
// "woff",
// "woff2",
// "ttf",
// "eot"
// )
)
else -> super.content(target)
}
public val preprocess: TaskReference<String> by task<String> {
pipeFrom<String, String>(dataByType<String>()) { text, _, meta ->
meta[TextProcessor.TEXT_TRANSFORMATION_KEY]?.let {
snark.textProcessor(it).process(text)
} ?: text
}
public val read: TaskReference<String> by task<String>{
}
public val parse: TaskReference<Any> by task<Any> {
from(preprocess).forEach { (dataName, data) ->
from(read).forEach { (dataName, data) ->
//remove extensions for data files
val filePath = meta[FileData.FILE_PATH_KEY]?.string ?: dataName.toString()
val fileType = URLConnection.guessContentTypeFromName(filePath) ?: Path(filePath).extension
val newName = dataName.replaceLast {
if (fileType in setOf("md", "html", "yaml", "json")) {
NameToken(it.body.substringBeforeLast("."), it.index)
} else {
it
}
}
val parser = snark.readers.values.filter { parser ->
fileType in parser.types
}.maxByOrNull {
it.priority
} ?: run {
logger.debug { "The parser is not found for file $filePath with meta $meta" }
byteArraySnarkParser
logger.debug { "The parser is not found for file $filePath with meta $meta. Passing data without parsing" }
data(dataName, data)
return@forEach
}
val newName = dataName.replaceLast {
NameToken(it.body.substringBeforeLast("."), it.index)
}
val preprocessor = meta[TextProcessor.TEXT_TRANSFORMATION_KEY]?.let{snark.preprocessor(it)}
data(newName, data.map { string: String ->
parser.readFrom(string)
val preprocessed = preprocessor?.process(string) ?: string
parser.readFrom(preprocessed)
})
}
}

18
snark-html/src/jvmMain/kotlin/space/kscience/snark/html/readers.kt
View File

@ -11,25 +11,25 @@ import space.kscience.snark.SnarkReader
import kotlin.reflect.KType
import kotlin.reflect.typeOf
public object HtmlReader : SnarkReader<DataFragment> {
public object HtmlReader : SnarkReader<PageFragment> {
override val types: Set<String> = setOf("html")
override fun readFrom(source: String): DataFragment = DataFragment { _, _ ->
override fun readFrom(source: String): PageFragment = PageFragment {
div {
unsafe { +source }
}
}
override fun readFrom(source: Source): DataFragment = readFrom(source.readString())
override val type: KType = typeOf<DataFragment>()
override fun readFrom(source: Source): PageFragment = readFrom(source.readString())
override val type: KType = typeOf<PageFragment>()
}
public object MarkdownReader : SnarkReader<DataFragment> {
override val type: KType = typeOf<DataFragment>()
public object MarkdownReader : SnarkReader<PageFragment> {
override val type: KType = typeOf<PageFragment>()
override val types: Set<String> = setOf("text/markdown", "md", "markdown")
override fun readFrom(source: String): DataFragment = DataFragment { _, _ ->
override fun readFrom(source: String): PageFragment = PageFragment {
val parsedTree = markdownParser.buildMarkdownTreeFromString(source)
val htmlString = HtmlGenerator(source, parsedTree, markdownFlavor).generateHtml()
@ -43,9 +43,9 @@ public object MarkdownReader : SnarkReader<DataFragment> {
private val markdownFlavor = CommonMarkFlavourDescriptor()
private val markdownParser = MarkdownParser(markdownFlavor)
override fun readFrom(source: Source): DataFragment = readFrom(source.readString())
override fun readFrom(source: Source): PageFragment = readFrom(source.readString())
public val snarkReader: SnarkReader<DataFragment> = SnarkReader(this, "text/markdown")
public val snarkReader: SnarkReader<PageFragment> = SnarkReader(this, "text/markdown")
}

66
snark-html/src/jvmMain/kotlin/space/kscience/snark/html/static/StaticSiteContext.kt
View File

@ -1,7 +1,7 @@
package space.kscience.snark.html.static
import kotlinx.html.html
import kotlinx.html.stream.createHTML
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.runBlocking
import kotlinx.io.asSink
import kotlinx.io.buffered
import space.kscience.dataforge.data.*
@ -14,8 +14,6 @@ import space.kscience.dataforge.names.plus
import space.kscience.dataforge.workspace.FileData
import space.kscience.snark.html.*
import java.nio.file.Path
import kotlin.contracts.InvocationKind
import kotlin.contracts.contract
import kotlin.io.path.*
import kotlin.reflect.typeOf
@ -63,7 +61,8 @@ internal class StaticSiteContext(
// }
@OptIn(ExperimentalPathApi::class)
override suspend fun static(route: Name, data: Data<Binary>) {
override fun static(route: Name, data: Data<Binary>) {
//if data is a file, copy it
data.meta[FileData.FILE_PATH_KEY]?.string?.let {
val file = Path.of(it)
val targetPath = outputPath.resolve(route.toWebPath())
@ -75,9 +74,11 @@ internal class StaticSiteContext(
if (data.type != typeOf<Binary>()) error("Can't directly serve file of type ${data.type}")
val targetPath = outputPath.resolve(route.toWebPath())
val binary = data.await()
targetPath.outputStream().asSink().buffered().use {
it.writeBinary(binary)
runBlocking(Dispatchers.IO) {
val binary = data.await()
targetPath.outputStream().asSink().buffered().use {
it.writeBinary(binary)
}
}
}
@ -99,7 +100,7 @@ internal class StaticSiteContext(
)
}
override suspend fun page(route: Name, data: DataSet<Any>, pageMeta: Meta, htmlPage: HtmlPage) {
override fun page(route: Name, data: DataSet<Any>, pageMeta: Meta, content: HtmlPage) {
val modifiedPageMeta = pageMeta.toMutableMeta().apply {
@ -115,17 +116,40 @@ internal class StaticSiteContext(
newPath.parent.createDirectories()
val pageContext = StaticPageContext(this, Laminate(modifiedPageMeta, siteMeta))
newPath.writeText(HtmlPage.createHtmlString(pageContext,htmlPage, data))
newPath.writeText(HtmlPage.createHtmlString(pageContext, data, content))
}
override suspend fun site(route: Name, data: DataSet<Any>, siteMeta: Meta, htmlSite: HtmlSite) {
with(htmlSite) {
override fun route(route: Name, data: DataSet<*>, siteMeta: Meta, content: HtmlSite) {
val siteContextWithData = SiteContextWithData(
StaticSiteContext(
siteMeta = Laminate(siteMeta, this@StaticSiteContext.siteMeta),
baseUrl = baseUrl,
route = route,
outputPath = outputPath.resolve(route.toWebPath())
),
data
)
with(content) {
with(siteContextWithData) {
renderSite()
}
}
}
override fun site(route: Name, data: DataSet<Any>, siteMeta: Meta, content: HtmlSite) {
val siteContextWithData = SiteContextWithData(
StaticSiteContext(
siteMeta = Laminate(siteMeta, this@StaticSiteContext.siteMeta),
baseUrl = if (baseUrl == "") "" else resolveRef(baseUrl, route.toWebPath()),
route = Name.EMPTY,
outputPath = outputPath.resolve(route.toWebPath())
).renderSite(data)
),
data
)
with(content) {
with(siteContextWithData) {
renderSite()
}
}
}
@ -136,15 +160,21 @@ internal class StaticSiteContext(
* Use [siteUrl] as a base for all resolved URLs. By default, use [outputPath] absolute path as a base.
*
*/
public fun SnarkHtml.staticSite(
@Suppress("UnusedReceiverParameter")
public suspend fun SnarkHtml.staticSite(
data: DataSet<*>,
outputPath: Path,
siteUrl: String = outputPath.absolutePathString().replace("\\", "/"),
siteMeta: Meta = data.meta,
block: SiteContext.() -> Unit,
content: HtmlSite,
) {
contract {
callsInPlace(block, InvocationKind.EXACTLY_ONCE)
val siteContextWithData = SiteContextWithData(
StaticSiteContext(siteMeta, siteUrl, Name.EMPTY, outputPath),
data
)
with(content){
with(siteContextWithData) {
renderSite()
}
}
StaticSiteContext(siteMeta, siteUrl, Name.EMPTY, outputPath).block()
}

84
snark-ktor/src/main/kotlin/space/kscience/snark/ktor/KtorSiteContext.kt
View File

@ -2,7 +2,6 @@ package space.kscience.snark.ktor
import io.ktor.http.*
import io.ktor.http.content.TextContent
import io.ktor.server.application.Application
import io.ktor.server.application.call
import io.ktor.server.http.content.staticFiles
import io.ktor.server.plugins.origin
@ -11,14 +10,12 @@ import io.ktor.server.response.respondBytes
import io.ktor.server.routing.Route
import io.ktor.server.routing.createRouteFromPath
import io.ktor.server.routing.get
import io.ktor.server.routing.routing
import space.kscience.dataforge.context.Context
import space.kscience.dataforge.context.ContextAware
import space.kscience.dataforge.context.error
import space.kscience.dataforge.context.logger
import space.kscience.dataforge.data.Data
import space.kscience.dataforge.data.DataSet
import space.kscience.dataforge.data.DataTree
import space.kscience.dataforge.data.await
import space.kscience.dataforge.io.Binary
import space.kscience.dataforge.io.toByteArray
@ -33,8 +30,6 @@ import space.kscience.dataforge.names.plus
import space.kscience.dataforge.workspace.FileData
import space.kscience.snark.html.*
import java.nio.file.Path
import kotlin.contracts.InvocationKind
import kotlin.contracts.contract
import kotlin.reflect.typeOf
//public fun CommonAttributeGroupFacade.css(block: CssBuilder.() -> Unit) {
@ -50,7 +45,7 @@ public class KtorSiteContext(
) : SiteContext, ContextAware {
override suspend fun static(route: Name, data: Data<Binary>) {
override fun static(route: Name, data: Data<Binary>) {
data.meta[FileData.FILE_PATH_KEY]?.string?.let {
val file = try {
Path.of(it).toFile()
@ -111,7 +106,7 @@ public class KtorSiteContext(
}
}
override suspend fun page(route: Name, data: DataSet<Any>, pageMeta: Meta, htmlPage: HtmlPage) {
override fun page(route: Name, data: DataSet<*>, pageMeta: Meta, content: HtmlPage) {
ktorRoute.get(route.toWebPath()) {
val request = call.request
//substitute host for url for backwards calls
@ -128,50 +123,75 @@ public class KtorSiteContext(
val pageContext =
KtorPageContext(this@KtorSiteContext, url.buildString(), Laminate(modifiedPageMeta, siteMeta))
//render page in suspend environment
val html = HtmlPage.createHtmlString(pageContext, htmlPage, data)
val html = HtmlPage.createHtmlString(pageContext, data, content)
call.respond(TextContent(html, ContentType.Text.Html.withCharset(Charsets.UTF_8), HttpStatusCode.OK))
}
}
override suspend fun site(route: Name, data: DataSet<Any>, siteMeta: Meta, htmlSite: HtmlSite) {
with(htmlSite) {
override fun route(route: Name, data: DataSet<*>, siteMeta: Meta, content: HtmlSite) {
val siteContext = SiteContextWithData(
KtorSiteContext(
context,
siteMeta = Laminate(siteMeta, this@KtorSiteContext.siteMeta),
baseUrl = baseUrl,
route = route,
ktorRoute = ktorRoute.createRouteFromPath(route.toWebPath())
),
data
)
with(content) {
with(siteContext) {
renderSite()
}
}
}
override fun site(route: Name, data: DataSet<*>, siteMeta: Meta, content: HtmlSite) {
val siteContext = SiteContextWithData(
KtorSiteContext(
context,
siteMeta = Laminate(siteMeta, this@KtorSiteContext.siteMeta),
baseUrl = resolveRef(baseUrl, route.toWebPath()),
route = Name.EMPTY,
ktorRoute = ktorRoute.createRouteFromPath(route.toWebPath())
).renderSite(data)
),
data
)
with(content) {
with(siteContext) {
renderSite()
}
}
}
}
private fun Route.site(
public fun Route.site(
context: Context,
data: DataTree<*>,
data: DataSet<*>,
baseUrl: String = "",
siteMeta: Meta = data.meta,
block: KtorSiteContext.() -> Unit,
content: HtmlSite,
) {
contract {
callsInPlace(block, InvocationKind.EXACTLY_ONCE)
}
block(KtorSiteContext(context, siteMeta, baseUrl, route = Name.EMPTY, this@Route))
}
public fun Application.site(
context: Context,
data: DataTree<*>,
baseUrl: String = "",
siteMeta: Meta = data.meta,
block: SiteContext.() -> Unit,
) {
contract {
callsInPlace(block, InvocationKind.EXACTLY_ONCE)
}
routing {
site(context, data, baseUrl, siteMeta, block)
val siteContext = SiteContextWithData(
KtorSiteContext(context, siteMeta, baseUrl, route = Name.EMPTY, this@Route),
data
)
with(content) {
with(siteContext) {
renderSite()
}
}
}
//
//public suspend fun Application.site(
// context: Context,
// data: DataSet<*>,
// baseUrl: String = "",
// siteMeta: Meta = data.meta,
// content: HtmlSite,
//) {
// routing {}.site(context, data, baseUrl, siteMeta, content)
//
//}