SPC/snark
5
1
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

WIP

Browse Source
This commit is contained in:
Alexander Nozik 2023-11-27 10:00:55 +03:00
parent d5edf5e989
commit c986ede110
26 changed files with 439 additions and 343 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
build.gradle.kts
View File

@ -6,7 +6,7 @@ plugins {
allprojects {
group = "space.kscience"
version = "0.1.0-dev-1"
version = "0.2.0-dev-1"
repositories {
mavenCentral()

56
snark-core/src/commonMain/kotlin/space/kscience/snark/Snark.kt Normal file
View File

@ -0,0 +1,56 @@
package space.kscience.snark
import kotlinx.io.readByteArray
import space.kscience.dataforge.context.Context
import space.kscience.dataforge.context.PluginFactory
import space.kscience.dataforge.context.PluginTag
import space.kscience.dataforge.context.gather
import space.kscience.dataforge.io.IOPlugin
import space.kscience.dataforge.io.IOReader
import space.kscience.dataforge.meta.Meta
import space.kscience.dataforge.meta.get
import space.kscience.dataforge.meta.string
import space.kscience.dataforge.names.Name
import space.kscience.dataforge.names.parseAsName
import space.kscience.dataforge.workspace.WorkspacePlugin
/**
* Represents a Snark workspace plugin.
*/
public class Snark : WorkspacePlugin() {
public val io: IOPlugin by require(IOPlugin)
override val tag: PluginTag get() = Companion.tag
public val readers: Map<Name, SnarkIOReader<Any>> by lazy {
context.gather<SnarkIOReader<Any>>(SnarkIOReader.DF_TYPE, inherit = true)
}
/**
* A lazy-initialized map of `TextProcessor` instances used for page-based text transformation.
*
* @property textProcessors The `TextProcessor` instances accessible by their names.
*/
public val textProcessors: Map<Name, TextProcessor> by lazy {
context.gather(TextProcessor.DF_TYPE, true)
}
public fun textProcessor(transformationMeta: Meta): TextProcessor {
val transformationName = transformationMeta.string
?: transformationMeta["name"].string ?: error("Transformation name not defined in $transformationMeta")
return textProcessors[transformationName.parseAsName()]
?: error("Text transformation with name $transformationName not found in $this")
}
public companion object : PluginFactory<Snark> {
override val tag: PluginTag = PluginTag("snark")
override fun build(context: Context, meta: Meta): Snark = Snark()
private val byteArrayIOReader = IOReader {
readByteArray()
}
internal val byteArraySnarkParser = SnarkIOReader(byteArrayIOReader)
}
}

36
snark-core/src/commonMain/kotlin/space/kscience/snark/SnarkIOReader.kt Normal file
View File

@ -0,0 +1,36 @@
package space.kscience.snark
import kotlinx.io.Source
import space.kscience.dataforge.io.IOReader
import space.kscience.dataforge.misc.DfId
import space.kscience.snark.SnarkIOReader.Companion.DF_TYPE
/**
* A wrapper class for IOReader that adds priority and MIME type handling.
*
* @param T The type of data to be read by the IOReader.
* @property reader The underlying IOReader instance used for reading data.
* @property types The set of supported types that can be read by the SnarkIOReader.
* @property priority The priority of the SnarkIOReader. Higher priority SnarkIOReader instances will be preferred over lower priority ones.
*/
@DfId(DF_TYPE)
public class SnarkIOReader<out T>(
private val reader: IOReader<T>,
public val types: Set<String>,
public val priority: Int = DEFAULT_PRIORITY,
) : IOReader<T> by reader {
public fun readFrom(source: String): T{
}
public companion object {
public const val DF_TYPE: String = "snark.reader"
public const val DEFAULT_PRIORITY: Int = 10
}
}
public fun <T : Any> SnarkIOReader(
reader: IOReader<T>,
vararg types: String,
): SnarkIOReader<T> = SnarkIOReader(reader, types.toSet())

56
snark-core/src/commonMain/kotlin/space/kscience/snark/SnarkParser.kt
View File

@ -1,56 +0,0 @@
package space.kscience.snark
import kotlinx.io.Source
import kotlinx.io.readByteArray
import space.kscience.dataforge.context.Context
import space.kscience.dataforge.io.IOReader
import space.kscience.dataforge.io.asBinary
import space.kscience.dataforge.io.readWith
import space.kscience.dataforge.meta.Meta
import space.kscience.dataforge.misc.DfId
import kotlin.reflect.KType
import kotlin.reflect.typeOf
/**
* A parser of binary content including priority flag and file extensions
*/
@DfId(SnarkParser.TYPE)
public interface SnarkParser<out R> {
public val type: KType
public val fileExtensions: Set<String>
public val priority: Int get() = DEFAULT_PRIORITY
//TODO use Binary instead of ByteArray
public fun parse(context: Context, meta: Meta, bytes: ByteArray): R
public fun asReader(context: Context, meta: Meta): IOReader<R> = object : IOReader<R> {
override val type: KType get() = this@SnarkParser.type
override fun readFrom(source: Source): R = parse(context, meta, source.readByteArray())
}
public companion object {
public const val TYPE: String = "snark.parser"
public const val DEFAULT_PRIORITY: Int = 10
}
}
@PublishedApi
internal class SnarkParserWrapper<R : Any>(
val reader: IOReader<R>,
override val type: KType,
override val fileExtensions: Set<String>,
) : SnarkParser<R> {
override fun parse(context: Context, meta: Meta, bytes: ByteArray): R = bytes.asBinary().readWith(reader)
}
/**
* Create a generic parser from reader
*/
@Suppress("FunctionName")
public inline fun <reified R : Any> SnarkParser(
reader: IOReader<R>,
vararg fileExtensions: String,
): SnarkParser<R> = SnarkParserWrapper(reader, typeOf<R>(), fileExtensions.toSet())

20
snark-core/src/commonMain/kotlin/space/kscience/snark/TextProcessor.kt Normal file
View File

@ -0,0 +1,20 @@
package space.kscience.snark
import space.kscience.dataforge.misc.DfId
import space.kscience.dataforge.names.NameToken
/**
* An object that conducts page-based text transformation. Like using link replacement or templating.
*/
@DfId(TextProcessor.DF_TYPE)
public fun interface TextProcessor {
public fun process(text: String): String
public companion object {
public const val DF_TYPE: String = "snark.textTransformation"
public val TEXT_TRANSFORMATION_KEY: NameToken = NameToken("transformation")
}
}

21
snark-core/src/jvmMain/kotlin/space/kscience/snark/ImageIOReader.kt Normal file
View File

@ -0,0 +1,21 @@
package space.kscience.snark
import kotlinx.io.Source
import kotlinx.io.asInputStream
import space.kscience.dataforge.io.IOReader
import java.awt.image.BufferedImage
import javax.imageio.ImageIO
import kotlin.reflect.KType
import kotlin.reflect.typeOf
/**
* The ImageIOReader class is an implementation of the IOReader interface specifically for reading images using the ImageIO library.
* It reads the image data from a given source and returns a BufferedImage object.
*
* @property type The KType of the data to be read by the ImageIOReader.
*/
public object ImageIOReader : IOReader<BufferedImage> {
override val type: KType get() = typeOf<BufferedImage>()
override fun readFrom(source: Source): BufferedImage = ImageIO.read(source.asInputStream())
}

66
snark-core/src/jvmMain/kotlin/space/kscience/snark/snarkWorkspace.kt Normal file
View File

@ -0,0 +1,66 @@
@file:OptIn(DFExperimental::class)
package space.kscience.snark
import space.kscience.dataforge.data.DataSet
import space.kscience.dataforge.data.DataTree
import space.kscience.dataforge.data.node
import space.kscience.dataforge.io.Binary
import space.kscience.dataforge.io.IOPlugin
import space.kscience.dataforge.meta.*
import space.kscience.dataforge.misc.DFExperimental
import space.kscience.dataforge.names.Name
import space.kscience.dataforge.workspace.Workspace
import space.kscience.dataforge.workspace.WorkspaceBuilder
import space.kscience.dataforge.workspace.readRawDirectory
import kotlin.io.path.Path
import kotlin.io.path.toPath
/**
* Reads the specified resources and returns a [DataTree] containing the data.
*
* @param resources The names of the resources to read.
* @param classLoader The class loader to use for loading the resources. By default, it uses the current thread's context class loader.
* @return A DataTree containing the data read from the resources.
*/
private fun IOPlugin.readResources(
vararg resources: String,
classLoader: ClassLoader = Thread.currentThread().contextClassLoader,
): DataTree<Binary> {
// require(resource.isNotBlank()) {"Can't mount root resource tree as data root"}
return DataTree {
resources.forEach { resource ->
val path = classLoader.getResource(resource)?.toURI()?.toPath() ?: error(
"Resource with name $resource is not resolved"
)
node(resource, readRawDirectory(path))
}
}
}
public fun Snark.workspace(
meta: Meta,
customData: DataSet<*> = DataSet.EMPTY,
workspaceBuilder: WorkspaceBuilder.() -> Unit = {},
): Workspace = Workspace {
data {
node(Name.EMPTY, customData)
meta.getIndexed("directory").forEach { (index, directoryMeta) ->
val dataDirectory = directoryMeta["path"].string ?: error("Directory path not defined")
val nodeName = directoryMeta["name"].string ?: directoryMeta.string ?: index ?: ""
val data = io.readRawDirectory(Path(dataDirectory))
node(nodeName, data)
}
meta.getIndexed("resource").forEach { (index, resourceMeta) ->
val resource = resourceMeta["path"]?.stringList ?: listOf("/")
val nodeName = resourceMeta["name"].string ?: resourceMeta.string ?: index ?: ""
val data: DataTree<Binary> = io.readResources(*resource.toTypedArray())
node(nodeName, data)
}
}
workspaceBuilder()
}

17
snark-html/build.gradle.kts
View File

@ -1,5 +1,5 @@
plugins {
id("space.kscience.gradle.jvm")
id("space.kscience.gradle.mpp")
`maven-publish`
}
@ -7,21 +7,22 @@ val dataforgeVersion: String by rootProject.extra
val ktorVersion = space.kscience.gradle.KScienceVersions.ktorVersion
kscience{
jvm()
useContextReceivers()
}
dependencies {
commonMain{
api(projects.snarkCore)
api("org.jetbrains.kotlinx:kotlinx-html:0.8.0")
api(spclibs.kotlinx.html)
api("org.jetbrains.kotlin-wrappers:kotlin-css")
api("io.ktor:ktor-utils:$ktorVersion")
api("io.ktor:ktor-http:$ktorVersion")
api("space.kscience:dataforge-io-yaml:$dataforgeVersion")
api("org.jetbrains:markdown:0.4.0")
api("org.jetbrains:markdown:0.5.2")
}
}
readme {
maturity = space.kscience.gradle.Maturity.EXPERIMENTAL
feature("data") { "Data-based processing. Instead of traditional layout-based" }

0
snark-html/src/main/kotlin/space/kscience/snark/html/DataRenderer.kt → snark-html/src/jvmMain/kotlin/space/kscience/snark/html/DataRenderer.kt
View File

0
snark-html/src/main/kotlin/space/kscience/snark/html/HtmlData.kt → snark-html/src/jvmMain/kotlin/space/kscience/snark/html/HtmlData.kt
View File

0
snark-html/src/main/kotlin/space/kscience/snark/html/Language.kt → snark-html/src/jvmMain/kotlin/space/kscience/snark/html/Language.kt
View File

4
snark-html/src/main/kotlin/space/kscience/snark/html/SiteBuilder.kt → snark-html/src/jvmMain/kotlin/space/kscience/snark/html/SiteBuilder.kt
View File

@ -39,7 +39,7 @@ public interface SiteBuilder : ContextAware, SnarkContext {
/**
* Snark plugin and context used for layout resolution, preprocessors, etc
*/
public val snark: SnarkHtmlPlugin
public val snark: SnarkHtml
override val context: Context get() = snark.context
@ -49,7 +49,7 @@ public interface SiteBuilder : ContextAware, SnarkContext {
public val siteMeta: Meta
/**
* Serve a static data as a file from [data] with given [dataName] at given [routeName].
* Serve static data as a file from [data] with given [dataName] at given [routeName].
*/
public fun static(dataName: Name, routeName: Name = dataName)

0
snark-html/src/main/kotlin/space/kscience/snark/html/SiteLayout.kt → snark-html/src/jvmMain/kotlin/space/kscience/snark/html/SiteLayout.kt
View File

141
snark-html/src/jvmMain/kotlin/space/kscience/snark/html/SnarkHtml.kt Normal file
View File

@ -0,0 +1,141 @@
@file:OptIn(DFExperimental::class)
package space.kscience.snark.html
import io.ktor.http.ContentType
import kotlinx.io.readByteArray
import space.kscience.dataforge.context.*
import space.kscience.dataforge.data.*
import space.kscience.dataforge.io.IOPlugin
import space.kscience.dataforge.io.IOReader
import space.kscience.dataforge.io.JsonMetaFormat
import space.kscience.dataforge.io.yaml.YamlMetaFormat
import space.kscience.dataforge.io.yaml.YamlPlugin
import space.kscience.dataforge.meta.*
import space.kscience.dataforge.misc.DFExperimental
import space.kscience.dataforge.names.*
import space.kscience.dataforge.provider.dfId
import space.kscience.dataforge.workspace.*
import space.kscience.snark.ImageIOReader
import space.kscience.snark.Snark
import space.kscience.snark.SnarkIOReader
import space.kscience.snark.TextProcessor
import java.net.URLConnection
import kotlin.io.path.Path
import kotlin.io.path.extension
public fun <T : Any> SnarkIOReader(
reader: IOReader<T>,
vararg types: ContentType,
priority: Int = SnarkIOReader.DEFAULT_PRIORITY,
): SnarkIOReader<T> = SnarkIOReader(reader, types.map { it.toString() }.toSet(), priority)
/**
* A plugin used for rendering a [DataTree] as HTML
*/
public class SnarkHtml : WorkspacePlugin() {
private val snark by require(Snark)
private val yaml by require(YamlPlugin)
public val io: IOPlugin get() = snark.io
override val tag: PluginTag get() = Companion.tag
/**
* Lazy-initialized variable that holds a map of site layouts.
*
* @property siteLayouts The map of site layouts, where the key is the layout name and the value is the corresponding SiteLayout object.
*/
private val siteLayouts: Map<Name, SiteLayout> by lazy {
context.gather(SiteLayout.TYPE, true)
}
internal fun siteLayout(layoutMeta: Meta): SiteLayout {
val layoutName = layoutMeta.string
?: layoutMeta["name"].string ?: error("Layout name not defined in $layoutMeta")
return siteLayouts[layoutName.parseAsName()] ?: error("Layout with name $layoutName not found in $this")
}
override fun content(target: String): Map<Name, Any> = when (target) {
SnarkIOReader::class.dfId -> mapOf(
"html".asName() to HtmlIOFormat.snarkReader,
"markdown".asName() to MarkdownIOFormat.snarkReader,
"json".asName() to SnarkIOReader(JsonMetaFormat, ContentType.Application.Json),
"yaml".asName() to SnarkIOReader(YamlMetaFormat, "text/yaml"),
"png".asName() to SnarkIOReader(ImageIOReader, ContentType.Image.PNG),
"jpg".asName() to SnarkIOReader(ImageIOReader, ContentType.Image.JPEG),
"gif".asName() to SnarkIOReader(ImageIOReader, ContentType.Image.GIF),
"svg".asName() to SnarkIOReader(IOReader.binary, ContentType.Image.SVG, ContentType.parse("svg")),
"raw".asName() to SnarkIOReader(
IOReader.binary,
"css",
"js",
"javascript",
"scss",
"woff",
"woff2",
"ttf",
"eot"
)
)
else -> super.content(target)
}
public val preprocess: TaskReference<String> by task<String> {
pipeFrom<String,String>(dataByType<String>()) { text, _, meta ->
meta[TextProcessor.TEXT_TRANSFORMATION_KEY]?.let {
snark.textProcessor(it).process(text)
} ?: text
}
}
public val parse: TaskReference<Any> by task<Any> {
from(preprocess).forEach { (dataName, data) ->
//remove extensions for data files
val filePath = meta[FileData.FILE_PATH_KEY]?.string ?: dataName.toString()
val fileType = URLConnection.guessContentTypeFromName(filePath) ?: Path(filePath).extension
val newName = dataName.replaceLast {
if (fileType in setOf("md", "html", "yaml", "json")) {
NameToken(it.body.substringBeforeLast("."), it.index)
} else {
it
}
}
val parser = snark.readers.values.filter { parser ->
fileType in parser.types
}.maxByOrNull {
it.priority
} ?: run {
logger.debug { "The parser is not found for file $filePath with meta $meta" }
byteArraySnarkParser
}
data(newName, data.map { string: String ->
parser.readFrom(string)
})
}
}
// public val textTransformationAction: Action<String, String> = Action.map<String, String> {
// val transformations = actionMeta.getIndexed("transformation").entries.sortedBy {
// it.key?.toIntOrNull() ?: 0
// }.map { it.value }
// }
public companion object : PluginFactory<SnarkHtml> {
override val tag: PluginTag = PluginTag("snark.html")
override fun build(context: Context, meta: Meta): SnarkHtml = SnarkHtml()
private val byteArrayIOReader = IOReader {
readByteArray()
}
internal val byteArraySnarkParser = SnarkIOReader(byteArrayIOReader)
}
}

6
snark-html/src/main/kotlin/space/kscience/snark/html/StaticSiteBuilder.kt → snark-html/src/jvmMain/kotlin/space/kscience/snark/html/StaticSiteBuilder.kt
View File

@ -28,7 +28,7 @@ import kotlin.reflect.typeOf
* An implementation of [SiteBuilder] to render site as a static directory [outputPath]
*/
internal class StaticSiteBuilder(
override val snark: SnarkHtmlPlugin,
override val snark: SnarkHtml,
override val data: DataTree<*>,
override val siteMeta: Meta,
private val baseUrl: String,
@ -121,7 +121,7 @@ internal class StaticSiteBuilder(
inner class StaticWebPage(override val pageMeta: Meta) : WebPage {
override val data: DataTree<*> get() = this@StaticSiteBuilder.data
override val snark: SnarkHtmlPlugin get() = this@StaticSiteBuilder.snark
override val snark: SnarkHtml get() = this@StaticSiteBuilder.snark
override fun resolveRef(ref: String): String =
@ -186,7 +186,7 @@ internal class StaticSiteBuilder(
* Use [siteUrl] as a base for all resolved URLs. By default, use [outputPath] absolute path as a base.
*
*/
public fun SnarkHtmlPlugin.static(
public fun SnarkHtml.static(
data: DataTree<*>,
outputPath: Path,
siteUrl: String = outputPath.absolutePathString().replace("\\", "/"),

2
snark-html/src/main/kotlin/space/kscience/snark/html/WebPage.kt → snark-html/src/jvmMain/kotlin/space/kscience/snark/html/WebPage.kt
View File

@ -25,7 +25,7 @@ public fun Name.toWebPath(): String = tokens.joinToString(separator = "/") {
@SnarkBuilder
public interface WebPage : ContextAware, SnarkContext {
public val snark: SnarkHtmlPlugin
public val snark: SnarkHtml
override val context: Context get() = snark.context

31
snark-html/src/main/kotlin/space/kscience/snark/html/TextProcessor.kt → snark-html/src/jvmMain/kotlin/space/kscience/snark/html/WebPagePreprocessor.kt
View File

@ -1,23 +1,8 @@
package space.kscience.snark.html
import space.kscience.dataforge.meta.string
import space.kscience.dataforge.misc.DfId
import space.kscience.dataforge.names.NameToken
import space.kscience.dataforge.names.parseAsName
/**
* An object that conducts page-based text transformation. Like using link replacement or templating.
*/
@DfId(TextProcessor.TYPE)
public fun interface TextProcessor {
context(WebPage)
public fun process(text: String): String
public companion object {
public const val TYPE: String = "snark.textTransformation"
public val TEXT_TRANSFORMATION_KEY: NameToken = NameToken("transformation")
}
}
import space.kscience.snark.TextProcessor
/**
* A basic [TextProcessor] that replaces `${...}` expressions in text. The following expressions are recognised:
@ -27,34 +12,32 @@ public fun interface TextProcessor {
* * `pageMeta.get("...") -> [WebPage.pageMeta] get string method
* Otherwise return unchanged string
*/
public object BasicTextProcessor : TextProcessor {
public class WebPagePreprocessor(public val page: WebPage) : TextProcessor {
private val regex = """\$\{([\w.]*)(?>\("(.*)"\))?}""".toRegex()
context(WebPage)
override fun process(text: String): String = text.replace(regex) { match ->
when (match.groups[1]!!.value) {
"homeRef" -> homeRef
"homeRef" -> page.homeRef
"resolveRef" -> {
val refString = match.groups[2]?.value ?: error("resolveRef requires a string (quoted) argument")
resolveRef(refString)
page.resolveRef(refString)
}
"resolvePageRef" -> {
val refString = match.groups[2]?.value
?: error("resolvePageRef requires a string (quoted) argument")
localisedPageRef(refString.parseAsName())
page.localisedPageRef(refString.parseAsName())
}
"pageMeta.get" -> {
val nameString = match.groups[2]?.value
?: error("resolvePageRef requires a string (quoted) argument")
pageMeta[nameString.parseAsName()].string ?: "@null"
page.pageMeta[nameString.parseAsName()].string ?: "@null"
}
else -> match.value
}
}
}

0
snark-html/src/main/kotlin/space/kscience/snark/html/htmlEnvironment.kt → snark-html/src/jvmMain/kotlin/space/kscience/snark/html/htmlEnvironment.kt
View File

50
snark-html/src/jvmMain/kotlin/space/kscience/snark/html/htmlIoFormats.kt Normal file
View File

@ -0,0 +1,50 @@
package space.kscience.snark.html
import io.ktor.http.ContentType
import kotlinx.html.div
import kotlinx.html.unsafe
import kotlinx.io.Source
import kotlinx.io.readString
import org.intellij.markdown.flavours.commonmark.CommonMarkFlavourDescriptor
import org.intellij.markdown.html.HtmlGenerator
import org.intellij.markdown.parser.MarkdownParser
import space.kscience.dataforge.io.IOReader
import space.kscience.snark.SnarkIOReader
import kotlin.reflect.KType
import kotlin.reflect.typeOf
public object HtmlIOFormat : IOReader<HtmlFragment> {
override val type: KType = typeOf<HtmlFragment>()
override fun readFrom(source: Source): HtmlFragment = HtmlFragment { page ->
div {
unsafe { +source.readString() }
}
}
public val snarkReader: SnarkIOReader<HtmlFragment> = SnarkIOReader(this, ContentType.Text.Html)
}
public object MarkdownIOFormat : IOReader<HtmlFragment> {
override val type: KType = typeOf<HtmlFragment>()
private val markdownFlavor = CommonMarkFlavourDescriptor()
private val markdownParser = MarkdownParser(markdownFlavor)
override fun readFrom(source: Source): HtmlFragment = HtmlFragment { page ->
val transformedText = source.readString()
val parsedTree = markdownParser.buildMarkdownTreeFromString(transformedText)
val htmlString = HtmlGenerator(transformedText, parsedTree, markdownFlavor).generateHtml()
div {
unsafe {
+htmlString
}
}
}
public val snarkReader: SnarkIOReader<HtmlFragment> = SnarkIOReader(this, ContentType.parse("text/markdown"))
}

0
snark-html/src/main/resources/application.conf → snark-html/src/jvmMain/resources/application.conf
View File

0
snark-html/src/main/resources/logback.xml → snark-html/src/jvmMain/resources/logback.xml
View File

16
snark-html/src/main/kotlin/space/kscience/snark/html/ImageIOReader.kt
View File

@ -1,16 +0,0 @@
package space.kscience.snark.html
import io.ktor.util.asStream
import kotlinx.io.Source
import kotlinx.io.asInputStream
import space.kscience.dataforge.io.IOReader
import java.awt.image.BufferedImage
import javax.imageio.ImageIO
import kotlin.reflect.KType
import kotlin.reflect.typeOf
internal object ImageIOReader : IOReader<BufferedImage> {
override val type: KType get() = typeOf<BufferedImage>()
override fun readFrom(source: Source): BufferedImage = ImageIO.read(source.asInputStream())
}

125
snark-html/src/main/kotlin/space/kscience/snark/html/SnarkHtmlPlugin.kt
View File

@ -1,125 +0,0 @@
package space.kscience.snark.html
import io.ktor.utils.io.core.readBytes
import kotlinx.io.readByteArray
import space.kscience.dataforge.context.*
import space.kscience.dataforge.data.DataTree
import space.kscience.dataforge.data.node
import space.kscience.dataforge.io.IOPlugin
import space.kscience.dataforge.io.IOReader
import space.kscience.dataforge.io.JsonMetaFormat
import space.kscience.dataforge.io.yaml.YamlMetaFormat
import space.kscience.dataforge.io.yaml.YamlPlugin
import space.kscience.dataforge.meta.Meta
import space.kscience.dataforge.meta.get
import space.kscience.dataforge.meta.string
import space.kscience.dataforge.misc.DFExperimental
import space.kscience.dataforge.names.Name
import space.kscience.dataforge.names.asName
import space.kscience.dataforge.names.parseAsName
import space.kscience.dataforge.workspace.FileData
import space.kscience.dataforge.workspace.readDataDirectory
import space.kscience.snark.SnarkParser
import java.nio.file.Path
import kotlin.io.path.extension
import kotlin.io.path.toPath
/**
* A plugin used for rendering a [DataTree] as HTML
*/
public class SnarkHtmlPlugin : AbstractPlugin() {
private val yaml by require(YamlPlugin)
public val io: IOPlugin get() = yaml.io
override val tag: PluginTag get() = Companion.tag
internal val parsers: Map<Name, SnarkParser<Any>> by lazy {
context.gather(SnarkParser.TYPE, true)
}
private val siteLayouts: Map<Name, SiteLayout> by lazy {
context.gather(SiteLayout.TYPE, true)
}
private val textProcessors: Map<Name, TextProcessor> by lazy {
context.gather(TextProcessor.TYPE, true)
}
internal fun siteLayout(layoutMeta: Meta): SiteLayout {
val layoutName = layoutMeta.string
?: layoutMeta["name"].string ?: error("Layout name not defined in $layoutMeta")
return siteLayouts[layoutName.parseAsName()] ?: error("Layout with name $layoutName not found in $this")
}
internal fun textProcessor(transformationMeta: Meta): TextProcessor {
val transformationName = transformationMeta.string
?: transformationMeta["name"].string ?: error("Transformation name not defined in $transformationMeta")
return textProcessors[transformationName.parseAsName()]
?: error("Text transformation with name $transformationName not found in $this")
}
override fun content(target: String): Map<Name, Any> = when (target) {
SnarkParser.TYPE -> mapOf(
"html".asName() to SnarkHtmlParser,
"markdown".asName() to SnarkMarkdownParser,
"json".asName() to SnarkParser(JsonMetaFormat, "json"),
"yaml".asName() to SnarkParser(YamlMetaFormat, "yaml", "yml"),
"png".asName() to SnarkParser(ImageIOReader, "png"),
"jpg".asName() to SnarkParser(ImageIOReader, "jpg", "jpeg"),
"gif".asName() to SnarkParser(ImageIOReader, "gif"),
"svg".asName() to SnarkParser(IOReader.binary, "svg"),
"raw".asName() to SnarkParser(IOReader.binary, "css", "js", "scss", "woff", "woff2", "ttf", "eot")
)
TextProcessor.TYPE -> mapOf(
"basic".asName() to BasicTextProcessor
)
else -> super.content(target)
}
public companion object : PluginFactory<SnarkHtmlPlugin> {
override val tag: PluginTag = PluginTag("snark")
override fun build(context: Context, meta: Meta): SnarkHtmlPlugin = SnarkHtmlPlugin()
private val byteArrayIOReader = IOReader {
readByteArray()
}
internal val byteArraySnarkParser = SnarkParser(byteArrayIOReader)
}
}
@OptIn(DFExperimental::class)
public fun SnarkHtmlPlugin.readDirectory(path: Path): DataTree<Any> = io.readDataDirectory(
path,
setOf("md", "html", "yaml", "json")
) { dataPath, meta ->
val fileExtension = meta[FileData.FILE_EXTENSION_KEY].string ?: dataPath.extension
val parser: SnarkParser<Any> = parsers.values.filter { parser ->
fileExtension in parser.fileExtensions
}.maxByOrNull {
it.priority
} ?: run {
logger.debug { "The parser is not found for file $dataPath with meta $meta" }
SnarkHtmlPlugin.byteArraySnarkParser
}
parser.asReader(context, meta)
}
public fun SnarkHtmlPlugin.readResources(
vararg resources: String,
classLoader: ClassLoader = Thread.currentThread().contextClassLoader,
): DataTree<Any> {
// require(resource.isNotBlank()) {"Can't mount root resource tree as data root"}
return DataTree {
resources.forEach { resource ->
val path = classLoader.getResource(resource)?.toURI()?.toPath() ?: error(
"Resource with name $resource is not resolved"
)
node(resource, readDirectory(path))
}
}
}

58
snark-html/src/main/kotlin/space/kscience/snark/html/SnarkTextParser.kt
View File

@ -1,58 +0,0 @@
package space.kscience.snark.html
import kotlinx.html.div
import kotlinx.html.unsafe
import org.intellij.markdown.flavours.commonmark.CommonMarkFlavourDescriptor
import org.intellij.markdown.html.HtmlGenerator
import org.intellij.markdown.parser.MarkdownParser
import space.kscience.dataforge.context.Context
import space.kscience.dataforge.meta.Meta
import space.kscience.dataforge.meta.get
import space.kscience.snark.SnarkParser
import kotlin.reflect.KType
import kotlin.reflect.typeOf
public abstract class SnarkTextParser<R> : SnarkParser<R> {
public abstract fun parseText(text: String, meta: Meta): R
override fun parse(context: Context, meta: Meta, bytes: ByteArray): R =
parseText(bytes.decodeToString(), meta)
public fun transformText(text: String, meta: Meta, page: WebPage): String =
meta[TextProcessor.TEXT_TRANSFORMATION_KEY]?.let {
with(page) { page.snark.textProcessor(it).process(text) }
} ?: text
}
internal object SnarkHtmlParser : SnarkTextParser<HtmlFragment>() {
override val fileExtensions: Set<String> = setOf("html")
override val type: KType = typeOf<HtmlFragment>()
override fun parseText(text: String, meta: Meta): HtmlFragment = HtmlFragment { page ->
div {
unsafe { +transformText(text, meta, page) }
}
}
}
internal object SnarkMarkdownParser : SnarkTextParser<HtmlFragment>() {
override val fileExtensions: Set<String> = setOf("markdown", "mdown", "mkdn", "mkd", "md")
override val type: KType = typeOf<HtmlFragment>()
private val markdownFlavor = CommonMarkFlavourDescriptor()
private val markdownParser = MarkdownParser(markdownFlavor)
override fun parseText(text: String, meta: Meta): HtmlFragment = HtmlFragment { page ->
val transformedText = SnarkHtmlParser.transformText(text, meta, page)
val parsedTree = markdownParser.buildMarkdownTreeFromString(transformedText)
val htmlString = HtmlGenerator(transformedText, parsedTree, markdownFlavor).generateHtml()
div {
unsafe {
+htmlString
}
}
}
}

16
snark-ktor/src/main/kotlin/space/kscience/snark/ktor/KtorSiteBuilder.kt
View File

@ -17,6 +17,7 @@ import io.ktor.server.routing.routing
import kotlinx.css.CssBuilder
import kotlinx.html.CommonAttributeGroupFacade
import kotlinx.html.HTML
import kotlinx.html.head
import kotlinx.html.style
import space.kscience.dataforge.context.error
import space.kscience.dataforge.context.logger
@ -33,7 +34,7 @@ import space.kscience.dataforge.names.endsWith
import space.kscience.dataforge.names.plus
import space.kscience.dataforge.workspace.FileData
import space.kscience.snark.html.SiteBuilder
import space.kscience.snark.html.SnarkHtmlPlugin
import space.kscience.snark.html.SnarkHtml
import space.kscience.snark.html.WebPage
import space.kscience.snark.html.toWebPath
import java.nio.file.Path
@ -46,7 +47,7 @@ public fun CommonAttributeGroupFacade.css(block: CssBuilder.() -> Unit) {
}
public class KtorSiteBuilder(
override val snark: SnarkHtmlPlugin,
override val snark: SnarkHtml,
override val data: DataTree<*>,
override val siteMeta: Meta,
private val baseUrl: String,
@ -134,7 +135,7 @@ public class KtorSiteBuilder(
val pageBaseUrl: String,
override val pageMeta: Meta,
) : WebPage {
override val snark: SnarkHtmlPlugin get() = this@KtorSiteBuilder.snark
override val snark: SnarkHtml get() = this@KtorSiteBuilder.snark
override val data: DataTree<*> get() = this@KtorSiteBuilder.data
override fun resolveRef(ref: String): String = this@KtorSiteBuilder.resolveRef(pageBaseUrl, ref)
@ -154,7 +155,6 @@ public class KtorSiteBuilder(
override fun page(route: Name, pageMeta: Meta, content: context(HTML, WebPage) () -> Unit) {
ktorRoute.get(route.toWebPath()) {
call.respondHtml {
val request = call.request
//substitute host for url for backwards calls
val url = URLBuilder(baseUrl).apply {
@ -167,8 +167,10 @@ public class KtorSiteBuilder(
"name" put route.toString()
"url" put url.buildString()
}
val pageBuilder = KtorWebPage(url.buildString(), Laminate(modifiedPageMeta, siteMeta))
call.respondHtml {
head{}
content(this, pageBuilder)
}
}
@ -211,7 +213,7 @@ public class KtorSiteBuilder(
}
private fun Route.site(
snarkHtmlPlugin: SnarkHtmlPlugin,
snarkHtmlPlugin: SnarkHtml,
data: DataTree<*>,
baseUrl: String = "",
siteMeta: Meta = data.meta,
@ -224,7 +226,7 @@ private fun Route.site(
}
public fun Application.site(
snark: SnarkHtmlPlugin,
snark: SnarkHtml,
data: DataTree<*>,
baseUrl: String = "",
siteMeta: Meta = data.meta,

25
snark-ktor/src/main/kotlin/space/kscience/snark/ktor/extractData.kt
View File

@ -9,31 +9,6 @@ import java.time.LocalDateTime
import kotlin.io.path.*
//public fun KtorSiteBuilder.extractResources(uri: URI, targetPath: Path): Path {
// if (Files.isDirectory(targetPath)) {
// logger.info { "Using existing data directory at $targetPath." }
// } else {
// logger.info { "Copying data from $uri into $targetPath." }
// targetPath.createDirectories()
// //Copy everything into a temporary directory
// FileSystems.newFileSystem(uri, emptyMap<String, Any>()).use { fs ->
// val rootPath: Path = fs.provider().getPath(uri)
// Files.walk(rootPath).forEach { source: Path ->
// if (source.isRegularFile()) {
// val relative = source.relativeTo(rootPath).toString()
// val destination: Path = targetPath.resolve(relative)
// destination.parent.createDirectories()
// Files.copy(source, destination)
// }
// }
// }
// }
// return targetPath
//}
//
//public fun KtorSiteBuilder.extractResources(resource: String, targetPath: Path): Path =
// extractResources(javaClass.getResource(resource)!!.toURI(), targetPath)
private const val DEPLOY_DATE_FILE = "deployDate"
private const val BUILD_DATE_FILE = "/buildDate"