From c986ede1101a6b37287aaacf47ddaebf1f0aecd3 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Mon, 27 Nov 2023 10:00:55 +0300 Subject: [PATCH] WIP --- build.gradle.kts | 2 +- .../kotlin/space/kscience/snark/Snark.kt | 56 +++++++ .../space/kscience/snark/SnarkIOReader.kt | 36 +++++ .../space/kscience/snark/SnarkParser.kt | 56 ------- .../space/kscience/snark/TextProcessor.kt | 20 +++ .../space/kscience/snark/ImageIOReader.kt | 21 +++ .../space/kscience/snark/snarkWorkspace.kt | 66 ++++++++ snark-html/build.gradle.kts | 25 ++-- .../space/kscience/snark/html/DataRenderer.kt | 0 .../space/kscience/snark/html/HtmlData.kt | 0 .../space/kscience/snark/html/Language.kt | 0 .../space/kscience/snark/html/SiteBuilder.kt | 4 +- .../space/kscience/snark/html/SiteLayout.kt | 0 .../space/kscience/snark/html/SnarkHtml.kt | 141 ++++++++++++++++++ .../kscience/snark/html/StaticSiteBuilder.kt | 6 +- .../space/kscience/snark/html/WebPage.kt | 2 +- .../snark/html/WebPagePreprocessor.kt} | 33 +--- .../kscience/snark/html/htmlEnvironment.kt | 0 .../kscience/snark/html/htmlIoFormats.kt | 50 +++++++ .../resources/application.conf | 0 .../{main => jvmMain}/resources/logback.xml | 0 .../kscience/snark/html/ImageIOReader.kt | 16 -- .../kscience/snark/html/SnarkHtmlPlugin.kt | 125 ---------------- .../kscience/snark/html/SnarkTextParser.kt | 58 ------- .../kscience/snark/ktor/KtorSiteBuilder.kt | 40 ++--- .../space/kscience/snark/ktor/extractData.kt | 25 ---- 26 files changed, 439 insertions(+), 343 deletions(-) create mode 100644 snark-core/src/commonMain/kotlin/space/kscience/snark/Snark.kt create mode 100644 snark-core/src/commonMain/kotlin/space/kscience/snark/SnarkIOReader.kt delete mode 100644 snark-core/src/commonMain/kotlin/space/kscience/snark/SnarkParser.kt create mode 100644 snark-core/src/commonMain/kotlin/space/kscience/snark/TextProcessor.kt create mode 100644 snark-core/src/jvmMain/kotlin/space/kscience/snark/ImageIOReader.kt create mode 100644 snark-core/src/jvmMain/kotlin/space/kscience/snark/snarkWorkspace.kt rename snark-html/src/{main => jvmMain}/kotlin/space/kscience/snark/html/DataRenderer.kt (100%) rename snark-html/src/{main => jvmMain}/kotlin/space/kscience/snark/html/HtmlData.kt (100%) rename snark-html/src/{main => jvmMain}/kotlin/space/kscience/snark/html/Language.kt (100%) rename snark-html/src/{main => jvmMain}/kotlin/space/kscience/snark/html/SiteBuilder.kt (98%) rename snark-html/src/{main => jvmMain}/kotlin/space/kscience/snark/html/SiteLayout.kt (100%) create mode 100644 snark-html/src/jvmMain/kotlin/space/kscience/snark/html/SnarkHtml.kt rename snark-html/src/{main => jvmMain}/kotlin/space/kscience/snark/html/StaticSiteBuilder.kt (97%) rename snark-html/src/{main => jvmMain}/kotlin/space/kscience/snark/html/WebPage.kt (98%) rename snark-html/src/{main/kotlin/space/kscience/snark/html/TextProcessor.kt => jvmMain/kotlin/space/kscience/snark/html/WebPagePreprocessor.kt} (61%) rename snark-html/src/{main => jvmMain}/kotlin/space/kscience/snark/html/htmlEnvironment.kt (100%) create mode 100644 snark-html/src/jvmMain/kotlin/space/kscience/snark/html/htmlIoFormats.kt rename snark-html/src/{main => jvmMain}/resources/application.conf (100%) rename snark-html/src/{main => jvmMain}/resources/logback.xml (100%) delete mode 100644 snark-html/src/main/kotlin/space/kscience/snark/html/ImageIOReader.kt delete mode 100644 snark-html/src/main/kotlin/space/kscience/snark/html/SnarkHtmlPlugin.kt delete mode 100644 snark-html/src/main/kotlin/space/kscience/snark/html/SnarkTextParser.kt diff --git a/build.gradle.kts b/build.gradle.kts index 38da2b8..494c37a 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -6,7 +6,7 @@ plugins { allprojects { group = "space.kscience" - version = "0.1.0-dev-1" + version = "0.2.0-dev-1" repositories { mavenCentral() diff --git a/snark-core/src/commonMain/kotlin/space/kscience/snark/Snark.kt b/snark-core/src/commonMain/kotlin/space/kscience/snark/Snark.kt new file mode 100644 index 0000000..f67e310 --- /dev/null +++ b/snark-core/src/commonMain/kotlin/space/kscience/snark/Snark.kt @@ -0,0 +1,56 @@ +package space.kscience.snark + +import kotlinx.io.readByteArray +import space.kscience.dataforge.context.Context +import space.kscience.dataforge.context.PluginFactory +import space.kscience.dataforge.context.PluginTag +import space.kscience.dataforge.context.gather +import space.kscience.dataforge.io.IOPlugin +import space.kscience.dataforge.io.IOReader +import space.kscience.dataforge.meta.Meta +import space.kscience.dataforge.meta.get +import space.kscience.dataforge.meta.string +import space.kscience.dataforge.names.Name +import space.kscience.dataforge.names.parseAsName +import space.kscience.dataforge.workspace.WorkspacePlugin + +/** + * Represents a Snark workspace plugin. + */ +public class Snark : WorkspacePlugin() { + public val io: IOPlugin by require(IOPlugin) + override val tag: PluginTag get() = Companion.tag + + public val readers: Map> by lazy { + context.gather>(SnarkIOReader.DF_TYPE, inherit = true) + } + + /** + * A lazy-initialized map of `TextProcessor` instances used for page-based text transformation. + * + * @property textProcessors The `TextProcessor` instances accessible by their names. + */ + public val textProcessors: Map by lazy { + context.gather(TextProcessor.DF_TYPE, true) + } + + public fun textProcessor(transformationMeta: Meta): TextProcessor { + val transformationName = transformationMeta.string + ?: transformationMeta["name"].string ?: error("Transformation name not defined in $transformationMeta") + return textProcessors[transformationName.parseAsName()] + ?: error("Text transformation with name $transformationName not found in $this") + } + + public companion object : PluginFactory { + override val tag: PluginTag = PluginTag("snark") + + override fun build(context: Context, meta: Meta): Snark = Snark() + + private val byteArrayIOReader = IOReader { + readByteArray() + } + + internal val byteArraySnarkParser = SnarkIOReader(byteArrayIOReader) + + } +} \ No newline at end of file diff --git a/snark-core/src/commonMain/kotlin/space/kscience/snark/SnarkIOReader.kt b/snark-core/src/commonMain/kotlin/space/kscience/snark/SnarkIOReader.kt new file mode 100644 index 0000000..7e82f62 --- /dev/null +++ b/snark-core/src/commonMain/kotlin/space/kscience/snark/SnarkIOReader.kt @@ -0,0 +1,36 @@ +package space.kscience.snark + +import kotlinx.io.Source +import space.kscience.dataforge.io.IOReader +import space.kscience.dataforge.misc.DfId +import space.kscience.snark.SnarkIOReader.Companion.DF_TYPE + +/** + * A wrapper class for IOReader that adds priority and MIME type handling. + * + * @param T The type of data to be read by the IOReader. + * @property reader The underlying IOReader instance used for reading data. + * @property types The set of supported types that can be read by the SnarkIOReader. + * @property priority The priority of the SnarkIOReader. Higher priority SnarkIOReader instances will be preferred over lower priority ones. + */ +@DfId(DF_TYPE) +public class SnarkIOReader( + private val reader: IOReader, + public val types: Set, + public val priority: Int = DEFAULT_PRIORITY, +) : IOReader by reader { + + public fun readFrom(source: String): T{ + + } + + public companion object { + public const val DF_TYPE: String = "snark.reader" + public const val DEFAULT_PRIORITY: Int = 10 + } +} + +public fun SnarkIOReader( + reader: IOReader, + vararg types: String, +): SnarkIOReader = SnarkIOReader(reader, types.toSet()) \ No newline at end of file diff --git a/snark-core/src/commonMain/kotlin/space/kscience/snark/SnarkParser.kt b/snark-core/src/commonMain/kotlin/space/kscience/snark/SnarkParser.kt deleted file mode 100644 index f8a718e..0000000 --- a/snark-core/src/commonMain/kotlin/space/kscience/snark/SnarkParser.kt +++ /dev/null @@ -1,56 +0,0 @@ -package space.kscience.snark - -import kotlinx.io.Source -import kotlinx.io.readByteArray -import space.kscience.dataforge.context.Context -import space.kscience.dataforge.io.IOReader -import space.kscience.dataforge.io.asBinary -import space.kscience.dataforge.io.readWith -import space.kscience.dataforge.meta.Meta -import space.kscience.dataforge.misc.DfId -import kotlin.reflect.KType -import kotlin.reflect.typeOf - -/** - * A parser of binary content including priority flag and file extensions - */ -@DfId(SnarkParser.TYPE) -public interface SnarkParser { - public val type: KType - - public val fileExtensions: Set - - public val priority: Int get() = DEFAULT_PRIORITY - - //TODO use Binary instead of ByteArray - public fun parse(context: Context, meta: Meta, bytes: ByteArray): R - - public fun asReader(context: Context, meta: Meta): IOReader = object : IOReader { - override val type: KType get() = this@SnarkParser.type - - override fun readFrom(source: Source): R = parse(context, meta, source.readByteArray()) - } - - public companion object { - public const val TYPE: String = "snark.parser" - public const val DEFAULT_PRIORITY: Int = 10 - } -} - -@PublishedApi -internal class SnarkParserWrapper( - val reader: IOReader, - override val type: KType, - override val fileExtensions: Set, -) : SnarkParser { - override fun parse(context: Context, meta: Meta, bytes: ByteArray): R = bytes.asBinary().readWith(reader) -} - -/** - * Create a generic parser from reader - */ -@Suppress("FunctionName") -public inline fun SnarkParser( - reader: IOReader, - vararg fileExtensions: String, -): SnarkParser = SnarkParserWrapper(reader, typeOf(), fileExtensions.toSet()) \ No newline at end of file diff --git a/snark-core/src/commonMain/kotlin/space/kscience/snark/TextProcessor.kt b/snark-core/src/commonMain/kotlin/space/kscience/snark/TextProcessor.kt new file mode 100644 index 0000000..ba77fa4 --- /dev/null +++ b/snark-core/src/commonMain/kotlin/space/kscience/snark/TextProcessor.kt @@ -0,0 +1,20 @@ +package space.kscience.snark + +import space.kscience.dataforge.misc.DfId +import space.kscience.dataforge.names.NameToken + +/** + * An object that conducts page-based text transformation. Like using link replacement or templating. + */ +@DfId(TextProcessor.DF_TYPE) +public fun interface TextProcessor { + + public fun process(text: String): String + + public companion object { + public const val DF_TYPE: String = "snark.textTransformation" + public val TEXT_TRANSFORMATION_KEY: NameToken = NameToken("transformation") + } +} + + diff --git a/snark-core/src/jvmMain/kotlin/space/kscience/snark/ImageIOReader.kt b/snark-core/src/jvmMain/kotlin/space/kscience/snark/ImageIOReader.kt new file mode 100644 index 0000000..59d4eb6 --- /dev/null +++ b/snark-core/src/jvmMain/kotlin/space/kscience/snark/ImageIOReader.kt @@ -0,0 +1,21 @@ +package space.kscience.snark + +import kotlinx.io.Source +import kotlinx.io.asInputStream +import space.kscience.dataforge.io.IOReader +import java.awt.image.BufferedImage +import javax.imageio.ImageIO +import kotlin.reflect.KType +import kotlin.reflect.typeOf + +/** + * The ImageIOReader class is an implementation of the IOReader interface specifically for reading images using the ImageIO library. + * It reads the image data from a given source and returns a BufferedImage object. + * + * @property type The KType of the data to be read by the ImageIOReader. + */ +public object ImageIOReader : IOReader { + override val type: KType get() = typeOf() + + override fun readFrom(source: Source): BufferedImage = ImageIO.read(source.asInputStream()) +} \ No newline at end of file diff --git a/snark-core/src/jvmMain/kotlin/space/kscience/snark/snarkWorkspace.kt b/snark-core/src/jvmMain/kotlin/space/kscience/snark/snarkWorkspace.kt new file mode 100644 index 0000000..72b7eb0 --- /dev/null +++ b/snark-core/src/jvmMain/kotlin/space/kscience/snark/snarkWorkspace.kt @@ -0,0 +1,66 @@ +@file:OptIn(DFExperimental::class) + +package space.kscience.snark + +import space.kscience.dataforge.data.DataSet +import space.kscience.dataforge.data.DataTree +import space.kscience.dataforge.data.node +import space.kscience.dataforge.io.Binary +import space.kscience.dataforge.io.IOPlugin +import space.kscience.dataforge.meta.* +import space.kscience.dataforge.misc.DFExperimental +import space.kscience.dataforge.names.Name +import space.kscience.dataforge.workspace.Workspace +import space.kscience.dataforge.workspace.WorkspaceBuilder +import space.kscience.dataforge.workspace.readRawDirectory +import kotlin.io.path.Path +import kotlin.io.path.toPath + + +/** + * Reads the specified resources and returns a [DataTree] containing the data. + * + * @param resources The names of the resources to read. + * @param classLoader The class loader to use for loading the resources. By default, it uses the current thread's context class loader. + * @return A DataTree containing the data read from the resources. + */ +private fun IOPlugin.readResources( + vararg resources: String, + classLoader: ClassLoader = Thread.currentThread().contextClassLoader, +): DataTree { +// require(resource.isNotBlank()) {"Can't mount root resource tree as data root"} + return DataTree { + resources.forEach { resource -> + val path = classLoader.getResource(resource)?.toURI()?.toPath() ?: error( + "Resource with name $resource is not resolved" + ) + node(resource, readRawDirectory(path)) + } + } +} + +public fun Snark.workspace( + meta: Meta, + customData: DataSet<*> = DataSet.EMPTY, + workspaceBuilder: WorkspaceBuilder.() -> Unit = {}, +): Workspace = Workspace { + + + data { + node(Name.EMPTY, customData) + meta.getIndexed("directory").forEach { (index, directoryMeta) -> + val dataDirectory = directoryMeta["path"].string ?: error("Directory path not defined") + val nodeName = directoryMeta["name"].string ?: directoryMeta.string ?: index ?: "" + val data = io.readRawDirectory(Path(dataDirectory)) + node(nodeName, data) + } + meta.getIndexed("resource").forEach { (index, resourceMeta) -> + val resource = resourceMeta["path"]?.stringList ?: listOf("/") + val nodeName = resourceMeta["name"].string ?: resourceMeta.string ?: index ?: "" + val data: DataTree = io.readResources(*resource.toTypedArray()) + node(nodeName, data) + } + } + + workspaceBuilder() +} \ No newline at end of file diff --git a/snark-html/build.gradle.kts b/snark-html/build.gradle.kts index a29eddf..9b07023 100644 --- a/snark-html/build.gradle.kts +++ b/snark-html/build.gradle.kts @@ -1,5 +1,5 @@ plugins { - id("space.kscience.gradle.jvm") + id("space.kscience.gradle.mpp") `maven-publish` } @@ -7,20 +7,21 @@ val dataforgeVersion: String by rootProject.extra val ktorVersion = space.kscience.gradle.KScienceVersions.ktorVersion kscience{ + jvm() useContextReceivers() + commonMain{ + api(projects.snarkCore) + + api(spclibs.kotlinx.html) + api("org.jetbrains.kotlin-wrappers:kotlin-css") + + api("io.ktor:ktor-http:$ktorVersion") + api("space.kscience:dataforge-io-yaml:$dataforgeVersion") + api("org.jetbrains:markdown:0.5.2") + } + } -dependencies { - api(projects.snarkCore) - - api("org.jetbrains.kotlinx:kotlinx-html:0.8.0") - api("org.jetbrains.kotlin-wrappers:kotlin-css") - - api("io.ktor:ktor-utils:$ktorVersion") - - api("space.kscience:dataforge-io-yaml:$dataforgeVersion") - api("org.jetbrains:markdown:0.4.0") -} readme { maturity = space.kscience.gradle.Maturity.EXPERIMENTAL diff --git a/snark-html/src/main/kotlin/space/kscience/snark/html/DataRenderer.kt b/snark-html/src/jvmMain/kotlin/space/kscience/snark/html/DataRenderer.kt similarity index 100% rename from snark-html/src/main/kotlin/space/kscience/snark/html/DataRenderer.kt rename to snark-html/src/jvmMain/kotlin/space/kscience/snark/html/DataRenderer.kt diff --git a/snark-html/src/main/kotlin/space/kscience/snark/html/HtmlData.kt b/snark-html/src/jvmMain/kotlin/space/kscience/snark/html/HtmlData.kt similarity index 100% rename from snark-html/src/main/kotlin/space/kscience/snark/html/HtmlData.kt rename to snark-html/src/jvmMain/kotlin/space/kscience/snark/html/HtmlData.kt diff --git a/snark-html/src/main/kotlin/space/kscience/snark/html/Language.kt b/snark-html/src/jvmMain/kotlin/space/kscience/snark/html/Language.kt similarity index 100% rename from snark-html/src/main/kotlin/space/kscience/snark/html/Language.kt rename to snark-html/src/jvmMain/kotlin/space/kscience/snark/html/Language.kt diff --git a/snark-html/src/main/kotlin/space/kscience/snark/html/SiteBuilder.kt b/snark-html/src/jvmMain/kotlin/space/kscience/snark/html/SiteBuilder.kt similarity index 98% rename from snark-html/src/main/kotlin/space/kscience/snark/html/SiteBuilder.kt rename to snark-html/src/jvmMain/kotlin/space/kscience/snark/html/SiteBuilder.kt index 89ba1bd..12b56e5 100644 --- a/snark-html/src/main/kotlin/space/kscience/snark/html/SiteBuilder.kt +++ b/snark-html/src/jvmMain/kotlin/space/kscience/snark/html/SiteBuilder.kt @@ -39,7 +39,7 @@ public interface SiteBuilder : ContextAware, SnarkContext { /** * Snark plugin and context used for layout resolution, preprocessors, etc */ - public val snark: SnarkHtmlPlugin + public val snark: SnarkHtml override val context: Context get() = snark.context @@ -49,7 +49,7 @@ public interface SiteBuilder : ContextAware, SnarkContext { public val siteMeta: Meta /** - * Serve a static data as a file from [data] with given [dataName] at given [routeName]. + * Serve static data as a file from [data] with given [dataName] at given [routeName]. */ public fun static(dataName: Name, routeName: Name = dataName) diff --git a/snark-html/src/main/kotlin/space/kscience/snark/html/SiteLayout.kt b/snark-html/src/jvmMain/kotlin/space/kscience/snark/html/SiteLayout.kt similarity index 100% rename from snark-html/src/main/kotlin/space/kscience/snark/html/SiteLayout.kt rename to snark-html/src/jvmMain/kotlin/space/kscience/snark/html/SiteLayout.kt diff --git a/snark-html/src/jvmMain/kotlin/space/kscience/snark/html/SnarkHtml.kt b/snark-html/src/jvmMain/kotlin/space/kscience/snark/html/SnarkHtml.kt new file mode 100644 index 0000000..110f65e --- /dev/null +++ b/snark-html/src/jvmMain/kotlin/space/kscience/snark/html/SnarkHtml.kt @@ -0,0 +1,141 @@ +@file:OptIn(DFExperimental::class) + +package space.kscience.snark.html + +import io.ktor.http.ContentType +import kotlinx.io.readByteArray +import space.kscience.dataforge.context.* +import space.kscience.dataforge.data.* +import space.kscience.dataforge.io.IOPlugin +import space.kscience.dataforge.io.IOReader +import space.kscience.dataforge.io.JsonMetaFormat +import space.kscience.dataforge.io.yaml.YamlMetaFormat +import space.kscience.dataforge.io.yaml.YamlPlugin +import space.kscience.dataforge.meta.* +import space.kscience.dataforge.misc.DFExperimental +import space.kscience.dataforge.names.* +import space.kscience.dataforge.provider.dfId +import space.kscience.dataforge.workspace.* +import space.kscience.snark.ImageIOReader +import space.kscience.snark.Snark +import space.kscience.snark.SnarkIOReader +import space.kscience.snark.TextProcessor +import java.net.URLConnection +import kotlin.io.path.Path +import kotlin.io.path.extension + +public fun SnarkIOReader( + reader: IOReader, + vararg types: ContentType, + priority: Int = SnarkIOReader.DEFAULT_PRIORITY, +): SnarkIOReader = SnarkIOReader(reader, types.map { it.toString() }.toSet(), priority) + + +/** + * A plugin used for rendering a [DataTree] as HTML + */ +public class SnarkHtml : WorkspacePlugin() { + private val snark by require(Snark) + private val yaml by require(YamlPlugin) + public val io: IOPlugin get() = snark.io + + override val tag: PluginTag get() = Companion.tag + + /** + * Lazy-initialized variable that holds a map of site layouts. + * + * @property siteLayouts The map of site layouts, where the key is the layout name and the value is the corresponding SiteLayout object. + */ + private val siteLayouts: Map by lazy { + context.gather(SiteLayout.TYPE, true) + } + + + internal fun siteLayout(layoutMeta: Meta): SiteLayout { + val layoutName = layoutMeta.string + ?: layoutMeta["name"].string ?: error("Layout name not defined in $layoutMeta") + return siteLayouts[layoutName.parseAsName()] ?: error("Layout with name $layoutName not found in $this") + } + + override fun content(target: String): Map = when (target) { + SnarkIOReader::class.dfId -> mapOf( + "html".asName() to HtmlIOFormat.snarkReader, + "markdown".asName() to MarkdownIOFormat.snarkReader, + "json".asName() to SnarkIOReader(JsonMetaFormat, ContentType.Application.Json), + "yaml".asName() to SnarkIOReader(YamlMetaFormat, "text/yaml"), + "png".asName() to SnarkIOReader(ImageIOReader, ContentType.Image.PNG), + "jpg".asName() to SnarkIOReader(ImageIOReader, ContentType.Image.JPEG), + "gif".asName() to SnarkIOReader(ImageIOReader, ContentType.Image.GIF), + "svg".asName() to SnarkIOReader(IOReader.binary, ContentType.Image.SVG, ContentType.parse("svg")), + "raw".asName() to SnarkIOReader( + IOReader.binary, + "css", + "js", + "javascript", + "scss", + "woff", + "woff2", + "ttf", + "eot" + ) + ) + + else -> super.content(target) + } + + + public val preprocess: TaskReference by task { + pipeFrom(dataByType()) { text, _, meta -> + meta[TextProcessor.TEXT_TRANSFORMATION_KEY]?.let { + snark.textProcessor(it).process(text) + } ?: text + } + } + + public val parse: TaskReference by task { + from(preprocess).forEach { (dataName, data) -> + //remove extensions for data files + val filePath = meta[FileData.FILE_PATH_KEY]?.string ?: dataName.toString() + val fileType = URLConnection.guessContentTypeFromName(filePath) ?: Path(filePath).extension + val newName = dataName.replaceLast { + if (fileType in setOf("md", "html", "yaml", "json")) { + NameToken(it.body.substringBeforeLast("."), it.index) + } else { + it + } + } + val parser = snark.readers.values.filter { parser -> + fileType in parser.types + }.maxByOrNull { + it.priority + } ?: run { + logger.debug { "The parser is not found for file $filePath with meta $meta" } + byteArraySnarkParser + } + data(newName, data.map { string: String -> + parser.readFrom(string) + }) + } + } + + +// public val textTransformationAction: Action = Action.map { +// val transformations = actionMeta.getIndexed("transformation").entries.sortedBy { +// it.key?.toIntOrNull() ?: 0 +// }.map { it.value } +// } + + + public companion object : PluginFactory { + override val tag: PluginTag = PluginTag("snark.html") + + override fun build(context: Context, meta: Meta): SnarkHtml = SnarkHtml() + + private val byteArrayIOReader = IOReader { + readByteArray() + } + + internal val byteArraySnarkParser = SnarkIOReader(byteArrayIOReader) + + } +} diff --git a/snark-html/src/main/kotlin/space/kscience/snark/html/StaticSiteBuilder.kt b/snark-html/src/jvmMain/kotlin/space/kscience/snark/html/StaticSiteBuilder.kt similarity index 97% rename from snark-html/src/main/kotlin/space/kscience/snark/html/StaticSiteBuilder.kt rename to snark-html/src/jvmMain/kotlin/space/kscience/snark/html/StaticSiteBuilder.kt index e6fef23..e71356c 100644 --- a/snark-html/src/main/kotlin/space/kscience/snark/html/StaticSiteBuilder.kt +++ b/snark-html/src/jvmMain/kotlin/space/kscience/snark/html/StaticSiteBuilder.kt @@ -28,7 +28,7 @@ import kotlin.reflect.typeOf * An implementation of [SiteBuilder] to render site as a static directory [outputPath] */ internal class StaticSiteBuilder( - override val snark: SnarkHtmlPlugin, + override val snark: SnarkHtml, override val data: DataTree<*>, override val siteMeta: Meta, private val baseUrl: String, @@ -121,7 +121,7 @@ internal class StaticSiteBuilder( inner class StaticWebPage(override val pageMeta: Meta) : WebPage { override val data: DataTree<*> get() = this@StaticSiteBuilder.data - override val snark: SnarkHtmlPlugin get() = this@StaticSiteBuilder.snark + override val snark: SnarkHtml get() = this@StaticSiteBuilder.snark override fun resolveRef(ref: String): String = @@ -186,7 +186,7 @@ internal class StaticSiteBuilder( * Use [siteUrl] as a base for all resolved URLs. By default, use [outputPath] absolute path as a base. * */ -public fun SnarkHtmlPlugin.static( +public fun SnarkHtml.static( data: DataTree<*>, outputPath: Path, siteUrl: String = outputPath.absolutePathString().replace("\\", "/"), diff --git a/snark-html/src/main/kotlin/space/kscience/snark/html/WebPage.kt b/snark-html/src/jvmMain/kotlin/space/kscience/snark/html/WebPage.kt similarity index 98% rename from snark-html/src/main/kotlin/space/kscience/snark/html/WebPage.kt rename to snark-html/src/jvmMain/kotlin/space/kscience/snark/html/WebPage.kt index 77b2fef..c9447e4 100644 --- a/snark-html/src/main/kotlin/space/kscience/snark/html/WebPage.kt +++ b/snark-html/src/jvmMain/kotlin/space/kscience/snark/html/WebPage.kt @@ -25,7 +25,7 @@ public fun Name.toWebPath(): String = tokens.joinToString(separator = "/") { @SnarkBuilder public interface WebPage : ContextAware, SnarkContext { - public val snark: SnarkHtmlPlugin + public val snark: SnarkHtml override val context: Context get() = snark.context diff --git a/snark-html/src/main/kotlin/space/kscience/snark/html/TextProcessor.kt b/snark-html/src/jvmMain/kotlin/space/kscience/snark/html/WebPagePreprocessor.kt similarity index 61% rename from snark-html/src/main/kotlin/space/kscience/snark/html/TextProcessor.kt rename to snark-html/src/jvmMain/kotlin/space/kscience/snark/html/WebPagePreprocessor.kt index 16e49c8..c33e549 100644 --- a/snark-html/src/main/kotlin/space/kscience/snark/html/TextProcessor.kt +++ b/snark-html/src/jvmMain/kotlin/space/kscience/snark/html/WebPagePreprocessor.kt @@ -1,23 +1,8 @@ package space.kscience.snark.html import space.kscience.dataforge.meta.string -import space.kscience.dataforge.misc.DfId -import space.kscience.dataforge.names.NameToken import space.kscience.dataforge.names.parseAsName - -/** - * An object that conducts page-based text transformation. Like using link replacement or templating. - */ -@DfId(TextProcessor.TYPE) -public fun interface TextProcessor { - context(WebPage) - public fun process(text: String): String - - public companion object { - public const val TYPE: String = "snark.textTransformation" - public val TEXT_TRANSFORMATION_KEY: NameToken = NameToken("transformation") - } -} +import space.kscience.snark.TextProcessor /** * A basic [TextProcessor] that replaces `${...}` expressions in text. The following expressions are recognised: @@ -27,34 +12,32 @@ public fun interface TextProcessor { * * `pageMeta.get("...") -> [WebPage.pageMeta] get string method * Otherwise return unchanged string */ -public object BasicTextProcessor : TextProcessor { +public class WebPagePreprocessor(public val page: WebPage) : TextProcessor { private val regex = """\$\{([\w.]*)(?>\("(.*)"\))?}""".toRegex() - context(WebPage) + override fun process(text: String): String = text.replace(regex) { match -> when (match.groups[1]!!.value) { - "homeRef" -> homeRef + "homeRef" -> page.homeRef "resolveRef" -> { val refString = match.groups[2]?.value ?: error("resolveRef requires a string (quoted) argument") - resolveRef(refString) + page.resolveRef(refString) } "resolvePageRef" -> { val refString = match.groups[2]?.value ?: error("resolvePageRef requires a string (quoted) argument") - localisedPageRef(refString.parseAsName()) + page.localisedPageRef(refString.parseAsName()) } "pageMeta.get" -> { val nameString = match.groups[2]?.value ?: error("resolvePageRef requires a string (quoted) argument") - pageMeta[nameString.parseAsName()].string ?: "@null" + page.pageMeta[nameString.parseAsName()].string ?: "@null" } else -> match.value } } -} - - +} \ No newline at end of file diff --git a/snark-html/src/main/kotlin/space/kscience/snark/html/htmlEnvironment.kt b/snark-html/src/jvmMain/kotlin/space/kscience/snark/html/htmlEnvironment.kt similarity index 100% rename from snark-html/src/main/kotlin/space/kscience/snark/html/htmlEnvironment.kt rename to snark-html/src/jvmMain/kotlin/space/kscience/snark/html/htmlEnvironment.kt diff --git a/snark-html/src/jvmMain/kotlin/space/kscience/snark/html/htmlIoFormats.kt b/snark-html/src/jvmMain/kotlin/space/kscience/snark/html/htmlIoFormats.kt new file mode 100644 index 0000000..771f47d --- /dev/null +++ b/snark-html/src/jvmMain/kotlin/space/kscience/snark/html/htmlIoFormats.kt @@ -0,0 +1,50 @@ +package space.kscience.snark.html + +import io.ktor.http.ContentType +import kotlinx.html.div +import kotlinx.html.unsafe +import kotlinx.io.Source +import kotlinx.io.readString +import org.intellij.markdown.flavours.commonmark.CommonMarkFlavourDescriptor +import org.intellij.markdown.html.HtmlGenerator +import org.intellij.markdown.parser.MarkdownParser +import space.kscience.dataforge.io.IOReader +import space.kscience.snark.SnarkIOReader +import kotlin.reflect.KType +import kotlin.reflect.typeOf + +public object HtmlIOFormat : IOReader { + override val type: KType = typeOf() + + override fun readFrom(source: Source): HtmlFragment = HtmlFragment { page -> + div { + unsafe { +source.readString() } + } + } + + public val snarkReader: SnarkIOReader = SnarkIOReader(this, ContentType.Text.Html) + +} + +public object MarkdownIOFormat : IOReader { + override val type: KType = typeOf() + + private val markdownFlavor = CommonMarkFlavourDescriptor() + private val markdownParser = MarkdownParser(markdownFlavor) + + override fun readFrom(source: Source): HtmlFragment = HtmlFragment { page -> + val transformedText = source.readString() + val parsedTree = markdownParser.buildMarkdownTreeFromString(transformedText) + val htmlString = HtmlGenerator(transformedText, parsedTree, markdownFlavor).generateHtml() + + div { + unsafe { + +htmlString + } + } + } + + public val snarkReader: SnarkIOReader = SnarkIOReader(this, ContentType.parse("text/markdown")) + +} + diff --git a/snark-html/src/main/resources/application.conf b/snark-html/src/jvmMain/resources/application.conf similarity index 100% rename from snark-html/src/main/resources/application.conf rename to snark-html/src/jvmMain/resources/application.conf diff --git a/snark-html/src/main/resources/logback.xml b/snark-html/src/jvmMain/resources/logback.xml similarity index 100% rename from snark-html/src/main/resources/logback.xml rename to snark-html/src/jvmMain/resources/logback.xml diff --git a/snark-html/src/main/kotlin/space/kscience/snark/html/ImageIOReader.kt b/snark-html/src/main/kotlin/space/kscience/snark/html/ImageIOReader.kt deleted file mode 100644 index 506881f..0000000 --- a/snark-html/src/main/kotlin/space/kscience/snark/html/ImageIOReader.kt +++ /dev/null @@ -1,16 +0,0 @@ -package space.kscience.snark.html - -import io.ktor.util.asStream -import kotlinx.io.Source -import kotlinx.io.asInputStream -import space.kscience.dataforge.io.IOReader -import java.awt.image.BufferedImage -import javax.imageio.ImageIO -import kotlin.reflect.KType -import kotlin.reflect.typeOf - -internal object ImageIOReader : IOReader { - override val type: KType get() = typeOf() - - override fun readFrom(source: Source): BufferedImage = ImageIO.read(source.asInputStream()) -} \ No newline at end of file diff --git a/snark-html/src/main/kotlin/space/kscience/snark/html/SnarkHtmlPlugin.kt b/snark-html/src/main/kotlin/space/kscience/snark/html/SnarkHtmlPlugin.kt deleted file mode 100644 index 621b01a..0000000 --- a/snark-html/src/main/kotlin/space/kscience/snark/html/SnarkHtmlPlugin.kt +++ /dev/null @@ -1,125 +0,0 @@ -package space.kscience.snark.html - -import io.ktor.utils.io.core.readBytes -import kotlinx.io.readByteArray -import space.kscience.dataforge.context.* -import space.kscience.dataforge.data.DataTree -import space.kscience.dataforge.data.node -import space.kscience.dataforge.io.IOPlugin -import space.kscience.dataforge.io.IOReader -import space.kscience.dataforge.io.JsonMetaFormat -import space.kscience.dataforge.io.yaml.YamlMetaFormat -import space.kscience.dataforge.io.yaml.YamlPlugin -import space.kscience.dataforge.meta.Meta -import space.kscience.dataforge.meta.get -import space.kscience.dataforge.meta.string -import space.kscience.dataforge.misc.DFExperimental -import space.kscience.dataforge.names.Name -import space.kscience.dataforge.names.asName -import space.kscience.dataforge.names.parseAsName -import space.kscience.dataforge.workspace.FileData -import space.kscience.dataforge.workspace.readDataDirectory -import space.kscience.snark.SnarkParser -import java.nio.file.Path -import kotlin.io.path.extension -import kotlin.io.path.toPath - -/** - * A plugin used for rendering a [DataTree] as HTML - */ -public class SnarkHtmlPlugin : AbstractPlugin() { - private val yaml by require(YamlPlugin) - public val io: IOPlugin get() = yaml.io - - override val tag: PluginTag get() = Companion.tag - - internal val parsers: Map> by lazy { - context.gather(SnarkParser.TYPE, true) - } - - private val siteLayouts: Map by lazy { - context.gather(SiteLayout.TYPE, true) - } - - private val textProcessors: Map by lazy { - context.gather(TextProcessor.TYPE, true) - } - - internal fun siteLayout(layoutMeta: Meta): SiteLayout { - val layoutName = layoutMeta.string - ?: layoutMeta["name"].string ?: error("Layout name not defined in $layoutMeta") - return siteLayouts[layoutName.parseAsName()] ?: error("Layout with name $layoutName not found in $this") - } - - internal fun textProcessor(transformationMeta: Meta): TextProcessor { - val transformationName = transformationMeta.string - ?: transformationMeta["name"].string ?: error("Transformation name not defined in $transformationMeta") - return textProcessors[transformationName.parseAsName()] - ?: error("Text transformation with name $transformationName not found in $this") - } - - override fun content(target: String): Map = when (target) { - SnarkParser.TYPE -> mapOf( - "html".asName() to SnarkHtmlParser, - "markdown".asName() to SnarkMarkdownParser, - "json".asName() to SnarkParser(JsonMetaFormat, "json"), - "yaml".asName() to SnarkParser(YamlMetaFormat, "yaml", "yml"), - "png".asName() to SnarkParser(ImageIOReader, "png"), - "jpg".asName() to SnarkParser(ImageIOReader, "jpg", "jpeg"), - "gif".asName() to SnarkParser(ImageIOReader, "gif"), - "svg".asName() to SnarkParser(IOReader.binary, "svg"), - "raw".asName() to SnarkParser(IOReader.binary, "css", "js", "scss", "woff", "woff2", "ttf", "eot") - ) - - TextProcessor.TYPE -> mapOf( - "basic".asName() to BasicTextProcessor - ) - - else -> super.content(target) - } - - public companion object : PluginFactory { - override val tag: PluginTag = PluginTag("snark") - - override fun build(context: Context, meta: Meta): SnarkHtmlPlugin = SnarkHtmlPlugin() - - private val byteArrayIOReader = IOReader { - readByteArray() - } - - internal val byteArraySnarkParser = SnarkParser(byteArrayIOReader) - } -} - -@OptIn(DFExperimental::class) -public fun SnarkHtmlPlugin.readDirectory(path: Path): DataTree = io.readDataDirectory( - path, - setOf("md", "html", "yaml", "json") -) { dataPath, meta -> - val fileExtension = meta[FileData.FILE_EXTENSION_KEY].string ?: dataPath.extension - val parser: SnarkParser = parsers.values.filter { parser -> - fileExtension in parser.fileExtensions - }.maxByOrNull { - it.priority - } ?: run { - logger.debug { "The parser is not found for file $dataPath with meta $meta" } - SnarkHtmlPlugin.byteArraySnarkParser - } - - parser.asReader(context, meta) -} - -public fun SnarkHtmlPlugin.readResources( - vararg resources: String, - classLoader: ClassLoader = Thread.currentThread().contextClassLoader, -): DataTree { -// require(resource.isNotBlank()) {"Can't mount root resource tree as data root"} - return DataTree { - resources.forEach { resource -> - val path = classLoader.getResource(resource)?.toURI()?.toPath() ?: error( - "Resource with name $resource is not resolved" - ) - node(resource, readDirectory(path)) - } - } -} \ No newline at end of file diff --git a/snark-html/src/main/kotlin/space/kscience/snark/html/SnarkTextParser.kt b/snark-html/src/main/kotlin/space/kscience/snark/html/SnarkTextParser.kt deleted file mode 100644 index 0d872d4..0000000 --- a/snark-html/src/main/kotlin/space/kscience/snark/html/SnarkTextParser.kt +++ /dev/null @@ -1,58 +0,0 @@ -package space.kscience.snark.html - -import kotlinx.html.div -import kotlinx.html.unsafe -import org.intellij.markdown.flavours.commonmark.CommonMarkFlavourDescriptor -import org.intellij.markdown.html.HtmlGenerator -import org.intellij.markdown.parser.MarkdownParser -import space.kscience.dataforge.context.Context -import space.kscience.dataforge.meta.Meta -import space.kscience.dataforge.meta.get -import space.kscience.snark.SnarkParser -import kotlin.reflect.KType -import kotlin.reflect.typeOf - -public abstract class SnarkTextParser : SnarkParser { - public abstract fun parseText(text: String, meta: Meta): R - - override fun parse(context: Context, meta: Meta, bytes: ByteArray): R = - parseText(bytes.decodeToString(), meta) - - public fun transformText(text: String, meta: Meta, page: WebPage): String = - meta[TextProcessor.TEXT_TRANSFORMATION_KEY]?.let { - with(page) { page.snark.textProcessor(it).process(text) } - } ?: text -} - - -internal object SnarkHtmlParser : SnarkTextParser() { - override val fileExtensions: Set = setOf("html") - override val type: KType = typeOf() - - override fun parseText(text: String, meta: Meta): HtmlFragment = HtmlFragment { page -> - div { - unsafe { +transformText(text, meta, page) } - } - } -} - -internal object SnarkMarkdownParser : SnarkTextParser() { - override val fileExtensions: Set = setOf("markdown", "mdown", "mkdn", "mkd", "md") - override val type: KType = typeOf() - - private val markdownFlavor = CommonMarkFlavourDescriptor() - private val markdownParser = MarkdownParser(markdownFlavor) - - override fun parseText(text: String, meta: Meta): HtmlFragment = HtmlFragment { page -> - val transformedText = SnarkHtmlParser.transformText(text, meta, page) - val parsedTree = markdownParser.buildMarkdownTreeFromString(transformedText) - val htmlString = HtmlGenerator(transformedText, parsedTree, markdownFlavor).generateHtml() - - div { - unsafe { - +htmlString - } - } - } -} - diff --git a/snark-ktor/src/main/kotlin/space/kscience/snark/ktor/KtorSiteBuilder.kt b/snark-ktor/src/main/kotlin/space/kscience/snark/ktor/KtorSiteBuilder.kt index 3e77a33..b322ff9 100644 --- a/snark-ktor/src/main/kotlin/space/kscience/snark/ktor/KtorSiteBuilder.kt +++ b/snark-ktor/src/main/kotlin/space/kscience/snark/ktor/KtorSiteBuilder.kt @@ -17,6 +17,7 @@ import io.ktor.server.routing.routing import kotlinx.css.CssBuilder import kotlinx.html.CommonAttributeGroupFacade import kotlinx.html.HTML +import kotlinx.html.head import kotlinx.html.style import space.kscience.dataforge.context.error import space.kscience.dataforge.context.logger @@ -33,7 +34,7 @@ import space.kscience.dataforge.names.endsWith import space.kscience.dataforge.names.plus import space.kscience.dataforge.workspace.FileData import space.kscience.snark.html.SiteBuilder -import space.kscience.snark.html.SnarkHtmlPlugin +import space.kscience.snark.html.SnarkHtml import space.kscience.snark.html.WebPage import space.kscience.snark.html.toWebPath import java.nio.file.Path @@ -46,7 +47,7 @@ public fun CommonAttributeGroupFacade.css(block: CssBuilder.() -> Unit) { } public class KtorSiteBuilder( - override val snark: SnarkHtmlPlugin, + override val snark: SnarkHtml, override val data: DataTree<*>, override val siteMeta: Meta, private val baseUrl: String, @@ -134,7 +135,7 @@ public class KtorSiteBuilder( val pageBaseUrl: String, override val pageMeta: Meta, ) : WebPage { - override val snark: SnarkHtmlPlugin get() = this@KtorSiteBuilder.snark + override val snark: SnarkHtml get() = this@KtorSiteBuilder.snark override val data: DataTree<*> get() = this@KtorSiteBuilder.data override fun resolveRef(ref: String): String = this@KtorSiteBuilder.resolveRef(pageBaseUrl, ref) @@ -154,21 +155,22 @@ public class KtorSiteBuilder( override fun page(route: Name, pageMeta: Meta, content: context(HTML, WebPage) () -> Unit) { ktorRoute.get(route.toWebPath()) { + val request = call.request + //substitute host for url for backwards calls + val url = URLBuilder(baseUrl).apply { + protocol = URLProtocol.createOrDefault(request.origin.scheme) + host = request.origin.serverHost + port = request.origin.serverPort + } + + val modifiedPageMeta = pageMeta.toMutableMeta().apply { + "name" put route.toString() + "url" put url.buildString() + } + val pageBuilder = KtorWebPage(url.buildString(), Laminate(modifiedPageMeta, siteMeta)) + call.respondHtml { - val request = call.request - //substitute host for url for backwards calls - val url = URLBuilder(baseUrl).apply { - protocol = URLProtocol.createOrDefault(request.origin.scheme) - host = request.origin.serverHost - port = request.origin.serverPort - } - - val modifiedPageMeta = pageMeta.toMutableMeta().apply { - "name" put route.toString() - "url" put url.buildString() - } - - val pageBuilder = KtorWebPage(url.buildString(), Laminate(modifiedPageMeta, siteMeta)) + head{} content(this, pageBuilder) } } @@ -211,7 +213,7 @@ public class KtorSiteBuilder( } private fun Route.site( - snarkHtmlPlugin: SnarkHtmlPlugin, + snarkHtmlPlugin: SnarkHtml, data: DataTree<*>, baseUrl: String = "", siteMeta: Meta = data.meta, @@ -224,7 +226,7 @@ private fun Route.site( } public fun Application.site( - snark: SnarkHtmlPlugin, + snark: SnarkHtml, data: DataTree<*>, baseUrl: String = "", siteMeta: Meta = data.meta, diff --git a/snark-ktor/src/main/kotlin/space/kscience/snark/ktor/extractData.kt b/snark-ktor/src/main/kotlin/space/kscience/snark/ktor/extractData.kt index 04daf87..5718f04 100644 --- a/snark-ktor/src/main/kotlin/space/kscience/snark/ktor/extractData.kt +++ b/snark-ktor/src/main/kotlin/space/kscience/snark/ktor/extractData.kt @@ -9,31 +9,6 @@ import java.time.LocalDateTime import kotlin.io.path.* -//public fun KtorSiteBuilder.extractResources(uri: URI, targetPath: Path): Path { -// if (Files.isDirectory(targetPath)) { -// logger.info { "Using existing data directory at $targetPath." } -// } else { -// logger.info { "Copying data from $uri into $targetPath." } -// targetPath.createDirectories() -// //Copy everything into a temporary directory -// FileSystems.newFileSystem(uri, emptyMap()).use { fs -> -// val rootPath: Path = fs.provider().getPath(uri) -// Files.walk(rootPath).forEach { source: Path -> -// if (source.isRegularFile()) { -// val relative = source.relativeTo(rootPath).toString() -// val destination: Path = targetPath.resolve(relative) -// destination.parent.createDirectories() -// Files.copy(source, destination) -// } -// } -// } -// } -// return targetPath -//} -// -//public fun KtorSiteBuilder.extractResources(resource: String, targetPath: Path): Path = -// extractResources(javaClass.getResource(resource)!!.toURI(), targetPath) - private const val DEPLOY_DATE_FILE = "deployDate" private const val BUILD_DATE_FILE = "/buildDate"