WIP

build.gradle.kts

@@ -6,7 +6,7 @@ plugins {
 
 allprojects {
     group = "space.kscience"
-    version = "0.1.0-dev-1"
+    version = "0.2.0-dev-1"
 
     repositories {
         mavenCentral()

snark-core/src/commonMain/kotlin/space/kscience/snark/Snark.kt

@@ -0,0 +1,56 @@
+package space.kscience.snark
+
+import kotlinx.io.readByteArray
+import space.kscience.dataforge.context.Context
+import space.kscience.dataforge.context.PluginFactory
+import space.kscience.dataforge.context.PluginTag
+import space.kscience.dataforge.context.gather
+import space.kscience.dataforge.io.IOPlugin
+import space.kscience.dataforge.io.IOReader
+import space.kscience.dataforge.meta.Meta
+import space.kscience.dataforge.meta.get
+import space.kscience.dataforge.meta.string
+import space.kscience.dataforge.names.Name
+import space.kscience.dataforge.names.parseAsName
+import space.kscience.dataforge.workspace.WorkspacePlugin
+
+/**
+ * Represents a Snark workspace plugin.
+ */
+public class Snark : WorkspacePlugin() {
+    public val io: IOPlugin by require(IOPlugin)
+    override val tag: PluginTag get() = Companion.tag
+
+    public val readers: Map<Name, SnarkIOReader<Any>> by lazy {
+        context.gather<SnarkIOReader<Any>>(SnarkIOReader.DF_TYPE, inherit = true)
+    }
+
+    /**
+     * A lazy-initialized map of `TextProcessor` instances used for page-based text transformation.
+     *
+     * @property textProcessors The `TextProcessor` instances accessible by their names.
+     */
+    public val textProcessors: Map<Name, TextProcessor> by lazy {
+        context.gather(TextProcessor.DF_TYPE, true)
+    }
+
+    public fun textProcessor(transformationMeta: Meta): TextProcessor {
+        val transformationName = transformationMeta.string
+            ?: transformationMeta["name"].string ?: error("Transformation name not defined in $transformationMeta")
+        return textProcessors[transformationName.parseAsName()]
+            ?: error("Text transformation with name $transformationName not found in $this")
+    }
+
+    public companion object : PluginFactory<Snark> {
+        override val tag: PluginTag = PluginTag("snark")
+
+        override fun build(context: Context, meta: Meta): Snark = Snark()
+
+        private val byteArrayIOReader = IOReader {
+            readByteArray()
+        }
+
+        internal val byteArraySnarkParser = SnarkIOReader(byteArrayIOReader)
+
+    }
+}

snark-core/src/commonMain/kotlin/space/kscience/snark/SnarkIOReader.kt

@@ -0,0 +1,36 @@
+package space.kscience.snark
+
+import kotlinx.io.Source
+import space.kscience.dataforge.io.IOReader
+import space.kscience.dataforge.misc.DfId
+import space.kscience.snark.SnarkIOReader.Companion.DF_TYPE
+
+/**
+ * A wrapper class for IOReader that adds priority and MIME type handling.
+ *
+ * @param T The type of data to be read by the IOReader.
+ * @property reader The underlying IOReader instance used for reading data.
+ * @property types The set of supported types that can be read by the SnarkIOReader.
+ * @property priority The priority of the SnarkIOReader. Higher priority SnarkIOReader instances will be preferred over lower priority ones.
+ */
+@DfId(DF_TYPE)
+public class SnarkIOReader<out T>(
+    private val reader: IOReader<T>,
+    public val types: Set<String>,
+    public val priority: Int = DEFAULT_PRIORITY,
+) : IOReader<T> by reader {
+
+    public fun readFrom(source: String): T{
+
+    }
+
+    public companion object {
+        public const val DF_TYPE: String = "snark.reader"
+        public const val DEFAULT_PRIORITY: Int = 10
+    }
+}
+
+public fun <T : Any> SnarkIOReader(
+    reader: IOReader<T>,
+    vararg types: String,
+): SnarkIOReader<T> = SnarkIOReader(reader, types.toSet())

snark-core/src/commonMain/kotlin/space/kscience/snark/SnarkParser.kt

@@ -1,56 +0,0 @@
-package space.kscience.snark
-
-import kotlinx.io.Source
-import kotlinx.io.readByteArray
-import space.kscience.dataforge.context.Context
-import space.kscience.dataforge.io.IOReader
-import space.kscience.dataforge.io.asBinary
-import space.kscience.dataforge.io.readWith
-import space.kscience.dataforge.meta.Meta
-import space.kscience.dataforge.misc.DfId
-import kotlin.reflect.KType
-import kotlin.reflect.typeOf
-
-/**
- * A parser of binary content including priority flag and file extensions
- */
-@DfId(SnarkParser.TYPE)
-public interface SnarkParser<out R> {
-    public val type: KType
-
-    public val fileExtensions: Set<String>
-
-    public val priority: Int get() = DEFAULT_PRIORITY
-
-    //TODO use Binary instead of ByteArray
-    public fun parse(context: Context, meta: Meta, bytes: ByteArray): R
-
-    public fun asReader(context: Context, meta: Meta): IOReader<R> = object : IOReader<R> {
-        override val type: KType get() = this@SnarkParser.type
-
-        override fun readFrom(source: Source): R = parse(context, meta, source.readByteArray())
-    }
-
-    public companion object {
-        public const val TYPE: String = "snark.parser"
-        public const val DEFAULT_PRIORITY: Int = 10
-    }
-}
-
-@PublishedApi
-internal class SnarkParserWrapper<R : Any>(
-    val reader: IOReader<R>,
-    override val type: KType,
-    override val fileExtensions: Set<String>,
-) : SnarkParser<R> {
-    override fun parse(context: Context, meta: Meta, bytes: ByteArray): R = bytes.asBinary().readWith(reader)
-}
-
-/**
- * Create a generic parser from reader
- */
-@Suppress("FunctionName")
-public inline fun <reified R : Any> SnarkParser(
-    reader: IOReader<R>,
-    vararg fileExtensions: String,
-): SnarkParser<R> = SnarkParserWrapper(reader, typeOf<R>(), fileExtensions.toSet())

snark-core/src/commonMain/kotlin/space/kscience/snark/TextProcessor.kt

@@ -0,0 +1,20 @@
+package space.kscience.snark
+
+import space.kscience.dataforge.misc.DfId
+import space.kscience.dataforge.names.NameToken
+
+/**
+ * An object that conducts page-based text transformation. Like using link replacement or templating.
+ */
+@DfId(TextProcessor.DF_TYPE)
+public fun interface TextProcessor {
+
+    public fun process(text: String): String
+
+    public companion object {
+        public const val DF_TYPE: String = "snark.textTransformation"
+        public val TEXT_TRANSFORMATION_KEY: NameToken = NameToken("transformation")
+    }
+}
+
+

snark-core/src/jvmMain/kotlin/space/kscience/snark/ImageIOReader.kt

@@ -0,0 +1,21 @@
+package space.kscience.snark
+
+import kotlinx.io.Source
+import kotlinx.io.asInputStream
+import space.kscience.dataforge.io.IOReader
+import java.awt.image.BufferedImage
+import javax.imageio.ImageIO
+import kotlin.reflect.KType
+import kotlin.reflect.typeOf
+
+/**
+ * The ImageIOReader class is an implementation of the IOReader interface specifically for reading images using the ImageIO library.
+ * It reads the image data from a given source and returns a BufferedImage object.
+ *
+ * @property type The KType of the data to be read by the ImageIOReader.
+ */
+public object ImageIOReader : IOReader<BufferedImage> {
+    override val type: KType get() = typeOf<BufferedImage>()
+
+    override fun readFrom(source: Source): BufferedImage = ImageIO.read(source.asInputStream())
+}

snark-core/src/jvmMain/kotlin/space/kscience/snark/snarkWorkspace.kt

@@ -0,0 +1,66 @@
+@file:OptIn(DFExperimental::class)
+
+package space.kscience.snark
+
+import space.kscience.dataforge.data.DataSet
+import space.kscience.dataforge.data.DataTree
+import space.kscience.dataforge.data.node
+import space.kscience.dataforge.io.Binary
+import space.kscience.dataforge.io.IOPlugin
+import space.kscience.dataforge.meta.*
+import space.kscience.dataforge.misc.DFExperimental
+import space.kscience.dataforge.names.Name
+import space.kscience.dataforge.workspace.Workspace
+import space.kscience.dataforge.workspace.WorkspaceBuilder
+import space.kscience.dataforge.workspace.readRawDirectory
+import kotlin.io.path.Path
+import kotlin.io.path.toPath
+
+
+/**
+ * Reads the specified resources and returns a [DataTree] containing the data.
+ *
+ * @param resources The names of the resources to read.
+ * @param classLoader The class loader to use for loading the resources. By default, it uses the current thread's context class loader.
+ * @return A DataTree containing the data read from the resources.
+ */
+private fun IOPlugin.readResources(
+    vararg resources: String,
+    classLoader: ClassLoader = Thread.currentThread().contextClassLoader,
+): DataTree<Binary> {
+//    require(resource.isNotBlank()) {"Can't mount root resource tree as data root"}
+    return DataTree {
+        resources.forEach { resource ->
+            val path = classLoader.getResource(resource)?.toURI()?.toPath() ?: error(
+                "Resource with name $resource is not resolved"
+            )
+            node(resource, readRawDirectory(path))
+        }
+    }
+}
+
+public fun Snark.workspace(
+    meta: Meta,
+    customData: DataSet<*> = DataSet.EMPTY,
+    workspaceBuilder: WorkspaceBuilder.() -> Unit = {},
+): Workspace = Workspace {
+
+
+    data {
+        node(Name.EMPTY, customData)
+        meta.getIndexed("directory").forEach { (index, directoryMeta) ->
+            val dataDirectory = directoryMeta["path"].string ?: error("Directory path not defined")
+            val nodeName = directoryMeta["name"].string ?: directoryMeta.string ?: index ?: ""
+            val data = io.readRawDirectory(Path(dataDirectory))
+            node(nodeName, data)
+        }
+        meta.getIndexed("resource").forEach { (index, resourceMeta) ->
+            val resource = resourceMeta["path"]?.stringList ?: listOf("/")
+            val nodeName = resourceMeta["name"].string ?: resourceMeta.string ?: index ?: ""
+            val data: DataTree<Binary> = io.readResources(*resource.toTypedArray())
+            node(nodeName, data)
+        }
+    }
+
+    workspaceBuilder()
+}

snark-html/build.gradle.kts

@@ -1,5 +1,5 @@
 plugins {
-    id("space.kscience.gradle.jvm")
+    id("space.kscience.gradle.mpp")
     `maven-publish`
 }
 
@@ -7,21 +7,22 @@ val dataforgeVersion: String by rootProject.extra
 val ktorVersion = space.kscience.gradle.KScienceVersions.ktorVersion
 
 kscience{
+    jvm()
     useContextReceivers()
-}
+    commonMain{
-
-dependencies {
         api(projects.snarkCore)
 
-    api("org.jetbrains.kotlinx:kotlinx-html:0.8.0")
+        api(spclibs.kotlinx.html)
         api("org.jetbrains.kotlin-wrappers:kotlin-css")
 
-    api("io.ktor:ktor-utils:$ktorVersion")
+        api("io.ktor:ktor-http:$ktorVersion")
-
         api("space.kscience:dataforge-io-yaml:$dataforgeVersion")
-    api("org.jetbrains:markdown:0.4.0")
+        api("org.jetbrains:markdown:0.5.2")
     }
 
+}
+
+
 readme {
     maturity = space.kscience.gradle.Maturity.EXPERIMENTAL
     feature("data") { "Data-based processing. Instead of traditional layout-based" }

snark-html/src/jvmMain/kotlin/space/kscience/snark/html/SiteBuilder.kt

@@ -39,7 +39,7 @@ public interface SiteBuilder : ContextAware, SnarkContext {
     /**
      * Snark plugin and context used for layout resolution, preprocessors, etc
      */
-    public val snark: SnarkHtmlPlugin
+    public val snark: SnarkHtml
 
     override val context: Context get() = snark.context
 
@@ -49,7 +49,7 @@ public interface SiteBuilder : ContextAware, SnarkContext {
     public val siteMeta: Meta
 
     /**
-     * Serve a static data as a file from [data] with given [dataName] at given [routeName].
+     * Serve static data as a file from [data] with given [dataName] at given [routeName].
      */
     public fun static(dataName: Name, routeName: Name = dataName)
 

snark-html/src/jvmMain/kotlin/space/kscience/snark/html/SnarkHtml.kt

@@ -0,0 +1,141 @@
+@file:OptIn(DFExperimental::class)
+
+package space.kscience.snark.html
+
+import io.ktor.http.ContentType
+import kotlinx.io.readByteArray
+import space.kscience.dataforge.context.*
+import space.kscience.dataforge.data.*
+import space.kscience.dataforge.io.IOPlugin
+import space.kscience.dataforge.io.IOReader
+import space.kscience.dataforge.io.JsonMetaFormat
+import space.kscience.dataforge.io.yaml.YamlMetaFormat
+import space.kscience.dataforge.io.yaml.YamlPlugin
+import space.kscience.dataforge.meta.*
+import space.kscience.dataforge.misc.DFExperimental
+import space.kscience.dataforge.names.*
+import space.kscience.dataforge.provider.dfId
+import space.kscience.dataforge.workspace.*
+import space.kscience.snark.ImageIOReader
+import space.kscience.snark.Snark
+import space.kscience.snark.SnarkIOReader
+import space.kscience.snark.TextProcessor
+import java.net.URLConnection
+import kotlin.io.path.Path
+import kotlin.io.path.extension
+
+public fun <T : Any> SnarkIOReader(
+    reader: IOReader<T>,
+    vararg types: ContentType,
+    priority: Int = SnarkIOReader.DEFAULT_PRIORITY,
+): SnarkIOReader<T> = SnarkIOReader(reader, types.map { it.toString() }.toSet(), priority)
+
+
+/**
+ * A plugin used for rendering a [DataTree] as HTML
+ */
+public class SnarkHtml : WorkspacePlugin() {
+    private val snark by require(Snark)
+    private val yaml by require(YamlPlugin)
+    public val io: IOPlugin get() = snark.io
+
+    override val tag: PluginTag get() = Companion.tag
+
+    /**
+     * Lazy-initialized variable that holds a map of site layouts.
+     *
+     * @property siteLayouts The map of site layouts, where the key is the layout name and the value is the corresponding SiteLayout object.
+     */
+    private val siteLayouts: Map<Name, SiteLayout> by lazy {
+        context.gather(SiteLayout.TYPE, true)
+    }
+
+
+    internal fun siteLayout(layoutMeta: Meta): SiteLayout {
+        val layoutName = layoutMeta.string
+            ?: layoutMeta["name"].string ?: error("Layout name not defined in $layoutMeta")
+        return siteLayouts[layoutName.parseAsName()] ?: error("Layout with name $layoutName not found in $this")
+    }
+
+    override fun content(target: String): Map<Name, Any> = when (target) {
+        SnarkIOReader::class.dfId -> mapOf(
+            "html".asName() to HtmlIOFormat.snarkReader,
+            "markdown".asName() to MarkdownIOFormat.snarkReader,
+            "json".asName() to SnarkIOReader(JsonMetaFormat, ContentType.Application.Json),
+            "yaml".asName() to SnarkIOReader(YamlMetaFormat, "text/yaml"),
+            "png".asName() to SnarkIOReader(ImageIOReader, ContentType.Image.PNG),
+            "jpg".asName() to SnarkIOReader(ImageIOReader, ContentType.Image.JPEG),
+            "gif".asName() to SnarkIOReader(ImageIOReader, ContentType.Image.GIF),
+            "svg".asName() to SnarkIOReader(IOReader.binary, ContentType.Image.SVG, ContentType.parse("svg")),
+            "raw".asName() to SnarkIOReader(
+                IOReader.binary,
+                "css",
+                "js",
+                "javascript",
+                "scss",
+                "woff",
+                "woff2",
+                "ttf",
+                "eot"
+            )
+        )
+
+        else -> super.content(target)
+    }
+
+
+    public val preprocess: TaskReference<String> by task<String> {
+        pipeFrom<String,String>(dataByType<String>()) { text, _, meta ->
+            meta[TextProcessor.TEXT_TRANSFORMATION_KEY]?.let {
+                snark.textProcessor(it).process(text)
+            } ?: text
+        }
+    }
+
+    public val parse: TaskReference<Any> by task<Any> {
+        from(preprocess).forEach { (dataName, data) ->
+            //remove extensions for data files
+            val filePath = meta[FileData.FILE_PATH_KEY]?.string ?: dataName.toString()
+            val fileType = URLConnection.guessContentTypeFromName(filePath) ?: Path(filePath).extension
+            val newName = dataName.replaceLast {
+                if (fileType in setOf("md", "html", "yaml", "json")) {
+                    NameToken(it.body.substringBeforeLast("."), it.index)
+                } else {
+                    it
+                }
+            }
+            val parser = snark.readers.values.filter { parser ->
+                fileType in parser.types
+            }.maxByOrNull {
+                it.priority
+            } ?: run {
+                logger.debug { "The parser is not found for file $filePath with meta $meta" }
+                byteArraySnarkParser
+            }
+            data(newName, data.map { string: String ->
+                parser.readFrom(string)
+            })
+        }
+    }
+
+
+//    public val textTransformationAction: Action<String, String> = Action.map<String, String> {
+//        val transformations = actionMeta.getIndexed("transformation").entries.sortedBy {
+//            it.key?.toIntOrNull() ?: 0
+//        }.map { it.value }
+//    }
+
+
+    public companion object : PluginFactory<SnarkHtml> {
+        override val tag: PluginTag = PluginTag("snark.html")
+
+        override fun build(context: Context, meta: Meta): SnarkHtml = SnarkHtml()
+
+        private val byteArrayIOReader = IOReader {
+            readByteArray()
+        }
+
+        internal val byteArraySnarkParser = SnarkIOReader(byteArrayIOReader)
+
+    }
+}

snark-html/src/jvmMain/kotlin/space/kscience/snark/html/StaticSiteBuilder.kt

@@ -28,7 +28,7 @@ import kotlin.reflect.typeOf
  * An implementation of [SiteBuilder] to render site as a static directory [outputPath]
  */
 internal class StaticSiteBuilder(
-    override val snark: SnarkHtmlPlugin,
+    override val snark: SnarkHtml,
     override val data: DataTree<*>,
     override val siteMeta: Meta,
     private val baseUrl: String,
@@ -121,7 +121,7 @@ internal class StaticSiteBuilder(
     inner class StaticWebPage(override val pageMeta: Meta) : WebPage {
         override val data: DataTree<*> get() = this@StaticSiteBuilder.data
 
-        override val snark: SnarkHtmlPlugin get() = this@StaticSiteBuilder.snark
+        override val snark: SnarkHtml get() = this@StaticSiteBuilder.snark
 
 
         override fun resolveRef(ref: String): String =
@@ -186,7 +186,7 @@ internal class StaticSiteBuilder(
  * Use [siteUrl] as a base for all resolved URLs. By default, use [outputPath] absolute path as a base.
  *
  */
-public fun SnarkHtmlPlugin.static(
+public fun SnarkHtml.static(
     data: DataTree<*>,
     outputPath: Path,
     siteUrl: String = outputPath.absolutePathString().replace("\\", "/"),

snark-html/src/jvmMain/kotlin/space/kscience/snark/html/WebPage.kt

@@ -25,7 +25,7 @@ public fun Name.toWebPath(): String = tokens.joinToString(separator = "/") {
 @SnarkBuilder
 public interface WebPage : ContextAware, SnarkContext {
 
-    public val snark: SnarkHtmlPlugin
+    public val snark: SnarkHtml
 
     override val context: Context get() = snark.context
 

snark-html/src/jvmMain/kotlin/space/kscience/snark/html/WebPagePreprocessor.kt

@@ -1,23 +1,8 @@
 package space.kscience.snark.html
 
 import space.kscience.dataforge.meta.string
-import space.kscience.dataforge.misc.DfId
-import space.kscience.dataforge.names.NameToken
 import space.kscience.dataforge.names.parseAsName
-
+import space.kscience.snark.TextProcessor
-/**
- * An object that conducts page-based text transformation. Like using link replacement or templating.
- */
-@DfId(TextProcessor.TYPE)
-public fun interface TextProcessor {
-    context(WebPage)
-    public fun process(text: String): String
-
-    public companion object {
-        public const val TYPE: String = "snark.textTransformation"
-        public val TEXT_TRANSFORMATION_KEY: NameToken = NameToken("transformation")
-    }
-}
 
 /**
  * A basic [TextProcessor] that replaces `${...}` expressions in text. The following expressions are recognised:
@@ -27,34 +12,32 @@ public fun interface TextProcessor {
  * * `pageMeta.get("...") -> [WebPage.pageMeta] get string method
  * Otherwise return unchanged string
  */
-public object BasicTextProcessor : TextProcessor {
+public class WebPagePreprocessor(public val page: WebPage) : TextProcessor {
 
     private val regex = """\$\{([\w.]*)(?>\("(.*)"\))?}""".toRegex()
 
-    context(WebPage)
+
     override fun process(text: String): String = text.replace(regex) { match ->
         when (match.groups[1]!!.value) {
-            "homeRef" -> homeRef
+            "homeRef" -> page.homeRef
             "resolveRef" -> {
                 val refString = match.groups[2]?.value ?: error("resolveRef requires a string (quoted) argument")
-                resolveRef(refString)
+                page.resolveRef(refString)
             }
 
             "resolvePageRef" -> {
                 val refString = match.groups[2]?.value
                     ?: error("resolvePageRef requires a string (quoted) argument")
-                localisedPageRef(refString.parseAsName())
+                page.localisedPageRef(refString.parseAsName())
             }
 
             "pageMeta.get" -> {
                 val nameString = match.groups[2]?.value
                     ?: error("resolvePageRef requires a string (quoted) argument")
-                pageMeta[nameString.parseAsName()].string ?: "@null"
+                page.pageMeta[nameString.parseAsName()].string ?: "@null"
             }
 
             else -> match.value
         }
     }
 }
-
-

snark-html/src/jvmMain/kotlin/space/kscience/snark/html/htmlIoFormats.kt

@@ -0,0 +1,50 @@
+package space.kscience.snark.html
+
+import io.ktor.http.ContentType
+import kotlinx.html.div
+import kotlinx.html.unsafe
+import kotlinx.io.Source
+import kotlinx.io.readString
+import org.intellij.markdown.flavours.commonmark.CommonMarkFlavourDescriptor
+import org.intellij.markdown.html.HtmlGenerator
+import org.intellij.markdown.parser.MarkdownParser
+import space.kscience.dataforge.io.IOReader
+import space.kscience.snark.SnarkIOReader
+import kotlin.reflect.KType
+import kotlin.reflect.typeOf
+
+public object HtmlIOFormat : IOReader<HtmlFragment> {
+    override val type: KType = typeOf<HtmlFragment>()
+
+    override fun readFrom(source: Source): HtmlFragment = HtmlFragment { page ->
+        div {
+            unsafe { +source.readString() }
+        }
+    }
+
+    public val snarkReader: SnarkIOReader<HtmlFragment> = SnarkIOReader(this, ContentType.Text.Html)
+
+}
+
+public object MarkdownIOFormat : IOReader<HtmlFragment> {
+    override val type: KType = typeOf<HtmlFragment>()
+
+    private val markdownFlavor = CommonMarkFlavourDescriptor()
+    private val markdownParser = MarkdownParser(markdownFlavor)
+
+    override fun readFrom(source: Source): HtmlFragment = HtmlFragment { page ->
+        val transformedText = source.readString()
+        val parsedTree = markdownParser.buildMarkdownTreeFromString(transformedText)
+        val htmlString = HtmlGenerator(transformedText, parsedTree, markdownFlavor).generateHtml()
+
+        div {
+            unsafe {
+                +htmlString
+            }
+        }
+    }
+
+    public val snarkReader: SnarkIOReader<HtmlFragment> = SnarkIOReader(this, ContentType.parse("text/markdown"))
+
+}
+

snark-html/src/main/kotlin/space/kscience/snark/html/ImageIOReader.kt

@@ -1,16 +0,0 @@
-package space.kscience.snark.html
-
-import io.ktor.util.asStream
-import kotlinx.io.Source
-import kotlinx.io.asInputStream
-import space.kscience.dataforge.io.IOReader
-import java.awt.image.BufferedImage
-import javax.imageio.ImageIO
-import kotlin.reflect.KType
-import kotlin.reflect.typeOf
-
-internal object ImageIOReader : IOReader<BufferedImage> {
-    override val type: KType get() = typeOf<BufferedImage>()
-
-    override fun readFrom(source: Source): BufferedImage = ImageIO.read(source.asInputStream())
-}

snark-html/src/main/kotlin/space/kscience/snark/html/SnarkHtmlPlugin.kt

@@ -1,125 +0,0 @@
-package space.kscience.snark.html
-
-import io.ktor.utils.io.core.readBytes
-import kotlinx.io.readByteArray
-import space.kscience.dataforge.context.*
-import space.kscience.dataforge.data.DataTree
-import space.kscience.dataforge.data.node
-import space.kscience.dataforge.io.IOPlugin
-import space.kscience.dataforge.io.IOReader
-import space.kscience.dataforge.io.JsonMetaFormat
-import space.kscience.dataforge.io.yaml.YamlMetaFormat
-import space.kscience.dataforge.io.yaml.YamlPlugin
-import space.kscience.dataforge.meta.Meta
-import space.kscience.dataforge.meta.get
-import space.kscience.dataforge.meta.string
-import space.kscience.dataforge.misc.DFExperimental
-import space.kscience.dataforge.names.Name
-import space.kscience.dataforge.names.asName
-import space.kscience.dataforge.names.parseAsName
-import space.kscience.dataforge.workspace.FileData
-import space.kscience.dataforge.workspace.readDataDirectory
-import space.kscience.snark.SnarkParser
-import java.nio.file.Path
-import kotlin.io.path.extension
-import kotlin.io.path.toPath
-
-/**
- * A plugin used for rendering a [DataTree] as HTML
- */
-public class SnarkHtmlPlugin : AbstractPlugin() {
-    private val yaml by require(YamlPlugin)
-    public val io: IOPlugin get() = yaml.io
-
-    override val tag: PluginTag get() = Companion.tag
-
-    internal val parsers: Map<Name, SnarkParser<Any>> by lazy {
-        context.gather(SnarkParser.TYPE, true)
-    }
-
-    private val siteLayouts: Map<Name, SiteLayout> by lazy {
-        context.gather(SiteLayout.TYPE, true)
-    }
-
-    private val textProcessors: Map<Name, TextProcessor> by lazy {
-        context.gather(TextProcessor.TYPE, true)
-    }
-
-    internal fun siteLayout(layoutMeta: Meta): SiteLayout {
-        val layoutName = layoutMeta.string
-            ?: layoutMeta["name"].string ?: error("Layout name not defined in $layoutMeta")
-        return siteLayouts[layoutName.parseAsName()] ?: error("Layout with name $layoutName not found in $this")
-    }
-
-    internal fun textProcessor(transformationMeta: Meta): TextProcessor {
-        val transformationName = transformationMeta.string
-            ?: transformationMeta["name"].string ?: error("Transformation name not defined in $transformationMeta")
-        return textProcessors[transformationName.parseAsName()]
-            ?: error("Text transformation with name $transformationName not found in $this")
-    }
-
-    override fun content(target: String): Map<Name, Any> = when (target) {
-        SnarkParser.TYPE -> mapOf(
-            "html".asName() to SnarkHtmlParser,
-            "markdown".asName() to SnarkMarkdownParser,
-            "json".asName() to SnarkParser(JsonMetaFormat, "json"),
-            "yaml".asName() to SnarkParser(YamlMetaFormat, "yaml", "yml"),
-            "png".asName() to SnarkParser(ImageIOReader, "png"),
-            "jpg".asName() to SnarkParser(ImageIOReader, "jpg", "jpeg"),
-            "gif".asName() to SnarkParser(ImageIOReader, "gif"),
-            "svg".asName() to SnarkParser(IOReader.binary, "svg"),
-            "raw".asName() to SnarkParser(IOReader.binary, "css", "js", "scss", "woff", "woff2", "ttf", "eot")
-        )
-
-        TextProcessor.TYPE -> mapOf(
-            "basic".asName() to BasicTextProcessor
-        )
-
-        else -> super.content(target)
-    }
-
-    public companion object : PluginFactory<SnarkHtmlPlugin> {
-        override val tag: PluginTag = PluginTag("snark")
-
-        override fun build(context: Context, meta: Meta): SnarkHtmlPlugin = SnarkHtmlPlugin()
-
-        private val byteArrayIOReader = IOReader {
-            readByteArray()
-        }
-
-        internal val byteArraySnarkParser = SnarkParser(byteArrayIOReader)
-    }
-}
-
-@OptIn(DFExperimental::class)
-public fun SnarkHtmlPlugin.readDirectory(path: Path): DataTree<Any> = io.readDataDirectory(
-    path,
-    setOf("md", "html", "yaml", "json")
-) { dataPath, meta ->
-    val fileExtension = meta[FileData.FILE_EXTENSION_KEY].string ?: dataPath.extension
-    val parser: SnarkParser<Any> = parsers.values.filter { parser ->
-        fileExtension in parser.fileExtensions
-    }.maxByOrNull {
-        it.priority
-    } ?: run {
-        logger.debug { "The parser is not found for file $dataPath with meta $meta" }
-        SnarkHtmlPlugin.byteArraySnarkParser
-    }
-
-    parser.asReader(context, meta)
-}
-
-public fun SnarkHtmlPlugin.readResources(
-    vararg resources: String,
-    classLoader: ClassLoader = Thread.currentThread().contextClassLoader,
-): DataTree<Any> {
-//    require(resource.isNotBlank()) {"Can't mount root resource tree as data root"}
-    return DataTree {
-        resources.forEach { resource ->
-            val path = classLoader.getResource(resource)?.toURI()?.toPath() ?: error(
-                "Resource with name $resource is not resolved"
-            )
-            node(resource, readDirectory(path))
-        }
-    }
-}

snark-html/src/main/kotlin/space/kscience/snark/html/SnarkTextParser.kt

@@ -1,58 +0,0 @@
-package space.kscience.snark.html
-
-import kotlinx.html.div
-import kotlinx.html.unsafe
-import org.intellij.markdown.flavours.commonmark.CommonMarkFlavourDescriptor
-import org.intellij.markdown.html.HtmlGenerator
-import org.intellij.markdown.parser.MarkdownParser
-import space.kscience.dataforge.context.Context
-import space.kscience.dataforge.meta.Meta
-import space.kscience.dataforge.meta.get
-import space.kscience.snark.SnarkParser
-import kotlin.reflect.KType
-import kotlin.reflect.typeOf
-
-public abstract class SnarkTextParser<R> : SnarkParser<R> {
-    public abstract fun parseText(text: String, meta: Meta): R
-
-    override fun parse(context: Context, meta: Meta, bytes: ByteArray): R =
-        parseText(bytes.decodeToString(), meta)
-
-    public fun transformText(text: String, meta: Meta, page: WebPage): String =
-        meta[TextProcessor.TEXT_TRANSFORMATION_KEY]?.let {
-            with(page) { page.snark.textProcessor(it).process(text) }
-        } ?: text
-}
-
-
-internal object SnarkHtmlParser : SnarkTextParser<HtmlFragment>() {
-    override val fileExtensions: Set<String> = setOf("html")
-    override val type: KType = typeOf<HtmlFragment>()
-
-    override fun parseText(text: String, meta: Meta): HtmlFragment = HtmlFragment { page ->
-        div {
-            unsafe { +transformText(text, meta, page) }
-        }
-    }
-}
-
-internal object SnarkMarkdownParser : SnarkTextParser<HtmlFragment>() {
-    override val fileExtensions: Set<String> = setOf("markdown", "mdown", "mkdn", "mkd", "md")
-    override val type: KType = typeOf<HtmlFragment>()
-
-    private val markdownFlavor = CommonMarkFlavourDescriptor()
-    private val markdownParser = MarkdownParser(markdownFlavor)
-
-    override fun parseText(text: String, meta: Meta): HtmlFragment = HtmlFragment { page ->
-        val transformedText = SnarkHtmlParser.transformText(text, meta, page)
-        val parsedTree = markdownParser.buildMarkdownTreeFromString(transformedText)
-        val htmlString = HtmlGenerator(transformedText, parsedTree, markdownFlavor).generateHtml()
-
-        div {
-            unsafe {
-                +htmlString
-            }
-        }
-    }
-}
-

snark-ktor/src/main/kotlin/space/kscience/snark/ktor/KtorSiteBuilder.kt

@@ -17,6 +17,7 @@ import io.ktor.server.routing.routing
 import kotlinx.css.CssBuilder
 import kotlinx.html.CommonAttributeGroupFacade
 import kotlinx.html.HTML
+import kotlinx.html.head
 import kotlinx.html.style
 import space.kscience.dataforge.context.error
 import space.kscience.dataforge.context.logger
@@ -33,7 +34,7 @@ import space.kscience.dataforge.names.endsWith
 import space.kscience.dataforge.names.plus
 import space.kscience.dataforge.workspace.FileData
 import space.kscience.snark.html.SiteBuilder
-import space.kscience.snark.html.SnarkHtmlPlugin
+import space.kscience.snark.html.SnarkHtml
 import space.kscience.snark.html.WebPage
 import space.kscience.snark.html.toWebPath
 import java.nio.file.Path
@@ -46,7 +47,7 @@ public fun CommonAttributeGroupFacade.css(block: CssBuilder.() -> Unit) {
 }
 
 public class KtorSiteBuilder(
-    override val snark: SnarkHtmlPlugin,
+    override val snark: SnarkHtml,
     override val data: DataTree<*>,
     override val siteMeta: Meta,
     private val baseUrl: String,
@@ -134,7 +135,7 @@ public class KtorSiteBuilder(
         val pageBaseUrl: String,
         override val pageMeta: Meta,
     ) : WebPage {
-        override val snark: SnarkHtmlPlugin get() = this@KtorSiteBuilder.snark
+        override val snark: SnarkHtml get() = this@KtorSiteBuilder.snark
         override val data: DataTree<*> get() = this@KtorSiteBuilder.data
 
         override fun resolveRef(ref: String): String = this@KtorSiteBuilder.resolveRef(pageBaseUrl, ref)
@@ -154,7 +155,6 @@ public class KtorSiteBuilder(
 
     override fun page(route: Name, pageMeta: Meta, content: context(HTML, WebPage) () -> Unit) {
         ktorRoute.get(route.toWebPath()) {
-            call.respondHtml {
             val request = call.request
             //substitute host for url for backwards calls
             val url = URLBuilder(baseUrl).apply {
@@ -167,8 +167,10 @@ public class KtorSiteBuilder(
                 "name" put route.toString()
                 "url" put url.buildString()
             }
-
             val pageBuilder = KtorWebPage(url.buildString(), Laminate(modifiedPageMeta, siteMeta))
+
+            call.respondHtml {
+                head{}
                 content(this, pageBuilder)
             }
         }
@@ -211,7 +213,7 @@ public class KtorSiteBuilder(
 }
 
 private fun Route.site(
-    snarkHtmlPlugin: SnarkHtmlPlugin,
+    snarkHtmlPlugin: SnarkHtml,
     data: DataTree<*>,
     baseUrl: String = "",
     siteMeta: Meta = data.meta,
@@ -224,7 +226,7 @@ private fun Route.site(
 }
 
 public fun Application.site(
-    snark: SnarkHtmlPlugin,
+    snark: SnarkHtml,
     data: DataTree<*>,
     baseUrl: String = "",
     siteMeta: Meta = data.meta,

snark-ktor/src/main/kotlin/space/kscience/snark/ktor/extractData.kt

@@ -9,31 +9,6 @@ import java.time.LocalDateTime
 import kotlin.io.path.*
 
 
-//public fun KtorSiteBuilder.extractResources(uri: URI, targetPath: Path): Path {
-//    if (Files.isDirectory(targetPath)) {
-//        logger.info { "Using existing data directory at $targetPath." }
-//    } else {
-//        logger.info { "Copying data from $uri into $targetPath." }
-//        targetPath.createDirectories()
-//        //Copy everything into a temporary directory
-//        FileSystems.newFileSystem(uri, emptyMap<String, Any>()).use { fs ->
-//            val rootPath: Path = fs.provider().getPath(uri)
-//            Files.walk(rootPath).forEach { source: Path ->
-//                if (source.isRegularFile()) {
-//                    val relative = source.relativeTo(rootPath).toString()
-//                    val destination: Path = targetPath.resolve(relative)
-//                    destination.parent.createDirectories()
-//                    Files.copy(source, destination)
-//                }
-//            }
-//        }
-//    }
-//    return targetPath
-//}
-//
-//public fun KtorSiteBuilder.extractResources(resource: String, targetPath: Path): Path =
-//    extractResources(javaClass.getResource(resource)!!.toURI(), targetPath)
-
 private const val DEPLOY_DATE_FILE = "deployDate"
 private const val BUILD_DATE_FILE = "/buildDate"