From eeaa080a8843c0603eadf5c9a5dc5bc19a1191df Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Tue, 28 Nov 2023 19:45:16 +0300 Subject: [PATCH] Explicit postprocessor --- .../space/kscience/snark/SnarkIOReader.kt | 37 ++++++---- .../space/kscience/snark/TextProcessor.kt | 2 +- .../space/kscience/snark/html/HtmlData.kt | 12 +-- .../space/kscience/snark/html/SnarkHtml.kt | 21 ++++-- .../snark/html/WebPagePostprocessor.kt | 74 +++++++++++++++++++ .../snark/html/WebPagePreprocessor.kt | 43 ----------- .../html/{htmlIoFormats.kt => readers.kt} | 30 ++++---- 7 files changed, 132 insertions(+), 87 deletions(-) create mode 100644 snark-html/src/jvmMain/kotlin/space/kscience/snark/html/WebPagePostprocessor.kt delete mode 100644 snark-html/src/jvmMain/kotlin/space/kscience/snark/html/WebPagePreprocessor.kt rename snark-html/src/jvmMain/kotlin/space/kscience/snark/html/{htmlIoFormats.kt => readers.kt} (58%) diff --git a/snark-core/src/commonMain/kotlin/space/kscience/snark/SnarkIOReader.kt b/snark-core/src/commonMain/kotlin/space/kscience/snark/SnarkIOReader.kt index 7e82f62..7fb293f 100644 --- a/snark-core/src/commonMain/kotlin/space/kscience/snark/SnarkIOReader.kt +++ b/snark-core/src/commonMain/kotlin/space/kscience/snark/SnarkIOReader.kt @@ -1,10 +1,23 @@ package space.kscience.snark -import kotlinx.io.Source import space.kscience.dataforge.io.IOReader +import space.kscience.dataforge.io.asBinary import space.kscience.dataforge.misc.DfId +import space.kscience.snark.SnarkIOReader.Companion.DEFAULT_PRIORITY import space.kscience.snark.SnarkIOReader.Companion.DF_TYPE +@DfId(DF_TYPE) +public interface SnarkIOReader: IOReader { + public val types: Set + public val priority: Int get() = DEFAULT_PRIORITY + public fun readFrom(source: String): T + + public companion object { + public const val DF_TYPE: String = "snark.reader" + public const val DEFAULT_PRIORITY: Int = 10 + } +} + /** * A wrapper class for IOReader that adds priority and MIME type handling. * @@ -13,24 +26,18 @@ import space.kscience.snark.SnarkIOReader.Companion.DF_TYPE * @property types The set of supported types that can be read by the SnarkIOReader. * @property priority The priority of the SnarkIOReader. Higher priority SnarkIOReader instances will be preferred over lower priority ones. */ -@DfId(DF_TYPE) -public class SnarkIOReader( + +private class SnarkIOReaderWrapper( private val reader: IOReader, - public val types: Set, - public val priority: Int = DEFAULT_PRIORITY, -) : IOReader by reader { + override val types: Set, + override val priority: Int = DEFAULT_PRIORITY, +) : IOReader by reader, SnarkIOReader { - public fun readFrom(source: String): T{ - - } - - public companion object { - public const val DF_TYPE: String = "snark.reader" - public const val DEFAULT_PRIORITY: Int = 10 - } + override fun readFrom(source: String): T = readFrom(source.encodeToByteArray().asBinary()) } public fun SnarkIOReader( reader: IOReader, vararg types: String, -): SnarkIOReader = SnarkIOReader(reader, types.toSet()) \ No newline at end of file + priority: Int = DEFAULT_PRIORITY +): SnarkIOReader = SnarkIOReaderWrapper(reader, types.toSet(), priority) \ No newline at end of file diff --git a/snark-core/src/commonMain/kotlin/space/kscience/snark/TextProcessor.kt b/snark-core/src/commonMain/kotlin/space/kscience/snark/TextProcessor.kt index ba77fa4..5112e72 100644 --- a/snark-core/src/commonMain/kotlin/space/kscience/snark/TextProcessor.kt +++ b/snark-core/src/commonMain/kotlin/space/kscience/snark/TextProcessor.kt @@ -9,7 +9,7 @@ import space.kscience.dataforge.names.NameToken @DfId(TextProcessor.DF_TYPE) public fun interface TextProcessor { - public fun process(text: String): String + public fun process(text: CharSequence): String public companion object { public const val DF_TYPE: String = "snark.textTransformation" diff --git a/snark-html/src/jvmMain/kotlin/space/kscience/snark/html/HtmlData.kt b/snark-html/src/jvmMain/kotlin/space/kscience/snark/html/HtmlData.kt index 6fe0bcb..cea0466 100644 --- a/snark-html/src/jvmMain/kotlin/space/kscience/snark/html/HtmlData.kt +++ b/snark-html/src/jvmMain/kotlin/space/kscience/snark/html/HtmlData.kt @@ -13,22 +13,18 @@ import space.kscience.snark.SnarkContext //TODO replace by VisionForge type -//typealias HtmlFragment = context(PageBuilder, TagConsumer<*>) () -> Unit public fun interface HtmlFragment { - public fun TagConsumer<*>.renderFragment(page: WebPage) - //TODO move pageBuilder to a context receiver after KT-52967 is fixed + public fun TagConsumer<*>.renderFragment() } public typealias HtmlData = Data -//fun HtmlData(meta: Meta, content: context(PageBuilder) TagConsumer<*>.() -> Unit): HtmlData = -// Data(HtmlFragment(content), meta) - - context(WebPage) public fun FlowContent.htmlData(data: HtmlData): Unit = runBlocking(Dispatchers.IO) { - with(data.await()) { consumer.renderFragment(page) } + withSnarkPage(page) { + with(data.await()) { consumer.renderFragment() } + } } context(SnarkContext) diff --git a/snark-html/src/jvmMain/kotlin/space/kscience/snark/html/SnarkHtml.kt b/snark-html/src/jvmMain/kotlin/space/kscience/snark/html/SnarkHtml.kt index 110f65e..ae52cde 100644 --- a/snark-html/src/jvmMain/kotlin/space/kscience/snark/html/SnarkHtml.kt +++ b/snark-html/src/jvmMain/kotlin/space/kscience/snark/html/SnarkHtml.kt @@ -6,12 +6,15 @@ import io.ktor.http.ContentType import kotlinx.io.readByteArray import space.kscience.dataforge.context.* import space.kscience.dataforge.data.* +import space.kscience.dataforge.io.Binary import space.kscience.dataforge.io.IOPlugin import space.kscience.dataforge.io.IOReader import space.kscience.dataforge.io.JsonMetaFormat import space.kscience.dataforge.io.yaml.YamlMetaFormat import space.kscience.dataforge.io.yaml.YamlPlugin -import space.kscience.dataforge.meta.* +import space.kscience.dataforge.meta.Meta +import space.kscience.dataforge.meta.get +import space.kscience.dataforge.meta.string import space.kscience.dataforge.misc.DFExperimental import space.kscience.dataforge.names.* import space.kscience.dataforge.provider.dfId @@ -28,7 +31,7 @@ public fun SnarkIOReader( reader: IOReader, vararg types: ContentType, priority: Int = SnarkIOReader.DEFAULT_PRIORITY, -): SnarkIOReader = SnarkIOReader(reader, types.map { it.toString() }.toSet(), priority) +): SnarkIOReader = SnarkIOReader(reader, *types.map { it.toString() }.toTypedArray(), priority = priority) /** @@ -59,10 +62,10 @@ public class SnarkHtml : WorkspacePlugin() { override fun content(target: String): Map = when (target) { SnarkIOReader::class.dfId -> mapOf( - "html".asName() to HtmlIOFormat.snarkReader, - "markdown".asName() to MarkdownIOFormat.snarkReader, + "html".asName() to HtmlReader, + "markdown".asName() to MarkdownReader, "json".asName() to SnarkIOReader(JsonMetaFormat, ContentType.Application.Json), - "yaml".asName() to SnarkIOReader(YamlMetaFormat, "text/yaml"), + "yaml".asName() to SnarkIOReader(YamlMetaFormat, "text/yaml", "yaml"), "png".asName() to SnarkIOReader(ImageIOReader, ContentType.Image.PNG), "jpg".asName() to SnarkIOReader(ImageIOReader, ContentType.Image.JPEG), "gif".asName() to SnarkIOReader(ImageIOReader, ContentType.Image.GIF), @@ -83,9 +86,15 @@ public class SnarkHtml : WorkspacePlugin() { else -> super.content(target) } +// public val assets: TaskReference by task { +// node(Name.EMPTY, from(allData).filter { name, meta -> +// +// }) +// } + public val preprocess: TaskReference by task { - pipeFrom(dataByType()) { text, _, meta -> + pipeFrom(dataByType()) { text, _, meta -> meta[TextProcessor.TEXT_TRANSFORMATION_KEY]?.let { snark.textProcessor(it).process(text) } ?: text diff --git a/snark-html/src/jvmMain/kotlin/space/kscience/snark/html/WebPagePostprocessor.kt b/snark-html/src/jvmMain/kotlin/space/kscience/snark/html/WebPagePostprocessor.kt new file mode 100644 index 0000000..6c64ad3 --- /dev/null +++ b/snark-html/src/jvmMain/kotlin/space/kscience/snark/html/WebPagePostprocessor.kt @@ -0,0 +1,74 @@ +package space.kscience.snark.html + +import kotlinx.html.A +import kotlinx.html.FlowContent +import kotlinx.html.Tag +import kotlinx.html.TagConsumer +import space.kscience.dataforge.meta.string +import space.kscience.dataforge.names.parseAsName +import space.kscience.snark.TextProcessor + +public class WebPageTextProcessor(private val page: WebPage) : TextProcessor { + private val regex = """\$\{([\w.]*)(?>\("(.*)"\))?}""".toRegex() + + /** + * A basic [TextProcessor] that replaces `${...}` expressions in text. The following expressions are recognised: + * * `homeRef` resolves to [homeRef] + * * `resolveRef("...")` -> [WebPage.resolveRef] + * * `resolvePageRef("...")` -> [WebPage.resolvePageRef] + * * `pageMeta.get("...") -> [WebPage.pageMeta] get string method + * Otherwise return unchanged string + */ + override fun process(text: CharSequence): String = text.replace(regex) { match -> + when (match.groups[1]!!.value) { + "homeRef" -> page.homeRef + "resolveRef" -> { + val refString = match.groups[2]?.value ?: error("resolveRef requires a string (quoted) argument") + page.resolveRef(refString) + } + + "resolvePageRef" -> { + val refString = match.groups[2]?.value + ?: error("resolvePageRef requires a string (quoted) argument") + page.localisedPageRef(refString.parseAsName()) + } + + "pageMeta.get" -> { + val nameString = match.groups[2]?.value + ?: error("resolvePageRef requires a string (quoted) argument") + page.pageMeta[nameString.parseAsName()].string ?: "@null" + } + + else -> match.value + } + } + +} + + +public class WebPagePostprocessor( + public val page: WebPage, + private val consumer: TagConsumer, +) : TagConsumer by consumer { + + private val processor = WebPageTextProcessor(page) + + override fun onTagAttributeChange(tag: Tag, attribute: String, value: String?) { + if (tag is A && attribute == "href" && value != null) { + consumer.onTagAttributeChange(tag, attribute, processor.process(value)) + } else { + consumer.onTagAttributeChange(tag, attribute, value) + } + } + + override fun onTagContent(content: CharSequence) { + consumer.onTagContent(processor.process(content)) + } +} + +public inline fun FlowContent.withSnarkPage(page: WebPage, block: FlowContent.() -> Unit) { + val fc = object : FlowContent by this { + override val consumer: TagConsumer<*> = WebPagePostprocessor(page, this@withSnarkPage.consumer) + } + fc.block() +} \ No newline at end of file diff --git a/snark-html/src/jvmMain/kotlin/space/kscience/snark/html/WebPagePreprocessor.kt b/snark-html/src/jvmMain/kotlin/space/kscience/snark/html/WebPagePreprocessor.kt deleted file mode 100644 index c33e549..0000000 --- a/snark-html/src/jvmMain/kotlin/space/kscience/snark/html/WebPagePreprocessor.kt +++ /dev/null @@ -1,43 +0,0 @@ -package space.kscience.snark.html - -import space.kscience.dataforge.meta.string -import space.kscience.dataforge.names.parseAsName -import space.kscience.snark.TextProcessor - -/** - * A basic [TextProcessor] that replaces `${...}` expressions in text. The following expressions are recognised: - * * `homeRef` resolves to [homeRef] - * * `resolveRef("...")` -> [WebPage.resolveRef] - * * `resolvePageRef("...")` -> [WebPage.resolvePageRef] - * * `pageMeta.get("...") -> [WebPage.pageMeta] get string method - * Otherwise return unchanged string - */ -public class WebPagePreprocessor(public val page: WebPage) : TextProcessor { - - private val regex = """\$\{([\w.]*)(?>\("(.*)"\))?}""".toRegex() - - - override fun process(text: String): String = text.replace(regex) { match -> - when (match.groups[1]!!.value) { - "homeRef" -> page.homeRef - "resolveRef" -> { - val refString = match.groups[2]?.value ?: error("resolveRef requires a string (quoted) argument") - page.resolveRef(refString) - } - - "resolvePageRef" -> { - val refString = match.groups[2]?.value - ?: error("resolvePageRef requires a string (quoted) argument") - page.localisedPageRef(refString.parseAsName()) - } - - "pageMeta.get" -> { - val nameString = match.groups[2]?.value - ?: error("resolvePageRef requires a string (quoted) argument") - page.pageMeta[nameString.parseAsName()].string ?: "@null" - } - - else -> match.value - } - } -} \ No newline at end of file diff --git a/snark-html/src/jvmMain/kotlin/space/kscience/snark/html/htmlIoFormats.kt b/snark-html/src/jvmMain/kotlin/space/kscience/snark/html/readers.kt similarity index 58% rename from snark-html/src/jvmMain/kotlin/space/kscience/snark/html/htmlIoFormats.kt rename to snark-html/src/jvmMain/kotlin/space/kscience/snark/html/readers.kt index 771f47d..2dc1ad0 100644 --- a/snark-html/src/jvmMain/kotlin/space/kscience/snark/html/htmlIoFormats.kt +++ b/snark-html/src/jvmMain/kotlin/space/kscience/snark/html/readers.kt @@ -8,34 +8,31 @@ import kotlinx.io.readString import org.intellij.markdown.flavours.commonmark.CommonMarkFlavourDescriptor import org.intellij.markdown.html.HtmlGenerator import org.intellij.markdown.parser.MarkdownParser -import space.kscience.dataforge.io.IOReader import space.kscience.snark.SnarkIOReader import kotlin.reflect.KType import kotlin.reflect.typeOf -public object HtmlIOFormat : IOReader { - override val type: KType = typeOf() +public object HtmlReader : SnarkIOReader { + override val types: Set = setOf("html") - override fun readFrom(source: Source): HtmlFragment = HtmlFragment { page -> + override fun readFrom(source: String): HtmlFragment = HtmlFragment { div { - unsafe { +source.readString() } + unsafe { +source } } } - public val snarkReader: SnarkIOReader = SnarkIOReader(this, ContentType.Text.Html) - + override fun readFrom(source: Source): HtmlFragment = readFrom(source.readString()) + override val type: KType = typeOf() } -public object MarkdownIOFormat : IOReader { +public object MarkdownReader : SnarkIOReader { override val type: KType = typeOf() - private val markdownFlavor = CommonMarkFlavourDescriptor() - private val markdownParser = MarkdownParser(markdownFlavor) + override val types: Set = setOf("text/markdown", "md", "markdown") - override fun readFrom(source: Source): HtmlFragment = HtmlFragment { page -> - val transformedText = source.readString() - val parsedTree = markdownParser.buildMarkdownTreeFromString(transformedText) - val htmlString = HtmlGenerator(transformedText, parsedTree, markdownFlavor).generateHtml() + override fun readFrom(source: String): HtmlFragment = HtmlFragment { + val parsedTree = markdownParser.buildMarkdownTreeFromString(source) + val htmlString = HtmlGenerator(source, parsedTree, markdownFlavor).generateHtml() div { unsafe { @@ -44,6 +41,11 @@ public object MarkdownIOFormat : IOReader { } } + private val markdownFlavor = CommonMarkFlavourDescriptor() + private val markdownParser = MarkdownParser(markdownFlavor) + + override fun readFrom(source: Source): HtmlFragment = readFrom(source.readString()) + public val snarkReader: SnarkIOReader = SnarkIOReader(this, ContentType.parse("text/markdown")) }