[WIP] refactor in progress

This commit is contained in:
Alexander Nozik 2024-01-04 19:06:09 +03:00
parent 3d44ea9a88
commit 018b52aaff
3 changed files with 72 additions and 52 deletions

View File

@ -0,0 +1,20 @@
package space.kscience.snark.html
import space.kscience.dataforge.data.Data
import space.kscience.dataforge.meta.Meta
import space.kscience.dataforge.meta.MutableMeta
import space.kscience.dataforge.meta.copy
private class MetaMaskData<T>(val origin: Data<T>, override val meta: Meta) : Data<T> by origin
/**
* A data with overriden meta. It reflects original data computed state.
*/
public fun <T> Data<T>.withMeta(newMeta: Meta): Data<T> = if (this is MetaMaskData) {
MetaMaskData(origin, newMeta)
} else {
MetaMaskData(this, newMeta)
}
public inline fun <T> Data<T>.mapMeta(block: MutableMeta.() -> Unit): Data<T> = withMeta(meta.copy(block))

View File

@ -125,7 +125,7 @@ public fun SiteContextWithData.page(
): Unit = page(route, siteData, pageMeta, content)
@SnarkBuilder
public suspend fun SiteContextWithData.route(
public fun SiteContextWithData.route(
route: String,
data: DataSet<*> = siteData,
siteMeta: Meta = Meta.EMPTY,
@ -133,7 +133,7 @@ public suspend fun SiteContextWithData.route(
): Unit = route(route.parseAsName(), data, siteMeta,content)
@SnarkBuilder
public suspend fun SiteContextWithData.site(
public fun SiteContextWithData.site(
route: String,
data: DataSet<*> = siteData,
siteMeta: Meta = Meta.EMPTY,

View File

@ -5,21 +5,20 @@ package space.kscience.snark.html
import io.ktor.http.ContentType
import kotlinx.io.readByteArray
import space.kscience.dataforge.actions.Action
import space.kscience.dataforge.context.*
import space.kscience.dataforge.context.Context
import space.kscience.dataforge.context.PluginFactory
import space.kscience.dataforge.context.PluginTag
import space.kscience.dataforge.data.*
import space.kscience.dataforge.io.IOPlugin
import space.kscience.dataforge.io.IOReader
import space.kscience.dataforge.io.JsonMetaFormat
import space.kscience.dataforge.io.*
import space.kscience.dataforge.io.yaml.YamlMetaFormat
import space.kscience.dataforge.io.yaml.YamlPlugin
import space.kscience.dataforge.meta.Meta
import space.kscience.dataforge.meta.copy
import space.kscience.dataforge.meta.get
import space.kscience.dataforge.meta.string
import space.kscience.dataforge.misc.DFExperimental
import space.kscience.dataforge.names.Name
import space.kscience.dataforge.names.NameToken
import space.kscience.dataforge.names.asName
import space.kscience.dataforge.names.replaceLast
import space.kscience.dataforge.provider.dfId
import space.kscience.dataforge.workspace.*
import space.kscience.snark.Snark
@ -28,6 +27,7 @@ import space.kscience.snark.TextProcessor
import java.net.URLConnection
import kotlin.io.path.Path
import kotlin.io.path.extension
import kotlin.reflect.typeOf
public fun <T : Any, R : Any> DataSet<T>.transform(action: Action<T, R>, meta: Meta = Meta.EMPTY): DataSet<R> =
@ -53,71 +53,71 @@ public class SnarkHtml : WorkspacePlugin() {
"markdown".asName() to MarkdownReader,
"json".asName() to SnarkReader(JsonMetaFormat, ContentType.Application.Json.toString()),
"yaml".asName() to SnarkReader(YamlMetaFormat, "text/yaml", "yaml"),
// "png".asName() to SnarkReader(ImageIOReader, ContentType.Image.PNG.toString()),
// "jpg".asName() to SnarkReader(ImageIOReader, ContentType.Image.JPEG.toString()),
// "gif".asName() to SnarkReader(ImageIOReader, ContentType.Image.GIF.toString()),
// "svg".asName() to SnarkReader(IOReader.binary, ContentType.Image.SVG.toString(), "svg"),
// "raw".asName() to SnarkReader(
// IOReader.binary,
// "css",
// "js",
// "javascript",
// "scss",
// "woff",
// "woff2",
// "ttf",
// "eot"
// )
)
else -> super.content(target)
}
public val read: TaskReference<String> by task<String>{
private fun getContentType(name: Name, meta: Meta): String = meta[CONTENT_TYPE_KEY].string ?: run {
val filePath = meta[FileData.FILE_PATH_KEY]?.string ?: name.toString()
URLConnection.guessContentTypeFromName(filePath) ?: Path(filePath).extension
}
public val parse: TaskReference<Any> by task<Any> {
from(read).forEach { (dataName, data) ->
//remove extensions for data files
val filePath = meta[FileData.FILE_PATH_KEY]?.string ?: dataName.toString()
val fileType = URLConnection.guessContentTypeFromName(filePath) ?: Path(filePath).extension
from(allData).forEach { (dataName, data) ->
val contentType = getContentType(dataName, data.meta)
val parser = snark.readers.values.filter { parser ->
fileType in parser.types
contentType in parser.types
}.maxByOrNull {
it.priority
} ?: run {
logger.debug { "The parser is not found for file $filePath with meta $meta. Passing data without parsing" }
data(dataName, data)
return@forEach
}
val newName = dataName.replaceLast {
NameToken(it.body.substringBeforeLast("."), it.index)
}
val preprocessor = meta[TextProcessor.TEXT_TRANSFORMATION_KEY]?.let{snark.preprocessor(it)}
} ?: return@forEach //ignore data for which parser is not found
data(newName, data.map { string: String ->
val preprocessed = preprocessor?.process(string) ?: string
parser.readFrom(preprocessed)
})
val preprocessor = meta[TextProcessor.TEXT_TRANSFORMATION_KEY]?.let { snark.preprocessor(it) }
val newMeta = data.meta.copy {
CONTENT_TYPE_KEY put contentType
}
when (data.type) {
typeOf<String>() -> {
data(dataName, data.map { content ->
val string = content as String
val preprocessed = preprocessor?.process(string) ?: string
parser.readFrom(preprocessed)
})
}
typeOf<Binary>() -> {
data(dataName, data.map(meta = newMeta) { content ->
val binary = content as Binary
if (preprocessor == null) {
parser.readFrom(binary)
} else {
//TODO provide encoding
val string = binary.toByteArray().decodeToString()
parser.readFrom(preprocessor.process(string))
}
})
}
// bypass for non textual-data
else -> data(dataName, data.withMeta(newMeta))
}
}
}
// public val site by task<Any> {
//
// }
// public val textTransformationAction: Action<String, String> = Action.map<String, String> {
// val transformations = actionMeta.getIndexed("transformation").entries.sortedBy {
// it.key?.toIntOrNull() ?: 0
// }.map { it.value }
// }
public val site: TaskReference<Any> by task<Any> {
fill(from(allData))
fill(from(parse))
}
public companion object : PluginFactory<SnarkHtml> {
override val tag: PluginTag = PluginTag("snark.html")
public val CONTENT_TYPE_KEY: Name = "contentType".asName()
override fun build(context: Context, meta: Meta): SnarkHtml = SnarkHtml()
private val byteArrayIOReader = IOReader {