From 4c98d62e8f94a53a1d5baaea21e136a367508d66 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Fri, 4 Dec 2020 10:19:42 +0300 Subject: [PATCH] Implement YamlMetaFormat with yaml.kt --- .../dataforge-io-yaml/build.gradle.kts | 24 ++-- .../io/yaml/FrontMatterEnvelopeFormat.kt | 2 +- .../hep/dataforge/io/yaml/YamlMetaFormat.kt | 122 ++++++++++++++++++ .../dataforge/io/yaml/YamlMetaFormatTest.kt | 0 .../hep/dataforge/io/yaml/YamlMetaFormat.kt | 56 -------- .../hep/dataforge/meta/MetaSerializer.kt | 2 +- 6 files changed, 140 insertions(+), 66 deletions(-) rename dataforge-io/dataforge-io-yaml/src/{main => jvmMain}/kotlin/hep/dataforge/io/yaml/FrontMatterEnvelopeFormat.kt (98%) create mode 100644 dataforge-io/dataforge-io-yaml/src/jvmMain/kotlin/hep/dataforge/io/yaml/YamlMetaFormat.kt rename dataforge-io/dataforge-io-yaml/src/{test => jvmTest}/kotlin/hep/dataforge/io/yaml/YamlMetaFormatTest.kt (100%) delete mode 100644 dataforge-io/dataforge-io-yaml/src/main/kotlin/hep/dataforge/io/yaml/YamlMetaFormat.kt diff --git a/dataforge-io/dataforge-io-yaml/build.gradle.kts b/dataforge-io/dataforge-io-yaml/build.gradle.kts index 720e1fba..e5b23ce7 100644 --- a/dataforge-io/dataforge-io-yaml/build.gradle.kts +++ b/dataforge-io/dataforge-io-yaml/build.gradle.kts @@ -1,16 +1,24 @@ plugins { - id("ru.mipt.npm.jvm") + id("ru.mipt.npm.mpp") } description = "YAML meta IO" -kscience { - useSerialization { - yaml() - } +repositories{ + jcenter() } -dependencies { - api(project(":dataforge-io")) - api("org.yaml:snakeyaml:1.26") +kscience { + useSerialization() +} + +kotlin { + sourceSets { + jvmMain{ + dependencies { + api(project(":dataforge-io")) + api("net.mamoe.yamlkt:yamlkt:${ru.mipt.npm.gradle.KScienceVersions.Serialization.yamlKtVersion}") + } + } + } } diff --git a/dataforge-io/dataforge-io-yaml/src/main/kotlin/hep/dataforge/io/yaml/FrontMatterEnvelopeFormat.kt b/dataforge-io/dataforge-io-yaml/src/jvmMain/kotlin/hep/dataforge/io/yaml/FrontMatterEnvelopeFormat.kt similarity index 98% rename from dataforge-io/dataforge-io-yaml/src/main/kotlin/hep/dataforge/io/yaml/FrontMatterEnvelopeFormat.kt rename to dataforge-io/dataforge-io-yaml/src/jvmMain/kotlin/hep/dataforge/io/yaml/FrontMatterEnvelopeFormat.kt index aa2537be..56a01d92 100644 --- a/dataforge-io/dataforge-io-yaml/src/main/kotlin/hep/dataforge/io/yaml/FrontMatterEnvelopeFormat.kt +++ b/dataforge-io/dataforge-io-yaml/src/jvmMain/kotlin/hep/dataforge/io/yaml/FrontMatterEnvelopeFormat.kt @@ -86,7 +86,7 @@ public class FrontMatterEnvelopeFormat( } public companion object : EnvelopeFormatFactory { - public const val SEPARATOR = "---" + public const val SEPARATOR: String = "---" private val metaTypeRegex = "---(\\w*)\\s*".toRegex() diff --git a/dataforge-io/dataforge-io-yaml/src/jvmMain/kotlin/hep/dataforge/io/yaml/YamlMetaFormat.kt b/dataforge-io/dataforge-io-yaml/src/jvmMain/kotlin/hep/dataforge/io/yaml/YamlMetaFormat.kt new file mode 100644 index 00000000..5769c5b1 --- /dev/null +++ b/dataforge-io/dataforge-io-yaml/src/jvmMain/kotlin/hep/dataforge/io/yaml/YamlMetaFormat.kt @@ -0,0 +1,122 @@ +package hep.dataforge.io.yaml + +import hep.dataforge.context.Context +import hep.dataforge.io.IOFormat.Companion.META_KEY +import hep.dataforge.io.IOFormat.Companion.NAME_KEY +import hep.dataforge.io.MetaFormat +import hep.dataforge.io.MetaFormatFactory +import hep.dataforge.meta.* +import hep.dataforge.meta.descriptors.ItemDescriptor +import hep.dataforge.meta.descriptors.NodeDescriptor +import hep.dataforge.names.NameToken +import hep.dataforge.names.withIndex +import hep.dataforge.values.ListValue +import hep.dataforge.values.Null +import hep.dataforge.values.parseValue +import kotlinx.io.Input +import kotlinx.io.Output +import kotlinx.io.text.readUtf8String +import kotlinx.io.text.writeUtf8String +import net.mamoe.yamlkt.* + +public fun Meta.toYaml(): YamlMap { + val map: Map = items.entries.associate { (key, item) -> + key.toString() to when (item) { + is MetaItem.ValueItem -> { + item.value.value + } + is MetaItem.NodeItem -> { + item.node.toYaml() + } + } + } + return YamlMap(map) +} + +private class YamlMeta(private val yamlMap: YamlMap, private val descriptor: NodeDescriptor? = null) : MetaBase() { + + private fun buildItems(): Map> { + val map = LinkedHashMap>() + + yamlMap.content.entries.forEach { (key, value) -> + val stringKey = key.toString() + val itemDescriptor = descriptor?.items?.get(stringKey) + val token = NameToken(stringKey) + when (value) { + YamlNull -> Null.asMetaItem() + is YamlLiteral -> map[token] = value.content.parseValue().asMetaItem() + is YamlMap -> map[token] = value.toMeta().asMetaItem() + is YamlList -> if (value.all { it is YamlLiteral }) { + val listValue = ListValue( + value.map { + //We already checked that all values are primitives + (it as YamlLiteral).content.parseValue() + } + ) + map[token] = MetaItem.ValueItem(listValue) + } else value.forEachIndexed { index, yamlElement -> + val indexKey = (itemDescriptor as? NodeDescriptor)?.indexKey ?: ItemDescriptor.DEFAULT_INDEX_KEY + val indexValue: String = (yamlElement as? YamlMap)?.getStringOrNull(indexKey) + ?: index.toString() //In case index is non-string, the backward transformation will be broken. + + val tokenWithIndex = token.withIndex(indexValue) + map[tokenWithIndex] = yamlElement.toMetaItem(itemDescriptor) + } + } + } + return map + } + + override val items: Map> get() = buildItems() +} + +public fun YamlElement.toMetaItem(descriptor: ItemDescriptor? = null): MetaItem<*> = when (this) { + YamlNull -> Null.asMetaItem() + is YamlLiteral -> content.parseValue().asMetaItem() + is YamlMap -> toMeta().asMetaItem() + //We can't return multiple items therefore we create top level node + is YamlList -> YamlMap(mapOf("@yamlArray" to this)).toMetaItem(descriptor) +} + +public fun YamlMap.toMeta(): Meta = YamlMeta(this) + + +/** + * Represent meta as Yaml + */ +@DFExperimental +public class YamlMetaFormat(private val meta: Meta) : MetaFormat { + private val coder = Yaml.default + + override fun writeMeta(output: Output, meta: Meta, descriptor: NodeDescriptor?) { + val yaml = meta.toYaml() + val string = coder.encodeToString(yaml) + output.writeUtf8String(string) + } + + override fun readMeta(input: Input, descriptor: NodeDescriptor?): Meta { + val yaml = coder.decodeYamlMapFromString(input.readUtf8String()) + return yaml.toMeta() + } + + override fun toMeta(): Meta = Meta { + NAME_KEY put FrontMatterEnvelopeFormat.name.toString() + META_KEY put meta + } + + public companion object : MetaFormatFactory { + override fun invoke(meta: Meta, context: Context): MetaFormat = YamlMetaFormat(meta) + + override val shortName: String = "yaml" + + override val key: Short = 0x594d //YM + + private val default = YamlMetaFormat() + + override fun writeMeta(output: Output, meta: Meta, descriptor: NodeDescriptor?): Unit = + default.writeMeta(output, meta, descriptor) + + override fun readMeta(input: kotlinx.io.Input, descriptor: NodeDescriptor?): Meta = + default.readMeta(input, descriptor) + } +} \ No newline at end of file diff --git a/dataforge-io/dataforge-io-yaml/src/test/kotlin/hep/dataforge/io/yaml/YamlMetaFormatTest.kt b/dataforge-io/dataforge-io-yaml/src/jvmTest/kotlin/hep/dataforge/io/yaml/YamlMetaFormatTest.kt similarity index 100% rename from dataforge-io/dataforge-io-yaml/src/test/kotlin/hep/dataforge/io/yaml/YamlMetaFormatTest.kt rename to dataforge-io/dataforge-io-yaml/src/jvmTest/kotlin/hep/dataforge/io/yaml/YamlMetaFormatTest.kt diff --git a/dataforge-io/dataforge-io-yaml/src/main/kotlin/hep/dataforge/io/yaml/YamlMetaFormat.kt b/dataforge-io/dataforge-io-yaml/src/main/kotlin/hep/dataforge/io/yaml/YamlMetaFormat.kt deleted file mode 100644 index 18e8af06..00000000 --- a/dataforge-io/dataforge-io-yaml/src/main/kotlin/hep/dataforge/io/yaml/YamlMetaFormat.kt +++ /dev/null @@ -1,56 +0,0 @@ -package hep.dataforge.io.yaml - -import hep.dataforge.context.Context -import hep.dataforge.io.IOFormat.Companion.META_KEY -import hep.dataforge.io.IOFormat.Companion.NAME_KEY -import hep.dataforge.io.MetaFormat -import hep.dataforge.io.MetaFormatFactory -import hep.dataforge.meta.DFExperimental -import hep.dataforge.meta.Meta -import hep.dataforge.meta.descriptors.NodeDescriptor -import hep.dataforge.meta.toMap -import hep.dataforge.meta.toMeta -import kotlinx.io.Input -import kotlinx.io.Output -import kotlinx.io.asInputStream -import kotlinx.io.text.writeUtf8String -import org.yaml.snakeyaml.Yaml - -/** - * Represent meta as Yaml - */ -@DFExperimental -public class YamlMetaFormat(private val meta: Meta) : MetaFormat { - private val yaml = Yaml() - - override fun writeMeta(output: Output, meta: Meta, descriptor: NodeDescriptor?) { - val string = yaml.dump(meta.toMap(descriptor)) - output.writeUtf8String(string) - } - - override fun readMeta(input: Input, descriptor: NodeDescriptor?): Meta { - val map: Map = yaml.load(input.asInputStream()) - return map.toMeta(descriptor) - } - - override fun toMeta(): Meta = Meta{ - NAME_KEY put FrontMatterEnvelopeFormat.name.toString() - META_KEY put meta - } - - public companion object : MetaFormatFactory { - override fun invoke(meta: Meta, context: Context): MetaFormat = YamlMetaFormat(meta) - - override val shortName: String = "yaml" - - override val key: Short = 0x594d //YM - - private val default = YamlMetaFormat() - - override fun writeMeta(output: Output, meta: Meta, descriptor: NodeDescriptor?): Unit = - default.writeMeta(output, meta, descriptor) - - override fun readMeta(input: kotlinx.io.Input, descriptor: NodeDescriptor?): Meta = - default.readMeta(input, descriptor) - } -} \ No newline at end of file diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MetaSerializer.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MetaSerializer.kt index 9bf5560d..ef7be41c 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MetaSerializer.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MetaSerializer.kt @@ -18,7 +18,7 @@ public object MetaItemSerializer : KSerializer> { @OptIn(InternalSerializationApi::class) override val descriptor: SerialDescriptor = buildClassSerialDescriptor("hep.dataforge.meta.MetaItem") { element("isNode") - element("content", buildSerialDescriptor("MetaItem.content", PolymorphicKind.SEALED)) + element("content", buildSerialDescriptor("MetaItem.content", SerialKind.CONTEXTUAL)) } override fun deserialize(decoder: Decoder): MetaItem<*> {