diff --git a/build.gradle.kts b/build.gradle.kts index 82d402e2..0d4cc1fb 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -1,12 +1,12 @@ plugins { - val toolsVersion = "0.3.1" + val toolsVersion = "0.4.0" id("scientifik.mpp") version toolsVersion apply false id("scientifik.jvm") version toolsVersion apply false id("scientifik.publish") version toolsVersion apply false } -val dataforgeVersion by extra("0.1.5-dev-9") +val dataforgeVersion by extra("0.1.5-dev-10") val bintrayRepo by extra("dataforge") val githubProject by extra("dataforge-core") diff --git a/dataforge-context/build.gradle.kts b/dataforge-context/build.gradle.kts index 896e7b89..104f4037 100644 --- a/dataforge-context/build.gradle.kts +++ b/dataforge-context/build.gradle.kts @@ -12,20 +12,20 @@ kotlin { dependencies { api(project(":dataforge-meta")) api(kotlin("reflect")) - api("io.github.microutils:kotlin-logging-common:1.7.2") + api("io.github.microutils:kotlin-logging-common:1.7.8") api("org.jetbrains.kotlinx:kotlinx-coroutines-core-common:$coroutinesVersion") } } val jvmMain by getting { dependencies { - api("io.github.microutils:kotlin-logging:1.7.2") + api("io.github.microutils:kotlin-logging:1.7.8") api("ch.qos.logback:logback-classic:1.2.3") api("org.jetbrains.kotlinx:kotlinx-coroutines-core:$coroutinesVersion") } } val jsMain by getting { dependencies { - api("io.github.microutils:kotlin-logging-js:1.7.2") + api("io.github.microutils:kotlin-logging-js:1.7.8") api("org.jetbrains.kotlinx:kotlinx-coroutines-core-js:$coroutinesVersion") } } diff --git a/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/Context.kt b/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/Context.kt index ffc5b197..b3adcbfd 100644 --- a/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/Context.kt +++ b/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/Context.kt @@ -103,7 +103,7 @@ open class Context( plugins.forEach { it.detach() } } - override fun toMeta(): Meta = buildMeta { + override fun toMeta(): Meta = Meta { "parent" to parent?.name "properties" put properties.seal() "plugins" put plugins.map { it.toMeta() } diff --git a/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/ContextBuilder.kt b/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/ContextBuilder.kt index 0aed9b7f..1f267c37 100644 --- a/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/ContextBuilder.kt +++ b/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/ContextBuilder.kt @@ -1,6 +1,7 @@ package hep.dataforge.context import hep.dataforge.meta.DFBuilder +import hep.dataforge.meta.Meta import hep.dataforge.meta.MetaBuilder import hep.dataforge.meta.buildMeta import hep.dataforge.names.toName @@ -22,11 +23,11 @@ class ContextBuilder(var name: String = "@anonymous", val parent: Context = Glob } fun plugin(tag: PluginTag, action: MetaBuilder.() -> Unit = {}) { - plugins.add(PluginRepository.fetch(tag, buildMeta(action))) + plugins.add(PluginRepository.fetch(tag, Meta(action))) } fun plugin(builder: PluginFactory<*>, action: MetaBuilder.() -> Unit = {}) { - plugins.add(builder.invoke(buildMeta(action))) + plugins.add(builder.invoke(Meta(action))) } fun plugin(name: String, group: String = "", version: String = "", action: MetaBuilder.() -> Unit = {}) { diff --git a/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/Plugin.kt b/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/Plugin.kt index 90d669e0..6a937962 100644 --- a/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/Plugin.kt +++ b/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/Plugin.kt @@ -65,7 +65,7 @@ interface Plugin : Named, ContextAware, Provider, MetaRepr { */ fun detach() - override fun toMeta(): Meta = buildMeta { + override fun toMeta(): Meta = Meta { "context" put context.name.toString() "type" to this::class.simpleName "tag" put tag diff --git a/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/PluginManager.kt b/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/PluginManager.kt index afac0392..5f35d876 100644 --- a/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/PluginManager.kt +++ b/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/PluginManager.kt @@ -102,7 +102,7 @@ class PluginManager(override val context: Context) : ContextAware, Iterable load(factory: PluginFactory, metaBuilder: MetaBuilder.() -> Unit): T = - load(factory, buildMeta(metaBuilder)) + load(factory, Meta(metaBuilder)) /** * Remove a plugin from [PluginManager] @@ -134,7 +134,7 @@ class PluginManager(override val context: Context) : ContextAware, Iterable, recursive: Boolean = true, metaBuilder: MetaBuilder.() -> Unit - ): T = fetch(factory, recursive, buildMeta(metaBuilder)) + ): T = fetch(factory, recursive, Meta(metaBuilder)) override fun iterator(): Iterator = plugins.iterator() diff --git a/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/PluginTag.kt b/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/PluginTag.kt index 390de7bc..f02308fd 100644 --- a/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/PluginTag.kt +++ b/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/PluginTag.kt @@ -36,7 +36,7 @@ data class PluginTag( override fun toString(): String = listOf(group, name, version).joinToString(separator = ":") - override fun toMeta(): Meta = buildMeta { + override fun toMeta(): Meta = Meta { "name" put name "group" put group "version" put version diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/Data.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/Data.kt index ea25e070..4d787b69 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/Data.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/Data.kt @@ -19,7 +19,7 @@ interface Data : Goal, MetaRepr{ */ val meta: Meta - override fun toMeta(): Meta = buildMeta { + override fun toMeta(): Meta = Meta { "type" put (type.simpleName?:"undefined") if(!meta.isEmpty()) { "meta" put meta diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataNode.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataNode.kt index 764aea62..ee8d9c4d 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataNode.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataNode.kt @@ -47,7 +47,7 @@ interface DataNode : MetaRepr { val meta: Meta - override fun toMeta(): Meta = buildMeta { + override fun toMeta(): Meta = Meta { "type" put (type.simpleName ?: "undefined") "items" put { this@DataNode.items.forEach { @@ -255,11 +255,11 @@ fun DataTreeBuilder.static(name: Name, data: T, meta: Meta = EmptyM } fun DataTreeBuilder.static(name: Name, data: T, block: MetaBuilder.() -> Unit = {}) { - this[name] = Data.static(data, buildMeta(block)) + this[name] = Data.static(data, Meta(block)) } fun DataTreeBuilder.static(name: String, data: T, block: MetaBuilder.() -> Unit = {}) { - this[name.toName()] = Data.static(data, buildMeta(block)) + this[name.toName()] = Data.static(data, Meta(block)) } fun DataTreeBuilder.node(name: Name, node: DataNode) { diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/JsonMetaFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/JsonMetaFormat.kt index 8ec244a5..1101b35f 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/JsonMetaFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/JsonMetaFormat.kt @@ -9,6 +9,8 @@ import hep.dataforge.meta.descriptors.ValueDescriptor import hep.dataforge.meta.Meta import hep.dataforge.meta.MetaBase import hep.dataforge.meta.MetaItem +import hep.dataforge.meta.serialization.toJson +import hep.dataforge.meta.serialization.toMeta import hep.dataforge.names.NameToken import hep.dataforge.names.toName import hep.dataforge.values.* @@ -16,6 +18,7 @@ import kotlinx.io.Input import kotlinx.io.Output import kotlinx.io.text.readUtf8String import kotlinx.io.text.writeUtf8String +import kotlinx.serialization.UnstableDefault import kotlinx.serialization.json.* @@ -23,8 +26,8 @@ import kotlin.collections.component1 import kotlin.collections.component2 import kotlin.collections.set - -class JsonMetaFormat(private val json: Json = Json.indented) : MetaFormat { +@OptIn(UnstableDefault::class) +class JsonMetaFormat(private val json: Json = DEFAULT_JSON) : MetaFormat { override fun Output.writeMeta(meta: Meta, descriptor: NodeDescriptor?) { val jsonObject = meta.toJson(descriptor) @@ -38,6 +41,8 @@ class JsonMetaFormat(private val json: Json = Json.indented) : MetaFormat { } companion object : MetaFormatFactory { + val DEFAULT_JSON = Json{prettyPrint = true} + override fun invoke(meta: Meta, context: Context): MetaFormat = default override val shortName = "json" @@ -52,125 +57,3 @@ class JsonMetaFormat(private val json: Json = Json.indented) : MetaFormat { default.run { readMeta(descriptor) } } } - -/** - * @param descriptor reserved for custom serialization in future - */ -fun Value.toJson(descriptor: ValueDescriptor? = null): JsonElement { - return if (isList()) { - JsonArray(list.map { it.toJson() }) - } else { - when (type) { - ValueType.NUMBER -> JsonPrimitive(number) - ValueType.STRING -> JsonPrimitive(string) - ValueType.BOOLEAN -> JsonPrimitive(boolean) - ValueType.NULL -> JsonNull - } - } -} - -//Use these methods to customize JSON key mapping -private fun NameToken.toJsonKey(descriptor: ItemDescriptor?) = toString() - -//private fun NodeDescriptor?.getDescriptor(key: String) = this?.items?.get(key) - -fun Meta.toJson(descriptor: NodeDescriptor? = null): JsonObject { - - //TODO search for same name siblings and arrange them into arrays - val map = this.items.entries.associate { (name, item) -> - val itemDescriptor = descriptor?.items?.get(name.body) - val key = name.toJsonKey(itemDescriptor) - val value = when (item) { - is MetaItem.ValueItem -> { - item.value.toJson(itemDescriptor as? ValueDescriptor) - } - is MetaItem.NodeItem -> { - item.node.toJson(itemDescriptor as? NodeDescriptor) - } - } - key to value - } - return JsonObject(map) -} - -fun JsonElement.toMeta(descriptor: NodeDescriptor? = null): Meta { - return when (val item = toMetaItem(descriptor)) { - is MetaItem.NodeItem<*> -> item.node - is MetaItem.ValueItem -> item.value.toMeta() - } -} - -fun JsonPrimitive.toValue(descriptor: ValueDescriptor?): Value { - return when (this) { - JsonNull -> Null - else -> this.content.parseValue() // Optimize number and boolean parsing - } -} - -fun JsonElement.toMetaItem(descriptor: ItemDescriptor? = null): MetaItem = when (this) { - is JsonPrimitive -> { - val value = this.toValue(descriptor as? ValueDescriptor) - MetaItem.ValueItem(value) - } - is JsonObject -> { - val meta = JsonMeta(this, descriptor as? NodeDescriptor) - MetaItem.NodeItem(meta) - } - is JsonArray -> { - if (this.all { it is JsonPrimitive }) { - val value = if (isEmpty()) { - Null - } else { - ListValue( - map { - //We already checked that all values are primitives - (it as JsonPrimitive).toValue(descriptor as? ValueDescriptor) - } - ) - } - MetaItem.ValueItem(value) - } else { - json { - "@value" to this@toMetaItem - }.toMetaItem(descriptor) - } - } -} - -class JsonMeta(val json: JsonObject, val descriptor: NodeDescriptor? = null) : MetaBase() { - - @Suppress("UNCHECKED_CAST") - private operator fun MutableMap>.set(key: String, value: JsonElement): Unit { - val itemDescriptor = descriptor?.items?.get(key) - return when (value) { - is JsonPrimitive -> { - this[key] = MetaItem.ValueItem(value.toValue(itemDescriptor as? ValueDescriptor)) as MetaItem - } - is JsonObject -> { - this[key] = MetaItem.NodeItem(JsonMeta(value, itemDescriptor as? NodeDescriptor)) - } - is JsonArray -> { - when { - value.all { it is JsonPrimitive } -> { - val listValue = ListValue( - value.map { - //We already checked that all values are primitives - (it as JsonPrimitive).toValue(itemDescriptor as? ValueDescriptor) - } - ) - this[key] = MetaItem.ValueItem(listValue) as MetaItem - } - else -> value.forEachIndexed { index, jsonElement -> - this["$key[$index]"] = jsonElement.toMetaItem(itemDescriptor) - } - } - } - } - } - - override val items: Map> by lazy { - val map = HashMap>() - json.forEach { (key, value) -> map[key] = value } - map.mapKeys { it.key.toName().first()!! } - } -} \ No newline at end of file diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaglessEnvelopeFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaglessEnvelopeFormat.kt index 0ed5214b..ebeedd6d 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaglessEnvelopeFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaglessEnvelopeFormat.kt @@ -119,7 +119,7 @@ class TaglessEnvelopeFormat( var line: String do { line = readUtf8Line()// ?: error("Input does not contain tagless envelope header") - offset += line.toUtf8Bytes().size.toUInt() + offset += line.encodeToByteArray().size.toUInt() } while (!line.startsWith(TAGLESS_ENVELOPE_HEADER)) val properties = HashMap() @@ -133,7 +133,7 @@ class TaglessEnvelopeFormat( } try { line = readUtf8Line() - offset += line.toUtf8Bytes().size.toUInt() + offset += line.encodeToByteArray().size.toUInt() } catch (ex: EOFException) { return PartialEnvelope(Meta.EMPTY, offset.toUInt(), 0.toULong()) } @@ -156,7 +156,7 @@ class TaglessEnvelopeFormat( do { line = readUtf8Line() ?: return PartialEnvelope(Meta.EMPTY, offset.toUInt(), 0.toULong()) - offset += line.toUtf8Bytes().size.toUInt() + offset += line.encodeToByteArray().size.toUInt() //returning an Envelope without data if end of input is reached } while (!line.startsWith(dataStart)) diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/serialization/MetaSerializer.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/serialization/MetaSerializer.kt deleted file mode 100644 index ef989869..00000000 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/serialization/MetaSerializer.kt +++ /dev/null @@ -1,142 +0,0 @@ -package hep.dataforge.io.serialization - -import hep.dataforge.io.toJson -import hep.dataforge.io.toMeta -import hep.dataforge.meta.* -import hep.dataforge.names.NameToken -import hep.dataforge.values.* -import kotlinx.serialization.* -import kotlinx.serialization.internal.* -import kotlinx.serialization.json.JsonInput -import kotlinx.serialization.json.JsonObjectSerializer -import kotlinx.serialization.json.JsonOutput - - -@Serializer(Value::class) -@UseExperimental(InternalSerializationApi::class) -object ValueSerializer : KSerializer { - private val valueTypeSerializer = EnumSerializer(ValueType::class) - private val listSerializer by lazy { ArrayListSerializer(ValueSerializer) } - - override val descriptor: SerialDescriptor = descriptor("Value") { - boolean("isList") - enum("valueType") - element("value", null) - } - - private fun Decoder.decodeValue(): Value { - return when (decode(valueTypeSerializer)) { - ValueType.NULL -> Null - ValueType.NUMBER -> decodeDouble().asValue() //TODO differentiate? - ValueType.BOOLEAN -> decodeBoolean().asValue() - ValueType.STRING -> decodeString().asValue() - else -> decodeString().parseValue() - } - } - - - override fun deserialize(decoder: Decoder): Value { - val isList = decoder.decodeBoolean() - return if (isList) { - listSerializer.deserialize(decoder).asValue() - } else { - decoder.decodeValue() - } - } - - private fun Encoder.encodeValue(value: Value) { - encode(valueTypeSerializer, value.type) - when (value.type) { - ValueType.NULL -> { - // do nothing - } - ValueType.NUMBER -> encodeDouble(value.double) - ValueType.BOOLEAN -> encodeBoolean(value.boolean) - ValueType.STRING -> encodeString(value.string) - else -> encodeString(value.string) - } - } - - override fun serialize(encoder: Encoder, obj: Value) { - encoder.encodeBoolean(obj.isList()) - if (obj.isList()) { - listSerializer.serialize(encoder, obj.list) - } else { - encoder.encodeValue(obj) - } - } -} - -@Serializer(MetaItem::class) -object MetaItemSerializer : KSerializer> { - override val descriptor: SerialDescriptor = descriptor("MetaItem") { - boolean("isNode") - element("value", null) - } - - - override fun deserialize(decoder: Decoder): MetaItem<*> { - val isNode = decoder.decodeBoolean() - return if (isNode) { - MetaItem.NodeItem(decoder.decode(MetaSerializer)) - } else { - MetaItem.ValueItem(decoder.decode(ValueSerializer)) - } - } - - override fun serialize(encoder: Encoder, obj: MetaItem<*>) { - encoder.encodeBoolean(obj is MetaItem.NodeItem) - when (obj) { - is MetaItem.NodeItem -> MetaSerializer.serialize(encoder, obj.node) - is MetaItem.ValueItem -> ValueSerializer.serialize(encoder, obj.value) - } - } -} - -private class DeserializedMeta(override val items: Map>) : MetaBase() - -/** - * Serialized for meta - */ -@Serializer(Meta::class) -object MetaSerializer : KSerializer { - private val mapSerializer = HashMapSerializer( - StringSerializer, - MetaItemSerializer - ) - - override val descriptor: SerialDescriptor = NamedMapClassDescriptor( - "hep.dataforge.meta.Meta", - StringSerializer.descriptor, - MetaItemSerializer.descriptor - ) - - override fun deserialize(decoder: Decoder): Meta { - return if (decoder is JsonInput) { - JsonObjectSerializer.deserialize(decoder).toMeta() - } else { - DeserializedMeta(mapSerializer.deserialize(decoder).mapKeys { NameToken(it.key) }) - } - } - - override fun serialize(encoder: Encoder, obj: Meta) { - if (encoder is JsonOutput) { - JsonObjectSerializer.serialize(encoder, obj.toJson()) - } else { - mapSerializer.serialize(encoder, obj.items.mapKeys { it.key.toString() }) - } - } -} - -@Serializer(Config::class) -object ConfigSerializer : KSerializer { - override val descriptor: SerialDescriptor = MetaSerializer.descriptor - - override fun deserialize(decoder: Decoder): Config { - return MetaSerializer.deserialize(decoder).asConfig() - } - - override fun serialize(encoder: Encoder, obj: Config) { - MetaSerializer.serialize(encoder, obj) - } -} \ No newline at end of file diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/serialization/nameSerializers.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/serialization/nameSerializers.kt deleted file mode 100644 index c12a6c19..00000000 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/serialization/nameSerializers.kt +++ /dev/null @@ -1,33 +0,0 @@ -package hep.dataforge.io.serialization - -import hep.dataforge.names.Name -import hep.dataforge.names.NameToken -import hep.dataforge.names.toName -import kotlinx.serialization.* -import kotlinx.serialization.internal.StringDescriptor - -@Serializer(Name::class) -object NameSerializer : KSerializer { - override val descriptor: SerialDescriptor = StringDescriptor.withName("Name") - - override fun deserialize(decoder: Decoder): Name { - return decoder.decodeString().toName() - } - - override fun serialize(encoder: Encoder, obj: Name) { - encoder.encodeString(obj.toString()) - } -} - -@Serializer(NameToken::class) -object NameTokenSerializer : KSerializer { - override val descriptor: SerialDescriptor = StringDescriptor.withName("NameToken") - - override fun deserialize(decoder: Decoder): NameToken { - return decoder.decodeString().toName().first()!! - } - - override fun serialize(encoder: Encoder, obj: NameToken) { - encoder.encodeString(obj.toString()) - } -} \ No newline at end of file diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/serialization/serializationUtils.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/serialization/serializationUtils.kt deleted file mode 100644 index 18e28423..00000000 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/serialization/serializationUtils.kt +++ /dev/null @@ -1,93 +0,0 @@ -package hep.dataforge.io.serialization - -import hep.dataforge.meta.DFExperimental -import kotlinx.serialization.* -import kotlinx.serialization.internal.* - -/** - * A convenience builder for serial descriptors - */ -inline class SerialDescriptorBuilder(private val impl: SerialClassDescImpl) { - fun element( - name: String, - descriptor: SerialDescriptor?, - isOptional: Boolean = false, - vararg annotations: Annotation - ) { - impl.addElement(name, isOptional) - descriptor?.let { impl.pushDescriptor(descriptor) } - annotations.forEach { - impl.pushAnnotation(it) - } - } - - fun element( - name: String, - isOptional: Boolean = false, - vararg annotations: Annotation, - block: SerialDescriptorBuilder.() -> Unit - ) { - impl.addElement(name, isOptional) - impl.pushDescriptor(SerialDescriptorBuilder(SerialClassDescImpl(name)).apply(block).build()) - annotations.forEach { - impl.pushAnnotation(it) - } - } - - fun boolean(name: String, isOptional: Boolean = false, vararg annotations: Annotation) = - element(name, BooleanDescriptor, isOptional, *annotations) - - fun string(name: String, isOptional: Boolean = false, vararg annotations: Annotation) = - element(name, StringDescriptor, isOptional, *annotations) - - fun int(name: String, isOptional: Boolean = false, vararg annotations: Annotation) = - element(name, IntDescriptor, isOptional, *annotations) - - fun double(name: String, isOptional: Boolean = false, vararg annotations: Annotation) = - element(name, DoubleDescriptor, isOptional, *annotations) - - fun float(name: String, isOptional: Boolean = false, vararg annotations: Annotation) = - element(name, FloatDescriptor, isOptional, *annotations) - - fun long(name: String, isOptional: Boolean = false, vararg annotations: Annotation) = - element(name, LongDescriptor, isOptional, *annotations) - - fun doubleArray(name: String, isOptional: Boolean = false, vararg annotations: Annotation) = - element(name, DoubleArraySerializer.descriptor, isOptional, *annotations) - - @UseExperimental(InternalSerializationApi::class) - inline fun > enum(name: String, isOptional: Boolean = false, vararg annotations: Annotation) = - element(name, EnumSerializer(E::class).descriptor, isOptional, *annotations) - - fun classAnnotation(a: Annotation) = impl.pushClassAnnotation(a) - - fun build(): SerialDescriptor = impl -} - -inline fun KSerializer.descriptor( - name: String, - block: SerialDescriptorBuilder.() -> Unit -): SerialDescriptor = - SerialDescriptorBuilder(SerialClassDescImpl(name)).apply(block).build() - -@DFExperimental -inline fun Decoder.decodeStructure( - desc: SerialDescriptor, - vararg typeParams: KSerializer<*> = emptyArray(), - crossinline block: CompositeDecoder.() -> R -): R { - val decoder = beginStructure(desc, *typeParams) - val res = decoder.block() - decoder.endStructure(desc) - return res -} - -inline fun Encoder.encodeStructure( - desc: SerialDescriptor, - vararg typeParams: KSerializer<*> = emptyArray(), - block: CompositeEncoder.() -> Unit -) { - val encoder = beginStructure(desc, *typeParams) - encoder.block() - encoder.endStructure(desc) -} \ No newline at end of file diff --git a/dataforge-meta/build.gradle.kts b/dataforge-meta/build.gradle.kts index 6f2a5160..f19a39fc 100644 --- a/dataforge-meta/build.gradle.kts +++ b/dataforge-meta/build.gradle.kts @@ -1,5 +1,9 @@ +import scientifik.useSerialization + plugins { id("scientifik.mpp") } +useSerialization() + description = "Meta definition and basic operations on meta" \ No newline at end of file diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Laminate.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Laminate.kt index beda1732..7dec0790 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Laminate.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Laminate.kt @@ -64,7 +64,7 @@ class Laminate(layers: List) : MetaBase() { } else -> map { when (it) { - is MetaItem.ValueItem -> MetaItem.NodeItem(buildMeta { Meta.VALUE_KEY put it.value }) + is MetaItem.ValueItem -> MetaItem.NodeItem(Meta { Meta.VALUE_KEY put it.value }) is MetaItem.NodeItem -> it } }.merge() diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Meta.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Meta.kt index 93e416de..b37188b4 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Meta.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Meta.kt @@ -3,11 +3,13 @@ package hep.dataforge.meta import hep.dataforge.meta.Meta.Companion.VALUE_KEY import hep.dataforge.meta.MetaItem.NodeItem import hep.dataforge.meta.MetaItem.ValueItem +import hep.dataforge.meta.serialization.toJson import hep.dataforge.names.* import hep.dataforge.values.EnumValue import hep.dataforge.values.Null import hep.dataforge.values.Value import hep.dataforge.values.boolean +import kotlinx.serialization.Serializable /** @@ -15,11 +17,14 @@ import hep.dataforge.values.boolean * * a [ValueItem] (leaf) * * a [NodeItem] (node) */ +@Serializable sealed class MetaItem { + @Serializable data class ValueItem(val value: Value) : MetaItem() { override fun toString(): String = value.toString() } + @Serializable data class NodeItem(val node: M) : MetaItem() { override fun toString(): String = node.toString() } @@ -161,7 +166,7 @@ abstract class MetaBase : Meta { override fun hashCode(): Int = items.hashCode() - override fun toString(): String = items.toString() + override fun toString(): String = toJson().toString() } /** diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MetaBuilder.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MetaBuilder.kt index b36b3316..da1bafad 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MetaBuilder.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MetaBuilder.kt @@ -152,9 +152,4 @@ fun buildMeta(builder: MetaBuilder.() -> Unit): MetaBuilder = MetaBuilder().appl * Build a [MetaBuilder] using given transformation */ @Suppress("FunctionName") -fun Meta(builder: MetaBuilder.() -> Unit): MetaBuilder = MetaBuilder().apply(builder) - -/** - * Build meta using given source meta as a base - */ -fun buildMeta(source: Meta, builder: MetaBuilder.() -> Unit): MetaBuilder = source.builder().apply(builder) \ No newline at end of file +fun Meta(builder: MetaBuilder.() -> Unit): MetaBuilder = MetaBuilder().apply(builder) \ No newline at end of file diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/annotations.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/annotations.kt index 865ef800..2085bc5d 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/annotations.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/annotations.kt @@ -6,6 +6,6 @@ package hep.dataforge.meta @DslMarker annotation class DFBuilder -@Experimental(level = Experimental.Level.WARNING) +@RequiresOptIn(level = RequiresOptIn.Level.WARNING) @Retention(AnnotationRetention.BINARY) annotation class DFExperimental \ No newline at end of file diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/descriptors/ItemDescriptor.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/descriptors/ItemDescriptor.kt index 9c5cf15f..4ff5de11 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/descriptors/ItemDescriptor.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/descriptors/ItemDescriptor.kt @@ -255,10 +255,10 @@ class ValueDescriptor : ItemDescriptor() { } companion object : SchemeSpec(::ValueDescriptor) { - inline fun > enum(name: String) = ValueDescriptor { - type(ValueType.STRING) - this.allowedValues = enumValues().map { Value.of(it.name) } - } +// inline fun > enum(name: String) = ValueDescriptor { +// type(ValueType.STRING) +// this.allowedValues = enumValues().map { Value.of(it.name) } +// } // /** // * Build a value descriptor from annotation diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/serialization/JsonMeta.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/serialization/JsonMeta.kt new file mode 100644 index 00000000..78357c6d --- /dev/null +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/serialization/JsonMeta.kt @@ -0,0 +1,135 @@ +package hep.dataforge.meta.serialization + +import hep.dataforge.meta.Meta +import hep.dataforge.meta.MetaBase +import hep.dataforge.meta.MetaItem +import hep.dataforge.meta.descriptors.ItemDescriptor +import hep.dataforge.meta.descriptors.NodeDescriptor +import hep.dataforge.meta.descriptors.ValueDescriptor +import hep.dataforge.names.NameToken +import hep.dataforge.names.toName +import hep.dataforge.values.* +import kotlinx.serialization.json.* + + +/** + * @param descriptor reserved for custom serialization in future + */ +fun Value.toJson(descriptor: ValueDescriptor? = null): JsonElement { + return if (isList()) { + JsonArray(list.map { it.toJson() }) + } else { + when (type) { + ValueType.NUMBER -> JsonPrimitive(number) + ValueType.STRING -> JsonPrimitive(string) + ValueType.BOOLEAN -> JsonPrimitive(boolean) + ValueType.NULL -> JsonNull + } + } +} + +//Use these methods to customize JSON key mapping +private fun NameToken.toJsonKey(descriptor: ItemDescriptor?) = toString() + +//private fun NodeDescriptor?.getDescriptor(key: String) = this?.items?.get(key) + +fun Meta.toJson(descriptor: NodeDescriptor? = null): JsonObject { + + //TODO search for same name siblings and arrange them into arrays + val map = this.items.entries.associate { (name, item) -> + val itemDescriptor = descriptor?.items?.get(name.body) + val key = name.toJsonKey(itemDescriptor) + val value = when (item) { + is MetaItem.ValueItem -> { + item.value.toJson(itemDescriptor as? ValueDescriptor) + } + is MetaItem.NodeItem -> { + item.node.toJson(itemDescriptor as? NodeDescriptor) + } + } + key to value + } + return JsonObject(map) +} + +fun JsonElement.toMeta(descriptor: NodeDescriptor? = null): Meta { + return when (val item = toMetaItem(descriptor)) { + is MetaItem.NodeItem<*> -> item.node + is MetaItem.ValueItem -> item.value.toMeta() + } +} + +fun JsonPrimitive.toValue(descriptor: ValueDescriptor?): Value { + return when (this) { + JsonNull -> Null + else -> this.content.parseValue() // Optimize number and boolean parsing + } +} + +fun JsonElement.toMetaItem(descriptor: ItemDescriptor? = null): MetaItem = when (this) { + is JsonPrimitive -> { + val value = this.toValue(descriptor as? ValueDescriptor) + MetaItem.ValueItem(value) + } + is JsonObject -> { + val meta = JsonMeta(this, descriptor as? NodeDescriptor) + MetaItem.NodeItem(meta) + } + is JsonArray -> { + if (this.all { it is JsonPrimitive }) { + val value = if (isEmpty()) { + Null + } else { + ListValue( + map { + //We already checked that all values are primitives + (it as JsonPrimitive).toValue(descriptor as? ValueDescriptor) + } + ) + } + MetaItem.ValueItem(value) + } else { + json { + "@value" to this@toMetaItem + }.toMetaItem(descriptor) + } + } +} + +class JsonMeta(val json: JsonObject, val descriptor: NodeDescriptor? = null) : MetaBase() { + + @Suppress("UNCHECKED_CAST") + private operator fun MutableMap>.set(key: String, value: JsonElement): Unit { + val itemDescriptor = descriptor?.items?.get(key) + return when (value) { + is JsonPrimitive -> { + this[key] = MetaItem.ValueItem(value.toValue(itemDescriptor as? ValueDescriptor)) as MetaItem + } + is JsonObject -> { + this[key] = MetaItem.NodeItem(JsonMeta(value, itemDescriptor as? NodeDescriptor)) + } + is JsonArray -> { + when { + value.all { it is JsonPrimitive } -> { + val listValue = ListValue( + value.map { + //We already checked that all values are primitives + (it as JsonPrimitive).toValue(itemDescriptor as? ValueDescriptor) + } + ) + this[key] = MetaItem.ValueItem(listValue) as MetaItem + } + else -> value.forEachIndexed { index, jsonElement -> + this["$key[$index]"] = jsonElement.toMetaItem(itemDescriptor) + } + } + } + } + } + + override val items: Map> by lazy { + val map = HashMap>() + json.forEach { (key, value) -> map[key] = value } + map.mapKeys { it.key.toName().first()!! } + } +} \ No newline at end of file diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/serialization/MetaSerializer.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/serialization/MetaSerializer.kt new file mode 100644 index 00000000..a98aa4c0 --- /dev/null +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/serialization/MetaSerializer.kt @@ -0,0 +1,110 @@ +package hep.dataforge.meta.serialization + +import hep.dataforge.meta.* +import hep.dataforge.names.NameToken +import hep.dataforge.values.* +import kotlinx.serialization.* +import kotlinx.serialization.builtins.MapSerializer +import kotlinx.serialization.builtins.list +import kotlinx.serialization.builtins.serializer +import kotlinx.serialization.json.JsonInput +import kotlinx.serialization.json.JsonObjectSerializer +import kotlinx.serialization.json.JsonOutput + + +@Serializer(Value::class) +@OptIn(InternalSerializationApi::class) +object ValueSerializer : KSerializer { + // private val valueTypeSerializer = EnumSerializer(ValueType::class) + private val listSerializer by lazy { ValueSerializer.list } + + override val descriptor: SerialDescriptor = SerialDescriptor("Value") { + boolean("isList") + enum("valueType") + string("value") + } + + private fun Decoder.decodeValue(): Value { + return when (decode(ValueType.serializer())) { + ValueType.NULL -> Null + ValueType.NUMBER -> decodeDouble().asValue() //TODO differentiate? + ValueType.BOOLEAN -> decodeBoolean().asValue() + ValueType.STRING -> decodeString().asValue() + } + } + + + override fun deserialize(decoder: Decoder): Value { + val isList = decoder.decodeBoolean() + return if (isList) { + listSerializer.deserialize(decoder).asValue() + } else { + decoder.decodeValue() + } + } + + private fun Encoder.encodeValue(value: Value) { + encode(ValueType.serializer(), value.type) + when (value.type) { + ValueType.NULL -> { + // do nothing + } + ValueType.NUMBER -> encodeDouble(value.double) + ValueType.BOOLEAN -> encodeBoolean(value.boolean) + ValueType.STRING -> encodeString(value.string) + } + } + + override fun serialize(encoder: Encoder, value: Value) { + encoder.encodeBoolean(value.isList()) + if (value.isList()) { + listSerializer.serialize(encoder, value.list) + } else { + encoder.encodeValue(value) + } + } +} + +private class DeserializedMeta(override val items: Map>) : MetaBase() + +/** + * Serialized for meta + */ +@Serializer(Meta::class) +object MetaSerializer : KSerializer { + private val mapSerializer = MapSerializer( + String.serializer(), + MetaItem.serializer(MetaSerializer) + ) + + override val descriptor: SerialDescriptor get() = mapSerializer.descriptor + + override fun deserialize(decoder: Decoder): Meta { + return if (decoder is JsonInput) { + JsonObjectSerializer.deserialize(decoder).toMeta() + } else { + DeserializedMeta(mapSerializer.deserialize(decoder).mapKeys { NameToken(it.key) }) + } + } + + override fun serialize(encoder: Encoder, value: Meta) { + if (encoder is JsonOutput) { + JsonObjectSerializer.serialize(encoder, value.toJson()) + } else { + mapSerializer.serialize(encoder, value.items.mapKeys { it.key.toString() }) + } + } +} + +@Serializer(Config::class) +object ConfigSerializer : KSerializer { + override val descriptor: SerialDescriptor = MetaSerializer.descriptor + + override fun deserialize(decoder: Decoder): Config { + return MetaSerializer.deserialize(decoder).asConfig() + } + + override fun serialize(encoder: Encoder, value: Config) { + MetaSerializer.serialize(encoder, value) + } +} \ No newline at end of file diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/serialization/serializationUtils.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/serialization/serializationUtils.kt new file mode 100644 index 00000000..a92609e9 --- /dev/null +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/serialization/serializationUtils.kt @@ -0,0 +1,67 @@ +package hep.dataforge.meta.serialization + +import hep.dataforge.meta.DFExperimental +import kotlinx.serialization.* +import kotlinx.serialization.builtins.DoubleArraySerializer +import kotlinx.serialization.internal.* + +fun SerialDescriptorBuilder.boolean(name: String, isOptional: Boolean = false, vararg annotations: Annotation) = + element(name, PrimitiveDescriptor(name, PrimitiveKind.BOOLEAN), isOptional = isOptional, annotations = annotations.toList()) + +fun SerialDescriptorBuilder.string(name: String, isOptional: Boolean = false, vararg annotations: Annotation) = + element(name, PrimitiveDescriptor(name, PrimitiveKind.STRING), isOptional = isOptional, annotations = annotations.toList()) + +fun SerialDescriptorBuilder.int(name: String, isOptional: Boolean = false, vararg annotations: Annotation) = + element(name, PrimitiveDescriptor(name, PrimitiveKind.INT), isOptional = isOptional, annotations = annotations.toList()) + +fun SerialDescriptorBuilder.double(name: String, isOptional: Boolean = false, vararg annotations: Annotation) = + element(name,PrimitiveDescriptor(name, PrimitiveKind.DOUBLE), isOptional = isOptional, annotations = annotations.toList()) + +fun SerialDescriptorBuilder.float(name: String, isOptional: Boolean = false, vararg annotations: Annotation) = + element(name, PrimitiveDescriptor(name, PrimitiveKind.FLOAT), isOptional = isOptional, annotations = annotations.toList()) + +fun SerialDescriptorBuilder.long(name: String, isOptional: Boolean = false, vararg annotations: Annotation) = + element(name, PrimitiveDescriptor(name, PrimitiveKind.LONG), isOptional = isOptional, annotations = annotations.toList()) + +fun SerialDescriptorBuilder.doubleArray(name: String, isOptional: Boolean = false, vararg annotations: Annotation) = + element(name, DoubleArraySerializer().descriptor, isOptional = isOptional, annotations = annotations.toList()) + +@OptIn(InternalSerializationApi::class) +inline fun > SerialDescriptorBuilder.enum(name: String, isOptional: Boolean = false, vararg annotations: Annotation) { + val enumDescriptor = SerialDescriptor(serialName, UnionKind.ENUM_KIND) { + enumValues().forEach { + val fqn = "$serialName.${it.name}" + val enumMemberDescriptor = SerialDescriptor(fqn, StructureKind.OBJECT) + element(it.name, enumMemberDescriptor) + } + } + element(name, enumDescriptor, isOptional = isOptional, annotations = annotations.toList()) +} + +//inline fun KSerializer.descriptor( +// name: String, +// block: SerialDescriptorBuilder.() -> Unit +//): SerialDescriptor = +// SerialDescriptorBuilder(SerialClassDescImpl(name)).apply(block).build() + +@DFExperimental +inline fun Decoder.decodeStructure( + desc: SerialDescriptor, + vararg typeParams: KSerializer<*> = emptyArray(), + crossinline block: CompositeDecoder.() -> R +): R { + val decoder = beginStructure(desc, *typeParams) + val res = decoder.block() + decoder.endStructure(desc) + return res +} + +inline fun Encoder.encodeStructure( + desc: SerialDescriptor, + vararg typeParams: KSerializer<*> = emptyArray(), + block: CompositeEncoder.() -> Unit +) { + val encoder = beginStructure(desc, *typeParams) + encoder.block() + encoder.endStructure(desc) +} \ No newline at end of file diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/transformations/MetaTransformation.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/transformations/MetaTransformation.kt index 4dad4466..d6f3bedf 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/transformations/MetaTransformation.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/transformations/MetaTransformation.kt @@ -90,7 +90,7 @@ inline class MetaTransformation(val transformations: Collection rule.selectItems(source).forEach { name -> rule.transformItem(name, source[name], this) @@ -102,7 +102,7 @@ inline class MetaTransformation(val transformations: Collection rule.selectItems(source).forEach { name -> remove(name) @@ -156,8 +156,8 @@ class MetaTransformationBuilder { fun keep(regex: String) { transformations.add( RegexItemTransformationRule(regex.toRegex()) { name, _, metaItem -> - setItem(name, metaItem) - }) + setItem(name, metaItem) + }) } /** diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/names/Name.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/names/Name.kt index 3c8d93d5..255a2ffd 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/names/Name.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/names/Name.kt @@ -1,11 +1,14 @@ package hep.dataforge.names +import kotlinx.serialization.* + /** * The general interface for working with names. * The name is a dot separated list of strings like `token1.token2.token3`. * Each token could contain additional index in square brackets. */ +@Serializable class Name(val tokens: List) { val length get() = tokens.size @@ -51,10 +54,21 @@ class Name(val tokens: List) { } - companion object { + @Serializer(Name::class) + companion object: KSerializer { const val NAME_SEPARATOR = "." val EMPTY = Name(emptyList()) + + override val descriptor: SerialDescriptor = PrimitiveDescriptor("Name", PrimitiveKind.STRING) + + override fun deserialize(decoder: Decoder): Name { + return decoder.decodeString().toName() + } + + override fun serialize(encoder: Encoder, value: Name) { + encoder.encodeString(value.toString()) + } } } @@ -63,6 +77,7 @@ class Name(val tokens: List) { * Following symbols are prohibited in name tokens: `{}.:\`. * A name token could have appendix in square brackets called *index* */ +@Serializable data class NameToken(val body: String, val index: String = "") { init { @@ -82,6 +97,19 @@ data class NameToken(val body: String, val index: String = "") { } fun hasIndex() = index.isNotEmpty() + + @Serializer(NameToken::class) + companion object :KSerializer{ + override val descriptor: SerialDescriptor = PrimitiveDescriptor("NameToken", PrimitiveKind.STRING) + + override fun deserialize(decoder: Decoder): NameToken { + return decoder.decodeString().toName().first()!! + } + + override fun serialize(encoder: Encoder, value: NameToken) { + encoder.encodeString(value.toString()) + } + } } /** diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/values/Value.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/values/Value.kt index f044b018..ccc92d50 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/values/Value.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/values/Value.kt @@ -1,5 +1,7 @@ package hep.dataforge.values +import kotlinx.serialization.Serializable + /** * The list of supported Value types. @@ -7,6 +9,7 @@ package hep.dataforge.values * Time value and binary value are represented by string * */ +@Serializable enum class ValueType { NUMBER, STRING, BOOLEAN, NULL }