From faeb7376723ac0ba248deb89c360e138b22ad5d5 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Fri, 3 Jul 2020 20:38:19 +0300 Subject: [PATCH] Correct meta item index representation in JSON --- build.gradle.kts | 2 +- .../kotlin/hep/dataforge/io/EnvelopeFormat.kt | 5 +- .../hep/dataforge/io/TaggedEnvelopeFormat.kt | 2 + .../kotlin/hep/dataforge/meta/JsonMeta.kt | 122 ++++++++---------- .../kotlin/hep/dataforge/meta/MutableMeta.kt | 8 +- .../hep/dataforge/meta/MutableMetaDelegate.kt | 6 +- .../meta/descriptors/ItemDescriptor.kt | 7 + .../kotlin/hep/dataforge/names/Name.kt | 2 + .../kotlin/hep/dataforge/meta/JsonMetaTest.kt | 22 +++- .../hep/dataforge/workspace/TaskModel.kt | 6 +- 10 files changed, 99 insertions(+), 83 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index c0415544..2a951fcd 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -7,7 +7,7 @@ plugins { id("org.jetbrains.dokka") version "0.10.1" } -val dataforgeVersion by extra("0.1.8-dev-5") +val dataforgeVersion by extra("0.1.8-dev-6") val bintrayRepo by extra("dataforge") val githubProject by extra("dataforge-core") diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/EnvelopeFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/EnvelopeFormat.kt index c634d696..9b819a97 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/EnvelopeFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/EnvelopeFormat.kt @@ -16,7 +16,6 @@ import kotlin.reflect.KClass @ExperimentalUnsignedTypes data class PartialEnvelope(val meta: Meta, val dataOffset: UInt, val dataSize: ULong?) - interface EnvelopeFormat : IOFormat { val defaultMetaFormat: MetaFormatFactory get() = JsonMetaFormat @@ -33,6 +32,10 @@ interface EnvelopeFormat : IOFormat { override fun Output.writeObject(obj: Envelope): Unit = writeEnvelope(obj) } +fun EnvelopeFormat.readPartial(input: Input) = input.readPartial() + +fun EnvelopeFormat.read(input: Input) = input.readObject() + @Type(ENVELOPE_FORMAT_TYPE) interface EnvelopeFormatFactory : IOFormatFactory, EnvelopeFormat { override val name: Name get() = "envelope".asName() diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt index 17fbc589..ebcca243 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt @@ -166,6 +166,8 @@ class TaggedEnvelopeFormat( override fun Input.readObject(): Envelope = default.run { readObject() } + + } } \ No newline at end of file diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/JsonMeta.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/JsonMeta.kt index 48ef6c4e..0059f658 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/JsonMeta.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/JsonMeta.kt @@ -6,7 +6,7 @@ import hep.dataforge.meta.descriptors.ItemDescriptor import hep.dataforge.meta.descriptors.NodeDescriptor import hep.dataforge.meta.descriptors.ValueDescriptor import hep.dataforge.names.NameToken -import hep.dataforge.names.toName +import hep.dataforge.names.withIndex import hep.dataforge.values.* import kotlinx.serialization.json.* @@ -35,29 +35,29 @@ private fun String.toJsonKey(descriptor: ItemDescriptor?) = descriptor?.attribut /** * Convert given [Meta] to [JsonObject]. Primitives and nodes are copied as is, same name siblings are treated as json arrays */ -fun Meta.toJson(descriptor: NodeDescriptor? = null, index: String? = null): JsonObject { +private fun Meta.toJsonWithIndex(descriptor: NodeDescriptor?, indexValue: String?): JsonObject { val elementMap = HashMap() - fun MetaItem<*>.toJsonElement(itemDescriptor: ItemDescriptor?, index: String? = null): JsonElement = when (this) { + fun MetaItem<*>.toJsonElement(itemDescriptor: ItemDescriptor?, index: String?): JsonElement = when (this) { is MetaItem.ValueItem -> { value.toJson(itemDescriptor as? ValueDescriptor) } is MetaItem.NodeItem -> { - node.toJson(itemDescriptor as? NodeDescriptor, index) + node.toJsonWithIndex(itemDescriptor as? NodeDescriptor, index) } } fun addElement(key: String) { val itemDescriptor = descriptor?.items?.get(key) val jsonKey = key.toJsonKey(itemDescriptor) - val items = getIndexed(key) + val items: Map> = getIndexed(key) when (items.size) { 0 -> { //do nothing } 1 -> { - elementMap[jsonKey] = items.values.first().toJsonElement(itemDescriptor) + elementMap[jsonKey] = items.values.first().toJsonElement(itemDescriptor, null) } else -> { val array = jsonArray { @@ -73,38 +73,29 @@ fun Meta.toJson(descriptor: NodeDescriptor? = null, index: String? = null): Json ((descriptor?.items?.keys ?: emptySet()) + items.keys.map { it.body }).forEach(::addElement) - if (index != null) { - elementMap["@index"] = JsonPrimitive(index) + if (indexValue != null) { + val indexKey = descriptor?.indexKey ?: NodeDescriptor.DEFAULT_INDEX_KEY + elementMap[indexKey] = JsonPrimitive(indexValue) } return JsonObject(elementMap) - -// // use descriptor keys in the order they are declared -// val keys = (descriptor?.items?.keys ?: emptySet()) + this.items.keys.map { it.body } -// -// //TODO search for same name siblings and arrange them into arrays -// val map = this.items.entries.associate { (name, item) -> -// val itemDescriptor = descriptor?.items?.get(name.body) -// val key = name.toJsonKey(itemDescriptor) -// val value = when (item) { -// is MetaItem.ValueItem -> { -// item.value.toJson(itemDescriptor as? ValueDescriptor) -// } -// is MetaItem.NodeItem -> { -// item.node.toJson(itemDescriptor as? NodeDescriptor) -// } -// } -// key to value -// } -// return JsonObject(map) } -fun JsonObject.toMeta(descriptor: NodeDescriptor? = null): Meta = JsonMeta(this, descriptor) +fun Meta.toJson(descriptor: NodeDescriptor? = null): JsonObject = toJsonWithIndex(descriptor, null) + +fun JsonObject.toMeta(descriptor: NodeDescriptor? = null): JsonMeta = JsonMeta(this, descriptor) fun JsonPrimitive.toValue(descriptor: ValueDescriptor?): Value { return when (this) { JsonNull -> Null - else -> this.content.parseValue() // Optimize number and boolean parsing + is JsonLiteral -> { + when (body) { + true -> True + false -> False + is Number -> NumberValue(body as Number) + else -> StringValue(content) + } + } } } @@ -131,9 +122,8 @@ fun JsonElement.toMetaItem(descriptor: ItemDescriptor? = null): MetaItem>.set(key: String, value: JsonElement): Unit { - val itemDescriptor = descriptor?.items?.get(key) - when (value) { - is JsonPrimitive -> { - this[key] = - MetaItem.ValueItem(value.toValue(itemDescriptor as? ValueDescriptor)) as MetaItem - } - is JsonObject -> { - this[key] = MetaItem.NodeItem( - JsonMeta( - value, - itemDescriptor as? NodeDescriptor - ) - ) - } - is JsonArray -> { - when { - value.all { it is JsonPrimitive } -> { - val listValue = ListValue( - value.map { - //We already checked that all values are primitives - (it as JsonPrimitive).toValue(itemDescriptor as? ValueDescriptor) - } + private fun buildItems(): Map> { + val map = LinkedHashMap>() + + json.forEach { (jsonKey, value) -> + val key = NameToken(jsonKey) + val itemDescriptor = descriptor?.items?.get(jsonKey) + when (value) { + is JsonPrimitive -> { + map[key] = MetaItem.ValueItem(value.toValue(itemDescriptor as? ValueDescriptor)) + } + is JsonObject -> { + map[key] = MetaItem.NodeItem( + JsonMeta( + value, + itemDescriptor as? NodeDescriptor ) - this[key] = MetaItem.ValueItem(listValue) as MetaItem - } - else -> value.forEachIndexed { index, jsonElement -> - this["$key[$index]"] = jsonElement.toMetaItem(itemDescriptor) - } + ) + } + is JsonArray -> if (value.all { it is JsonPrimitive }) { + val listValue = ListValue( + value.map { + //We already checked that all values are primitives + (it as JsonPrimitive).toValue(itemDescriptor as? ValueDescriptor) + } + ) + map[key] = MetaItem.ValueItem(listValue) + } else value.forEachIndexed { index, jsonElement -> + val indexKey = (itemDescriptor as? NodeDescriptor)?.indexKey ?: NodeDescriptor.DEFAULT_INDEX_KEY + val indexValue: String = (jsonElement as? JsonObject) + ?.get(indexKey)?.contentOrNull ?: index.toString() //In case index is non-string, the backward transformation will be broken. + + val token = key.withIndex(indexValue) + map[token] = jsonElement.toMetaItem(itemDescriptor) } } } + return map } - override val items: Map> by lazy { - val map = LinkedHashMap>() - json.forEach { (key, value) -> map[key] = value } - map.mapKeys { it.key.toName().first()!! } - } + override val items: Map> by lazy(::buildItems) } \ No newline at end of file diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MutableMeta.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MutableMeta.kt index 16fcf40d..91e74ce2 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MutableMeta.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MutableMeta.kt @@ -135,12 +135,12 @@ fun > M.update(meta: Meta) { fun MutableMeta<*>.setIndexedItems( name: Name, items: Iterable>, - indexFactory: MetaItem<*>.(index: Int) -> String = { it.toString() } + indexFactory: (MetaItem<*>, index: Int) -> String = {_, index-> index.toString() } ) { val tokens = name.tokens.toMutableList() val last = tokens.last() items.forEachIndexed { index, meta -> - val indexedToken = NameToken(last.body, last.index + meta.indexFactory(index)) + val indexedToken = NameToken(last.body, last.index + indexFactory(meta, index)) tokens[tokens.lastIndex] = indexedToken set(Name(tokens), meta) } @@ -149,9 +149,9 @@ fun MutableMeta<*>.setIndexedItems( fun MutableMeta<*>.setIndexed( name: Name, metas: Iterable, - indexFactory: MetaItem<*>.(index: Int) -> String = { it.toString() } + indexFactory: (Meta, index: Int) -> String = { _, index-> index.toString() } ) { - setIndexedItems(name, metas.map { MetaItem.NodeItem(it) }, indexFactory) + setIndexedItems(name, metas.map { MetaItem.NodeItem(it) }){item, index ->indexFactory(item.node!!, index)} } operator fun MutableMeta<*>.set(name: Name, metas: Iterable): Unit = setIndexed(name, metas) diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MutableMetaDelegate.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MutableMetaDelegate.kt index 3230afbe..78589fbd 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MutableMetaDelegate.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MutableMetaDelegate.kt @@ -50,9 +50,6 @@ fun ReadWriteProperty?>.transform(reader: (MetaItem<*>?) - fun ReadWriteProperty.transform(reader: (Value?) -> R): ReadWriteDelegateWrapper = map(reader = reader, writer = { Value.of(it) }) -/** - * A delegate that throws - */ fun ReadWriteProperty.notNull(default: () -> R): ReadWriteProperty { return ReadWriteDelegateWrapper(this, reader = { it ?: default() }, @@ -60,7 +57,6 @@ fun ReadWriteProperty.notNull(default: () -> R): ReadWritePr ) } - fun > M.item(default: Any? = null, key: Name? = null): MutableMetaDelegate = MutableMetaDelegate(this, key, default?.let { MetaItem.of(it) }) @@ -117,4 +113,4 @@ fun > M.number(key: Name? = null, default: () -> Number) = inline fun , reified E : Enum> M.enum(default: E, key: Name? = null) = - item(default, key).transform { it.enum()!! } \ No newline at end of file + item(default, key).transform { it.enum()!! } diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/descriptors/ItemDescriptor.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/descriptors/ItemDescriptor.kt index 7f748ff2..2888ac59 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/descriptors/ItemDescriptor.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/descriptors/ItemDescriptor.kt @@ -95,6 +95,11 @@ class NodeDescriptor(config: Config = Config()) : ItemDescriptor(config) { */ var default by config.node() + /** + * An index field by which this node is identified in case of same name siblings construct + */ + var indexKey by config.string(DEFAULT_INDEX_KEY) + val items: Map get() = config.getIndexed(ITEM_KEY).mapValues { (_, item) -> val node = item.node ?: error("Node descriptor must be a node") @@ -182,6 +187,8 @@ class NodeDescriptor(config: Config = Config()) : ItemDescriptor(config) { val ITEM_KEY = "item".asName() val IS_NODE_KEY = "@isNode".asName() + const val DEFAULT_INDEX_KEY = "@index" + inline operator fun invoke(block: NodeDescriptor.() -> Unit) = NodeDescriptor().apply(block) //TODO infer descriptor from spec diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/names/Name.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/names/Name.kt index 141af13c..b3f8231a 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/names/Name.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/names/Name.kt @@ -113,6 +113,8 @@ data class NameToken(val body: String, val index: String = "") { } } +fun NameToken.withIndex(newIndex: String) = NameToken(body, newIndex) + /** * Convert a [String] to name parsing it and extracting name tokens and index syntax. * This operation is rather heavy so it should be used with care in high performance code. diff --git a/dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/JsonMetaTest.kt b/dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/JsonMetaTest.kt index 127649da..9086e156 100644 --- a/dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/JsonMetaTest.kt +++ b/dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/JsonMetaTest.kt @@ -1,5 +1,6 @@ package hep.dataforge.meta +import hep.dataforge.meta.descriptors.NodeDescriptor import kotlinx.serialization.json.int import kotlinx.serialization.json.json import kotlinx.serialization.json.jsonArray @@ -17,23 +18,34 @@ class JsonMetaTest { } "nodeArray" to jsonArray { +json { - "index" to 1 + "index" to "1" + "value" to 2 } +json { - "index" to 2 + "index" to "2" + "value" to 3 } +json { - "index" to 3 + "index" to "3" + "value" to 4 } } } + val descriptor = NodeDescriptor{ + node("nodeArray"){ + indexKey = "index" + } + } + @Test fun jsonMetaConversion() { - val meta = json.toMeta() - val reconstructed = meta.toJson() println(json) + val meta = json.toMeta(descriptor) + //println(meta) + val reconstructed = meta.toJson(descriptor) println(reconstructed) assertEquals(2, reconstructed["nodeArray"]?.jsonArray?.get(1)?.jsonObject?.get("index")?.int) + assertEquals(json,reconstructed) } } \ No newline at end of file diff --git a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskModel.kt b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskModel.kt index 2f7b6998..acd16b2d 100644 --- a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskModel.kt +++ b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskModel.kt @@ -36,7 +36,11 @@ data class TaskModel( val dataDependencies = dependencies.filterIsInstance() val taskDependencies = dependencies.filterIsInstance>() setIndexed("data".toName(), dataDependencies.map { it.toMeta() }) - setIndexed("task".toName(), taskDependencies.map { it.toMeta() }) { taskDependencies[it].name.toString() } + setIndexed( + "task".toName(), + taskDependencies.map { it.toMeta() }) { _, index -> + taskDependencies[index].name.toString() + } //TODO ensure all dependencies are listed } }