Correct meta item index representation in JSON

build.gradle.kts

@@ -7,7 +7,7 @@ plugins {
     id("org.jetbrains.dokka") version "0.10.1"
 }
 
-val dataforgeVersion by extra("0.1.8-dev-5")
+val dataforgeVersion by extra("0.1.8-dev-6")
 
 val bintrayRepo by extra("dataforge")
 val githubProject by extra("dataforge-core")

dataforge-io/src/commonMain/kotlin/hep/dataforge/io/EnvelopeFormat.kt

@@ -16,7 +16,6 @@ import kotlin.reflect.KClass
 @ExperimentalUnsignedTypes
 data class PartialEnvelope(val meta: Meta, val dataOffset: UInt, val dataSize: ULong?)
 
-
 interface EnvelopeFormat : IOFormat<Envelope> {
     val defaultMetaFormat: MetaFormatFactory get() = JsonMetaFormat
 
@@ -33,6 +32,10 @@ interface EnvelopeFormat : IOFormat<Envelope> {
     override fun Output.writeObject(obj: Envelope): Unit = writeEnvelope(obj)
 }
 
+fun EnvelopeFormat.readPartial(input: Input) = input.readPartial()
+
+fun EnvelopeFormat.read(input: Input) = input.readObject()
+
 @Type(ENVELOPE_FORMAT_TYPE)
 interface EnvelopeFormatFactory : IOFormatFactory<Envelope>, EnvelopeFormat {
     override val name: Name get() = "envelope".asName()

dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt

@@ -166,6 +166,8 @@ class TaggedEnvelopeFormat(
 
         override fun Input.readObject(): Envelope =
             default.run { readObject() }
+
+
     }
 
 }

dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/JsonMeta.kt

@@ -6,7 +6,7 @@ import hep.dataforge.meta.descriptors.ItemDescriptor
 import hep.dataforge.meta.descriptors.NodeDescriptor
 import hep.dataforge.meta.descriptors.ValueDescriptor
 import hep.dataforge.names.NameToken
-import hep.dataforge.names.toName
+import hep.dataforge.names.withIndex
 import hep.dataforge.values.*
 import kotlinx.serialization.json.*
 
@@ -35,29 +35,29 @@ private fun String.toJsonKey(descriptor: ItemDescriptor?) = descriptor?.attribut
 /**
  * Convert given [Meta] to [JsonObject]. Primitives and nodes are copied as is, same name siblings are treated as json arrays
  */
-fun Meta.toJson(descriptor: NodeDescriptor? = null, index: String? = null): JsonObject {
+private fun Meta.toJsonWithIndex(descriptor: NodeDescriptor?, indexValue: String?): JsonObject {
 
     val elementMap = HashMap<String, JsonElement>()
 
-    fun MetaItem<*>.toJsonElement(itemDescriptor: ItemDescriptor?, index: String? = null): JsonElement = when (this) {
+    fun MetaItem<*>.toJsonElement(itemDescriptor: ItemDescriptor?, index: String?): JsonElement = when (this) {
         is MetaItem.ValueItem -> {
             value.toJson(itemDescriptor as? ValueDescriptor)
         }
         is MetaItem.NodeItem -> {
-            node.toJson(itemDescriptor as? NodeDescriptor, index)
+            node.toJsonWithIndex(itemDescriptor as? NodeDescriptor, index)
         }
     }
 
     fun addElement(key: String) {
         val itemDescriptor = descriptor?.items?.get(key)
         val jsonKey = key.toJsonKey(itemDescriptor)
-        val items = getIndexed(key)
+        val items: Map<String, MetaItem<*>> = getIndexed(key)
         when (items.size) {
             0 -> {
                 //do nothing
             }
             1 -> {
-                elementMap[jsonKey] = items.values.first().toJsonElement(itemDescriptor)
+                elementMap[jsonKey] = items.values.first().toJsonElement(itemDescriptor, null)
             }
             else -> {
                 val array = jsonArray {
@@ -73,38 +73,29 @@ fun Meta.toJson(descriptor: NodeDescriptor? = null, index: String? = null): Json
     ((descriptor?.items?.keys ?: emptySet()) + items.keys.map { it.body }).forEach(::addElement)
 
 
-    if (index != null) {
+    if (indexValue != null) {
-        elementMap["@index"] = JsonPrimitive(index)
+        val indexKey = descriptor?.indexKey ?: NodeDescriptor.DEFAULT_INDEX_KEY
+        elementMap[indexKey] = JsonPrimitive(indexValue)
     }
 
     return JsonObject(elementMap)
-
-//    // use descriptor keys in the order they are declared
-//    val keys = (descriptor?.items?.keys ?: emptySet()) + this.items.keys.map { it.body }
-//
-//    //TODO search for same name siblings and arrange them into arrays
-//    val map = this.items.entries.associate { (name, item) ->
-//        val itemDescriptor = descriptor?.items?.get(name.body)
-//        val key = name.toJsonKey(itemDescriptor)
-//        val value = when (item) {
-//            is MetaItem.ValueItem -> {
-//                item.value.toJson(itemDescriptor as? ValueDescriptor)
-//            }
-//            is MetaItem.NodeItem -> {
-//                item.node.toJson(itemDescriptor as? NodeDescriptor)
-//            }
-//        }
-//        key to value
-//    }
-//    return JsonObject(map)
 }
 
-fun JsonObject.toMeta(descriptor: NodeDescriptor? = null): Meta = JsonMeta(this, descriptor)
+fun Meta.toJson(descriptor: NodeDescriptor? = null): JsonObject = toJsonWithIndex(descriptor, null)
+
+fun JsonObject.toMeta(descriptor: NodeDescriptor? = null): JsonMeta = JsonMeta(this, descriptor)
 
 fun JsonPrimitive.toValue(descriptor: ValueDescriptor?): Value {
     return when (this) {
         JsonNull -> Null
-        else -> this.content.parseValue() // Optimize number and boolean parsing
+        is JsonLiteral -> {
+            when (body) {
+                true -> True
+                false -> False
+                is Number -> NumberValue(body as Number)
+                else -> StringValue(content)
+            }
+        }
     }
 }
 
@@ -131,9 +122,8 @@ fun JsonElement.toMetaItem(descriptor: ItemDescriptor? = null): MetaItem<JsonMet
             }
             MetaItem.ValueItem(value)
         } else {
-            json {
+            //We can't return multiple items therefore we create top level node
-                "@value" to this@toMetaItem
+            json { "@json" to this@toMetaItem }.toMetaItem(descriptor)
-            }.toMetaItem(descriptor)
         }
     }
 }
@@ -143,44 +133,44 @@ fun JsonElement.toMetaItem(descriptor: ItemDescriptor? = null): MetaItem<JsonMet
  */
 class JsonMeta(val json: JsonObject, val descriptor: NodeDescriptor? = null) : MetaBase() {
 
-    @Suppress("UNCHECKED_CAST")
+    private fun buildItems(): Map<NameToken, MetaItem<JsonMeta>> {
-    private operator fun MutableMap<String, MetaItem<JsonMeta>>.set(key: String, value: JsonElement): Unit {
+        val map = LinkedHashMap<NameToken, MetaItem<JsonMeta>>()
-        val itemDescriptor = descriptor?.items?.get(key)
+
+        json.forEach { (jsonKey, value) ->
+            val key = NameToken(jsonKey)
+            val itemDescriptor = descriptor?.items?.get(jsonKey)
             when (value) {
                 is JsonPrimitive -> {
-                this[key] =
+                    map[key] = MetaItem.ValueItem(value.toValue(itemDescriptor as? ValueDescriptor))
-                    MetaItem.ValueItem(value.toValue(itemDescriptor as? ValueDescriptor)) as MetaItem<JsonMeta>
                 }
                 is JsonObject -> {
-                this[key] = MetaItem.NodeItem(
+                    map[key] = MetaItem.NodeItem(
                         JsonMeta(
                             value,
                             itemDescriptor as? NodeDescriptor
                         )
                     )
                 }
-            is JsonArray -> {
+                is JsonArray -> if (value.all { it is JsonPrimitive }) {
-                when {
-                    value.all { it is JsonPrimitive } -> {
                     val listValue = ListValue(
                         value.map {
                             //We already checked that all values are primitives
                             (it as JsonPrimitive).toValue(itemDescriptor as? ValueDescriptor)
                         }
                     )
-                        this[key] = MetaItem.ValueItem(listValue) as MetaItem<JsonMeta>
+                    map[key] = MetaItem.ValueItem(listValue)
-                    }
+                } else value.forEachIndexed { index, jsonElement ->
-                    else -> value.forEachIndexed { index, jsonElement ->
+                    val indexKey = (itemDescriptor as? NodeDescriptor)?.indexKey ?: NodeDescriptor.DEFAULT_INDEX_KEY
-                        this["$key[$index]"] = jsonElement.toMetaItem(itemDescriptor)
+                    val indexValue: String = (jsonElement as? JsonObject)
-                    }
+                        ?.get(indexKey)?.contentOrNull ?: index.toString() //In case index is non-string, the backward transformation will be broken.
+
+                    val token = key.withIndex(indexValue)
+                    map[token] = jsonElement.toMetaItem(itemDescriptor)
                 }
             }
         }
+        return map
     }
 
-    override val items: Map<NameToken, MetaItem<JsonMeta>> by lazy {
+    override val items: Map<NameToken, MetaItem<JsonMeta>> by lazy(::buildItems)
-        val map = LinkedHashMap<String, MetaItem<JsonMeta>>()
-        json.forEach { (key, value) -> map[key] = value }
-        map.mapKeys { it.key.toName().first()!! }
-    }
 }

dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MutableMeta.kt

@@ -135,12 +135,12 @@ fun <M : MutableMeta<M>> M.update(meta: Meta) {
 fun MutableMeta<*>.setIndexedItems(
     name: Name,
     items: Iterable<MetaItem<*>>,
-    indexFactory: MetaItem<*>.(index: Int) -> String = { it.toString() }
+    indexFactory: (MetaItem<*>, index: Int) -> String = {_, index-> index.toString() }
 ) {
     val tokens = name.tokens.toMutableList()
     val last = tokens.last()
     items.forEachIndexed { index, meta ->
-        val indexedToken = NameToken(last.body, last.index + meta.indexFactory(index))
+        val indexedToken = NameToken(last.body, last.index + indexFactory(meta, index))
         tokens[tokens.lastIndex] = indexedToken
         set(Name(tokens), meta)
     }
@@ -149,9 +149,9 @@ fun MutableMeta<*>.setIndexedItems(
 fun MutableMeta<*>.setIndexed(
     name: Name,
     metas: Iterable<Meta>,
-    indexFactory: MetaItem<*>.(index: Int) -> String = { it.toString() }
+    indexFactory: (Meta, index: Int) -> String = { _, index-> index.toString() }
 ) {
-    setIndexedItems(name, metas.map { MetaItem.NodeItem(it) }, indexFactory)
+    setIndexedItems(name, metas.map { MetaItem.NodeItem(it) }){item, index ->indexFactory(item.node!!, index)}
 }
 
 operator fun MutableMeta<*>.set(name: Name, metas: Iterable<Meta>): Unit = setIndexed(name, metas)

dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MutableMetaDelegate.kt

@@ -50,9 +50,6 @@ fun <R> ReadWriteProperty<Any?, MetaItem<*>?>.transform(reader: (MetaItem<*>?) -
 fun <R> ReadWriteProperty<Any?, Value?>.transform(reader: (Value?) -> R): ReadWriteDelegateWrapper<Value?, R> =
     map(reader = reader, writer = { Value.of(it) })
 
-/**
- * A delegate that throws
- */
 fun <R : Any> ReadWriteProperty<Any?, R?>.notNull(default: () -> R): ReadWriteProperty<Any?, R> {
     return ReadWriteDelegateWrapper(this,
         reader = { it ?: default() },
@@ -60,7 +57,6 @@ fun <R : Any> ReadWriteProperty<Any?, R?>.notNull(default: () -> R): ReadWritePr
     )
 }
 
-
 fun <M : MutableMeta<M>> M.item(default: Any? = null, key: Name? = null): MutableMetaDelegate<M> =
     MutableMetaDelegate(this, key, default?.let { MetaItem.of(it) })
 

dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/descriptors/ItemDescriptor.kt

@@ -95,6 +95,11 @@ class NodeDescriptor(config: Config = Config()) : ItemDescriptor(config) {
      */
     var default by config.node()
 
+    /**
+     * An index field by which this node is identified in case of same name siblings construct
+     */
+    var indexKey by config.string(DEFAULT_INDEX_KEY)
+
     val items: Map<String, ItemDescriptor>
         get() = config.getIndexed(ITEM_KEY).mapValues { (_, item) ->
             val node = item.node ?: error("Node descriptor must be a node")
@@ -182,6 +187,8 @@ class NodeDescriptor(config: Config = Config()) : ItemDescriptor(config) {
         val ITEM_KEY = "item".asName()
         val IS_NODE_KEY = "@isNode".asName()
 
+        const val DEFAULT_INDEX_KEY = "@index"
+
         inline operator fun invoke(block: NodeDescriptor.() -> Unit) = NodeDescriptor().apply(block)
 
         //TODO infer descriptor from spec

dataforge-meta/src/commonMain/kotlin/hep/dataforge/names/Name.kt

@@ -113,6 +113,8 @@ data class NameToken(val body: String, val index: String = "") {
     }
 }
 
+fun NameToken.withIndex(newIndex: String) = NameToken(body, newIndex)
+
 /**
  * Convert a [String] to name parsing it and extracting name tokens and index syntax.
  * This operation is rather heavy so it should be used with care in high performance code.

dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/JsonMetaTest.kt

@@ -1,5 +1,6 @@
 package hep.dataforge.meta
 
+import hep.dataforge.meta.descriptors.NodeDescriptor
 import kotlinx.serialization.json.int
 import kotlinx.serialization.json.json
 import kotlinx.serialization.json.jsonArray
@@ -17,23 +18,34 @@ class JsonMetaTest {
         }
         "nodeArray" to jsonArray {
             +json {
-                "index" to 1
+                "index" to "1"
+                "value" to 2
             }
             +json {
-                "index" to 2
+                "index" to "2"
+                "value" to 3
             }
             +json {
-                "index" to 3
+                "index" to "3"
+                "value" to 4
             }
         }
     }
 
+    val descriptor = NodeDescriptor{
+        node("nodeArray"){
+            indexKey = "index"
+        }
+    }
+
     @Test
     fun jsonMetaConversion() {
-        val meta = json.toMeta()
-        val reconstructed = meta.toJson()
         println(json)
+        val meta = json.toMeta(descriptor)
+        //println(meta)
+        val reconstructed = meta.toJson(descriptor)
         println(reconstructed)
         assertEquals(2, reconstructed["nodeArray"]?.jsonArray?.get(1)?.jsonObject?.get("index")?.int)
+        assertEquals(json,reconstructed)
     }
 }

dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskModel.kt

@@ -36,7 +36,11 @@ data class TaskModel(
             val dataDependencies = dependencies.filterIsInstance<DataDependency>()
             val taskDependencies = dependencies.filterIsInstance<TaskDependency<*>>()
             setIndexed("data".toName(), dataDependencies.map { it.toMeta() })
-            setIndexed("task".toName(), taskDependencies.map { it.toMeta() }) { taskDependencies[it].name.toString() }
+            setIndexed(
+                "task".toName(),
+                taskDependencies.map { it.toMeta() }) { _, index ->
+                taskDependencies[index].name.toString()
+            }
             //TODO ensure all dependencies are listed
         }
     }