kscience/dataforge-core
8
0
Fork 0
Code Issues Pull Requests 1 Packages Projects Releases 12 Wiki Activity

Correct meta item index representation in JSON

Browse Source
This commit is contained in:
Alexander Nozik 2020-07-03 20:38:19 +03:00
parent 87326d05c7
commit faeb737672
10 changed files with 99 additions and 83 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
build.gradle.kts
View File

@ -7,7 +7,7 @@ plugins {
id("org.jetbrains.dokka") version "0.10.1"
}
val dataforgeVersion by extra("0.1.8-dev-5")
val dataforgeVersion by extra("0.1.8-dev-6")
val bintrayRepo by extra("dataforge")
val githubProject by extra("dataforge-core")

5
dataforge-io/src/commonMain/kotlin/hep/dataforge/io/EnvelopeFormat.kt
View File

@ -16,7 +16,6 @@ import kotlin.reflect.KClass
@ExperimentalUnsignedTypes
data class PartialEnvelope(val meta: Meta, val dataOffset: UInt, val dataSize: ULong?)
interface EnvelopeFormat : IOFormat<Envelope> {
val defaultMetaFormat: MetaFormatFactory get() = JsonMetaFormat
@ -33,6 +32,10 @@ interface EnvelopeFormat : IOFormat<Envelope> {
override fun Output.writeObject(obj: Envelope): Unit = writeEnvelope(obj)
}
fun EnvelopeFormat.readPartial(input: Input) = input.readPartial()
fun EnvelopeFormat.read(input: Input) = input.readObject()
@Type(ENVELOPE_FORMAT_TYPE)
interface EnvelopeFormatFactory : IOFormatFactory<Envelope>, EnvelopeFormat {
override val name: Name get() = "envelope".asName()

2
dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt
View File

@ -166,6 +166,8 @@ class TaggedEnvelopeFormat(
override fun Input.readObject(): Envelope =
default.run { readObject() }
}
}

122
dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/JsonMeta.kt
View File

@ -6,7 +6,7 @@ import hep.dataforge.meta.descriptors.ItemDescriptor
import hep.dataforge.meta.descriptors.NodeDescriptor
import hep.dataforge.meta.descriptors.ValueDescriptor
import hep.dataforge.names.NameToken
import hep.dataforge.names.toName
import hep.dataforge.names.withIndex
import hep.dataforge.values.*
import kotlinx.serialization.json.*
@ -35,29 +35,29 @@ private fun String.toJsonKey(descriptor: ItemDescriptor?) = descriptor?.attribut
/**
* Convert given [Meta] to [JsonObject]. Primitives and nodes are copied as is, same name siblings are treated as json arrays
*/
fun Meta.toJson(descriptor: NodeDescriptor? = null, index: String? = null): JsonObject {
private fun Meta.toJsonWithIndex(descriptor: NodeDescriptor?, indexValue: String?): JsonObject {
val elementMap = HashMap<String, JsonElement>()
fun MetaItem<*>.toJsonElement(itemDescriptor: ItemDescriptor?, index: String? = null): JsonElement = when (this) {
fun MetaItem<*>.toJsonElement(itemDescriptor: ItemDescriptor?, index: String?): JsonElement = when (this) {
is MetaItem.ValueItem -> {
value.toJson(itemDescriptor as? ValueDescriptor)
}
is MetaItem.NodeItem -> {
node.toJson(itemDescriptor as? NodeDescriptor, index)
node.toJsonWithIndex(itemDescriptor as? NodeDescriptor, index)
}
}
fun addElement(key: String) {
val itemDescriptor = descriptor?.items?.get(key)
val jsonKey = key.toJsonKey(itemDescriptor)
val items = getIndexed(key)
val items: Map<String, MetaItem<*>> = getIndexed(key)
when (items.size) {
0 -> {
//do nothing
}
1 -> {
elementMap[jsonKey] = items.values.first().toJsonElement(itemDescriptor)
elementMap[jsonKey] = items.values.first().toJsonElement(itemDescriptor, null)
}
else -> {
val array = jsonArray {
@ -73,38 +73,29 @@ fun Meta.toJson(descriptor: NodeDescriptor? = null, index: String? = null): Json
((descriptor?.items?.keys ?: emptySet()) + items.keys.map { it.body }).forEach(::addElement)
if (index != null) {
elementMap["@index"] = JsonPrimitive(index)
if (indexValue != null) {
val indexKey = descriptor?.indexKey ?: NodeDescriptor.DEFAULT_INDEX_KEY
elementMap[indexKey] = JsonPrimitive(indexValue)
}
return JsonObject(elementMap)
// // use descriptor keys in the order they are declared
// val keys = (descriptor?.items?.keys ?: emptySet()) + this.items.keys.map { it.body }
//
// //TODO search for same name siblings and arrange them into arrays
// val map = this.items.entries.associate { (name, item) ->
// val itemDescriptor = descriptor?.items?.get(name.body)
// val key = name.toJsonKey(itemDescriptor)
// val value = when (item) {
// is MetaItem.ValueItem -> {
// item.value.toJson(itemDescriptor as? ValueDescriptor)
// }
// is MetaItem.NodeItem -> {
// item.node.toJson(itemDescriptor as? NodeDescriptor)
// }
// }
// key to value
// }
// return JsonObject(map)
}
fun JsonObject.toMeta(descriptor: NodeDescriptor? = null): Meta = JsonMeta(this, descriptor)
fun Meta.toJson(descriptor: NodeDescriptor? = null): JsonObject = toJsonWithIndex(descriptor, null)
fun JsonObject.toMeta(descriptor: NodeDescriptor? = null): JsonMeta = JsonMeta(this, descriptor)
fun JsonPrimitive.toValue(descriptor: ValueDescriptor?): Value {
return when (this) {
JsonNull -> Null
else -> this.content.parseValue() // Optimize number and boolean parsing
is JsonLiteral -> {
when (body) {
true -> True
false -> False
is Number -> NumberValue(body as Number)
else -> StringValue(content)
}
}
}
}
@ -131,9 +122,8 @@ fun JsonElement.toMetaItem(descriptor: ItemDescriptor? = null): MetaItem<JsonMet
}
MetaItem.ValueItem(value)
} else {
json {
"@value" to this@toMetaItem
}.toMetaItem(descriptor)
//We can't return multiple items therefore we create top level node
json { "@json" to this@toMetaItem }.toMetaItem(descriptor)
}
}
}
@ -143,44 +133,44 @@ fun JsonElement.toMetaItem(descriptor: ItemDescriptor? = null): MetaItem<JsonMet
*/
class JsonMeta(val json: JsonObject, val descriptor: NodeDescriptor? = null) : MetaBase() {
@Suppress("UNCHECKED_CAST")
private operator fun MutableMap<String, MetaItem<JsonMeta>>.set(key: String, value: JsonElement): Unit {
val itemDescriptor = descriptor?.items?.get(key)
when (value) {
is JsonPrimitive -> {
this[key] =
MetaItem.ValueItem(value.toValue(itemDescriptor as? ValueDescriptor)) as MetaItem<JsonMeta>
}
is JsonObject -> {
this[key] = MetaItem.NodeItem(
JsonMeta(
value,
itemDescriptor as? NodeDescriptor
)
)
}
is JsonArray -> {
when {
value.all { it is JsonPrimitive } -> {
val listValue = ListValue(
value.map {
//We already checked that all values are primitives
(it as JsonPrimitive).toValue(itemDescriptor as? ValueDescriptor)
}
private fun buildItems(): Map<NameToken, MetaItem<JsonMeta>> {
val map = LinkedHashMap<NameToken, MetaItem<JsonMeta>>()
json.forEach { (jsonKey, value) ->
val key = NameToken(jsonKey)
val itemDescriptor = descriptor?.items?.get(jsonKey)
when (value) {
is JsonPrimitive -> {
map[key] = MetaItem.ValueItem(value.toValue(itemDescriptor as? ValueDescriptor))
}
is JsonObject -> {
map[key] = MetaItem.NodeItem(
JsonMeta(
value,
itemDescriptor as? NodeDescriptor
)
this[key] = MetaItem.ValueItem(listValue) as MetaItem<JsonMeta>
}
else -> value.forEachIndexed { index, jsonElement ->
this["$key[$index]"] = jsonElement.toMetaItem(itemDescriptor)
}
)
}
is JsonArray -> if (value.all { it is JsonPrimitive }) {
val listValue = ListValue(
value.map {
//We already checked that all values are primitives
(it as JsonPrimitive).toValue(itemDescriptor as? ValueDescriptor)
}
)
map[key] = MetaItem.ValueItem(listValue)
} else value.forEachIndexed { index, jsonElement ->
val indexKey = (itemDescriptor as? NodeDescriptor)?.indexKey ?: NodeDescriptor.DEFAULT_INDEX_KEY
val indexValue: String = (jsonElement as? JsonObject)
?.get(indexKey)?.contentOrNull ?: index.toString() //In case index is non-string, the backward transformation will be broken.
val token = key.withIndex(indexValue)
map[token] = jsonElement.toMetaItem(itemDescriptor)
}
}
}
return map
}
override val items: Map<NameToken, MetaItem<JsonMeta>> by lazy {
val map = LinkedHashMap<String, MetaItem<JsonMeta>>()
json.forEach { (key, value) -> map[key] = value }
map.mapKeys { it.key.toName().first()!! }
}
override val items: Map<NameToken, MetaItem<JsonMeta>> by lazy(::buildItems)
}

8
dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MutableMeta.kt
View File

@ -135,12 +135,12 @@ fun <M : MutableMeta<M>> M.update(meta: Meta) {
fun MutableMeta<*>.setIndexedItems(
name: Name,
items: Iterable<MetaItem<*>>,
indexFactory: MetaItem<*>.(index: Int) -> String = { it.toString() }
indexFactory: (MetaItem<*>, index: Int) -> String = {_, index-> index.toString() }
) {
val tokens = name.tokens.toMutableList()
val last = tokens.last()
items.forEachIndexed { index, meta ->
val indexedToken = NameToken(last.body, last.index + meta.indexFactory(index))
val indexedToken = NameToken(last.body, last.index + indexFactory(meta, index))
tokens[tokens.lastIndex] = indexedToken
set(Name(tokens), meta)
}
@ -149,9 +149,9 @@ fun MutableMeta<*>.setIndexedItems(
fun MutableMeta<*>.setIndexed(
name: Name,
metas: Iterable<Meta>,
indexFactory: MetaItem<*>.(index: Int) -> String = { it.toString() }
indexFactory: (Meta, index: Int) -> String = { _, index-> index.toString() }
) {
setIndexedItems(name, metas.map { MetaItem.NodeItem(it) }, indexFactory)
setIndexedItems(name, metas.map { MetaItem.NodeItem(it) }){item, index ->indexFactory(item.node!!, index)}
}
operator fun MutableMeta<*>.set(name: Name, metas: Iterable<Meta>): Unit = setIndexed(name, metas)

6
dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MutableMetaDelegate.kt
View File

@ -50,9 +50,6 @@ fun <R> ReadWriteProperty<Any?, MetaItem<*>?>.transform(reader: (MetaItem<*>?) -
fun <R> ReadWriteProperty<Any?, Value?>.transform(reader: (Value?) -> R): ReadWriteDelegateWrapper<Value?, R> =
map(reader = reader, writer = { Value.of(it) })
/**
* A delegate that throws
*/
fun <R : Any> ReadWriteProperty<Any?, R?>.notNull(default: () -> R): ReadWriteProperty<Any?, R> {
return ReadWriteDelegateWrapper(this,
reader = { it ?: default() },
@ -60,7 +57,6 @@ fun <R : Any> ReadWriteProperty<Any?, R?>.notNull(default: () -> R): ReadWritePr
)
}
fun <M : MutableMeta<M>> M.item(default: Any? = null, key: Name? = null): MutableMetaDelegate<M> =
MutableMetaDelegate(this, key, default?.let { MetaItem.of(it) })
@ -117,4 +113,4 @@ fun <M : MutableMeta<M>> M.number(key: Name? = null, default: () -> Number) =
inline fun <M : MutableMeta<M>, reified E : Enum<E>> M.enum(default: E, key: Name? = null) =
item(default, key).transform { it.enum<E>()!! }
item(default, key).transform { it.enum<E>()!! }

7
dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/descriptors/ItemDescriptor.kt
View File

@ -95,6 +95,11 @@ class NodeDescriptor(config: Config = Config()) : ItemDescriptor(config) {
*/
var default by config.node()
/**
* An index field by which this node is identified in case of same name siblings construct
*/
var indexKey by config.string(DEFAULT_INDEX_KEY)
val items: Map<String, ItemDescriptor>
get() = config.getIndexed(ITEM_KEY).mapValues { (_, item) ->
val node = item.node ?: error("Node descriptor must be a node")
@ -182,6 +187,8 @@ class NodeDescriptor(config: Config = Config()) : ItemDescriptor(config) {
val ITEM_KEY = "item".asName()
val IS_NODE_KEY = "@isNode".asName()
const val DEFAULT_INDEX_KEY = "@index"
inline operator fun invoke(block: NodeDescriptor.() -> Unit) = NodeDescriptor().apply(block)
//TODO infer descriptor from spec

2
dataforge-meta/src/commonMain/kotlin/hep/dataforge/names/Name.kt
View File

@ -113,6 +113,8 @@ data class NameToken(val body: String, val index: String = "") {
}
}
fun NameToken.withIndex(newIndex: String) = NameToken(body, newIndex)
/**
* Convert a [String] to name parsing it and extracting name tokens and index syntax.
* This operation is rather heavy so it should be used with care in high performance code.

22
dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/JsonMetaTest.kt
View File

@ -1,5 +1,6 @@
package hep.dataforge.meta
import hep.dataforge.meta.descriptors.NodeDescriptor
import kotlinx.serialization.json.int
import kotlinx.serialization.json.json
import kotlinx.serialization.json.jsonArray
@ -17,23 +18,34 @@ class JsonMetaTest {
}
"nodeArray" to jsonArray {
+json {
"index" to 1
"index" to "1"
"value" to 2
}
+json {
"index" to 2
"index" to "2"
"value" to 3
}
+json {
"index" to 3
"index" to "3"
"value" to 4
}
}
}
val descriptor = NodeDescriptor{
node("nodeArray"){
indexKey = "index"
}
}
@Test
fun jsonMetaConversion() {
val meta = json.toMeta()
val reconstructed = meta.toJson()
println(json)
val meta = json.toMeta(descriptor)
//println(meta)
val reconstructed = meta.toJson(descriptor)
println(reconstructed)
assertEquals(2, reconstructed["nodeArray"]?.jsonArray?.get(1)?.jsonObject?.get("index")?.int)
assertEquals(json,reconstructed)
}
}

6
dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskModel.kt
View File

@ -36,7 +36,11 @@ data class TaskModel(
val dataDependencies = dependencies.filterIsInstance<DataDependency>()
val taskDependencies = dependencies.filterIsInstance<TaskDependency<*>>()
setIndexed("data".toName(), dataDependencies.map { it.toMeta() })
setIndexed("task".toName(), taskDependencies.map { it.toMeta() }) { taskDependencies[it].name.toString() }
setIndexed(
"task".toName(),
taskDependencies.map { it.toMeta() }) { _, index ->
taskDependencies[index].name.toString()
}
//TODO ensure all dependencies are listed
}
}