From 3d65a1b4090f7fd4ff9052765742f656391ec17f Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Thu, 8 Nov 2018 20:59:41 +0300 Subject: [PATCH] Switched to kotlinx serialization for json processing --- .../kotlin/hep/dataforge/context/Global.kt | 3 + .../hep/dataforge/context/JVMContext.kt | 1 + dataforge-meta-io/build.gradle | 9 +- .../hep/dataforge/meta/io/JsonMetaFormat.kt | 77 +++++++++++ .../hep/dataforge/meta/io/MetaFormat.kt | 16 +-- .../hep/dataforge/meta/io/MetaFormatTest.kt | 4 +- .../kotlin/hep/dataforge/meta/io/JSMeta.kt | 31 ----- .../hep/dataforge/meta/io/JSONMetaFormat.kt | 17 --- .../kotlin/hep/dataforge/meta/JSMetaTest.kt | 18 --- .../hep/dataforge/meta/io/JSONMetaFormat.kt | 124 ------------------ 10 files changed, 89 insertions(+), 211 deletions(-) create mode 100644 dataforge-meta-io/src/commonMain/kotlin/hep/dataforge/meta/io/JsonMetaFormat.kt delete mode 100644 dataforge-meta-io/src/jsMain/kotlin/hep/dataforge/meta/io/JSMeta.kt delete mode 100644 dataforge-meta-io/src/jsMain/kotlin/hep/dataforge/meta/io/JSONMetaFormat.kt delete mode 100644 dataforge-meta-io/src/jsTest/kotlin/hep/dataforge/meta/JSMetaTest.kt delete mode 100644 dataforge-meta-io/src/jvmMain/kotlin/hep/dataforge/meta/io/JSONMetaFormat.kt diff --git a/dataforge-context/src/jvmMain/kotlin/hep/dataforge/context/Global.kt b/dataforge-context/src/jvmMain/kotlin/hep/dataforge/context/Global.kt index 7564b5fb..d560ddd2 100644 --- a/dataforge-context/src/jvmMain/kotlin/hep/dataforge/context/Global.kt +++ b/dataforge-context/src/jvmMain/kotlin/hep/dataforge/context/Global.kt @@ -15,6 +15,9 @@ */ package hep.dataforge.context +import hep.dataforge.meta.Meta +import hep.dataforge.meta.buildMeta +import hep.dataforge.names.toName import java.util.* import kotlin.collections.HashMap diff --git a/dataforge-context/src/jvmMain/kotlin/hep/dataforge/context/JVMContext.kt b/dataforge-context/src/jvmMain/kotlin/hep/dataforge/context/JVMContext.kt index 448eb63e..942fafb8 100644 --- a/dataforge-context/src/jvmMain/kotlin/hep/dataforge/context/JVMContext.kt +++ b/dataforge-context/src/jvmMain/kotlin/hep/dataforge/context/JVMContext.kt @@ -15,6 +15,7 @@ */ package hep.dataforge.context +import hep.dataforge.meta.* import mu.KLogger import mu.KotlinLogging import java.lang.ref.WeakReference diff --git a/dataforge-meta-io/build.gradle b/dataforge-meta-io/build.gradle index 242532bb..1af12d27 100644 --- a/dataforge-meta-io/build.gradle +++ b/dataforge-meta-io/build.gradle @@ -1,6 +1,6 @@ plugins { id 'kotlin-multiplatform' - //id 'kotlinx-serialization' + id 'kotlinx-serialization' } repositories { @@ -21,7 +21,7 @@ kotlin { dependencies { api project(":dataforge-meta") //implementation 'org.jetbrains.kotlin:kotlin-reflect' - //implementation "org.jetbrains.kotlinx:kotlinx-serialization-runtime-common:$serialization_version" + implementation "org.jetbrains.kotlinx:kotlinx-serialization-runtime-common:$serialization_version" implementation "org.jetbrains.kotlinx:kotlinx-io:$kotlinx_io_version" } } @@ -33,8 +33,7 @@ kotlin { } jvmMain { dependencies { - implementation 'com.github.cliftonlabs:json-simple:3.0.2' - //implementation "org.jetbrains.kotlinx:kotlinx-serialization-runtime:$serialization_version" + implementation "org.jetbrains.kotlinx:kotlinx-serialization-runtime:$serialization_version" implementation "org.jetbrains.kotlinx:kotlinx-io-jvm:$kotlinx_io_version" } } @@ -46,7 +45,7 @@ kotlin { } jsMain { dependencies { - //implementation "org.jetbrains.kotlinx:kotlinx-serialization-runtime-js:$serialization_version" + implementation "org.jetbrains.kotlinx:kotlinx-serialization-runtime-js:$serialization_version" implementation "org.jetbrains.kotlinx:kotlinx-io-js:$kotlinx_io_version" } } diff --git a/dataforge-meta-io/src/commonMain/kotlin/hep/dataforge/meta/io/JsonMetaFormat.kt b/dataforge-meta-io/src/commonMain/kotlin/hep/dataforge/meta/io/JsonMetaFormat.kt new file mode 100644 index 00000000..ec588160 --- /dev/null +++ b/dataforge-meta-io/src/commonMain/kotlin/hep/dataforge/meta/io/JsonMetaFormat.kt @@ -0,0 +1,77 @@ +package hep.dataforge.meta.io + +import hep.dataforge.meta.Meta +import hep.dataforge.meta.MetaItem +import hep.dataforge.names.NameToken +import hep.dataforge.values.* +import kotlinx.io.core.Input +import kotlinx.io.core.Output +import kotlinx.io.core.readText +import kotlinx.io.core.writeText +import kotlinx.serialization.json.* + + +object JSONMetaFormat : MetaFormat { + override val name: String = "json" + override val key: Short = 0x4a53//"JS" + + override fun write(meta: Meta, out: Output) { + val str = meta.toJson().toString() + out.writeText(str) + } + + override fun read(input: Input): Meta { + val str = input.readText() + val json = JsonTreeParser.parse(str) + return json.toMeta() + } +} + +fun Value.toJson(): JsonElement { + return when (type) { + ValueType.NUMBER -> JsonPrimitive(number) + ValueType.STRING -> JsonPrimitive(string) + ValueType.BOOLEAN -> JsonPrimitive(boolean) + ValueType.NULL -> JsonNull + } +} + +fun Meta.toJson(): JsonObject { + val map = this.items.mapValues { entry -> + val value = entry.value + when (value) { + is MetaItem.ValueItem -> value.value.toJson() + is MetaItem.NodeItem -> value.node.toJson() + } + }.mapKeys { it.key.toString() } + return JsonObject(map) +} + +fun JsonObject.toMeta() = JsonMeta(this) + +private fun JsonPrimitive.toValue(): Value { + return when (this) { + JsonNull -> Null + else -> this.content.parseValue() // Optimize number and boolean parsing + } +} + +class JsonMeta(val json: JsonObject) : Meta { + override val items: Map> by lazy { + json.mapKeys { NameToken(it.key) }.mapValues { entry -> + val element = entry.value + when (element) { + is JsonPrimitive -> MetaItem.ValueItem(element.toValue()) + is JsonObject -> MetaItem.NodeItem(element.toMeta()) + is JsonArray -> { + if (element.all { it is JsonPrimitive }) { + val value = ListValue(element.map { (it as JsonPrimitive).toValue() }) + MetaItem.ValueItem(value) + } else { + TODO("mixed nodes json") + } + } + } + } + } +} \ No newline at end of file diff --git a/dataforge-meta-io/src/commonMain/kotlin/hep/dataforge/meta/io/MetaFormat.kt b/dataforge-meta-io/src/commonMain/kotlin/hep/dataforge/meta/io/MetaFormat.kt index 4f78661b..ec9d9f2b 100644 --- a/dataforge-meta-io/src/commonMain/kotlin/hep/dataforge/meta/io/MetaFormat.kt +++ b/dataforge-meta-io/src/commonMain/kotlin/hep/dataforge/meta/io/MetaFormat.kt @@ -15,28 +15,16 @@ interface MetaFormat { fun read(input: Input): Meta } -fun MetaFormat.stringify(meta: Meta): String { +fun Meta.asString(format: MetaFormat = JSONMetaFormat): String{ val builder = BytePacketBuilder() - write(meta,builder) + format.write(this,builder) return builder.build().readText() } -fun Meta.asString() = JSONMetaFormat.stringify(this) - fun MetaFormat.parse(str: String): Meta{ return read(ByteReadPacket(str.toByteArray())) } -internal expect fun writeJson(meta: Meta, out: Output) -internal expect fun readJson(input: Input, length: Int = -1): Meta - -object JSONMetaFormat : MetaFormat { - override val name: String = "json" - override val key: Short = 0x4a53//"JS" - - override fun write(meta: Meta, out: Output) = writeJson(meta, out) - override fun read(input: Input): Meta = readJson(input) -} object BinaryMetaFormat : MetaFormat { override val name: String = "bin" diff --git a/dataforge-meta-io/src/commonTest/kotlin/hep/dataforge/meta/io/MetaFormatTest.kt b/dataforge-meta-io/src/commonTest/kotlin/hep/dataforge/meta/io/MetaFormatTest.kt index 72851d90..a077f920 100644 --- a/dataforge-meta-io/src/commonTest/kotlin/hep/dataforge/meta/io/MetaFormatTest.kt +++ b/dataforge-meta-io/src/commonTest/kotlin/hep/dataforge/meta/io/MetaFormatTest.kt @@ -14,7 +14,7 @@ class MetaFormatTest{ "c" to 11.1 } } - val string = BinaryMetaFormat.stringify(meta) + val string = meta.asString(BinaryMetaFormat) val result = BinaryMetaFormat.parse(string) assertEquals(meta,result) } @@ -28,7 +28,7 @@ class MetaFormatTest{ "c" to 11.1 } } - val string = JSONMetaFormat.stringify(meta) + val string = meta.asString(JSONMetaFormat) val result = JSONMetaFormat.parse(string) assertEquals(meta,result) } diff --git a/dataforge-meta-io/src/jsMain/kotlin/hep/dataforge/meta/io/JSMeta.kt b/dataforge-meta-io/src/jsMain/kotlin/hep/dataforge/meta/io/JSMeta.kt deleted file mode 100644 index e024c319..00000000 --- a/dataforge-meta-io/src/jsMain/kotlin/hep/dataforge/meta/io/JSMeta.kt +++ /dev/null @@ -1,31 +0,0 @@ -package hep.dataforge.meta.io - -import hep.dataforge.meta.Meta -import hep.dataforge.meta.MetaItem -import hep.dataforge.names.NameToken -import hep.dataforge.values.Value - -/** - * Represent any js object as meta - */ -class JSMeta(val obj: Any) : Meta { - override val items: Map> - get() = listKeys(obj).map { NameToken(it) }.associateWith { convert(js("obj[it]")) } - - private fun listKeys(obj: Any): List = js("Object").keys(obj) as List - - private fun isList(obj: Any): Boolean = js("Array").isArray(obj) as Boolean - - private fun isPrimitive(@Suppress("UNUSED_PARAMETER") obj: Any?): Boolean = js("obj !== Object(obj)") as Boolean - - private fun convert(obj: Any?): MetaItem { - return when (obj) { - null, isPrimitive(obj), is Number, is String, is Boolean -> MetaItem.ValueItem(Value.of(obj)) - isList(obj) -> { - val list = obj as List<*> - MetaItem.ValueItem(Value.of(list)) - } - else -> MetaItem.NodeItem(JSMeta(obj)) - } - } -} \ No newline at end of file diff --git a/dataforge-meta-io/src/jsMain/kotlin/hep/dataforge/meta/io/JSONMetaFormat.kt b/dataforge-meta-io/src/jsMain/kotlin/hep/dataforge/meta/io/JSONMetaFormat.kt deleted file mode 100644 index 60b1446e..00000000 --- a/dataforge-meta-io/src/jsMain/kotlin/hep/dataforge/meta/io/JSONMetaFormat.kt +++ /dev/null @@ -1,17 +0,0 @@ -package hep.dataforge.meta.io - -import hep.dataforge.meta.Meta -import kotlinx.io.core.Input -import kotlinx.io.core.Output -import kotlinx.io.core.readText -import kotlinx.io.core.writeText -import kotlin.js.Json - -internal actual fun writeJson(meta: Meta, out: Output) { - out.writeText(JSON.stringify(meta)) -} - -internal actual fun readJson(input: Input, length: Int): Meta { - val json: Json = JSON.parse(input.readText(max = if (length > 0) length else Int.MAX_VALUE)) - return JSMeta(json) -} \ No newline at end of file diff --git a/dataforge-meta-io/src/jsTest/kotlin/hep/dataforge/meta/JSMetaTest.kt b/dataforge-meta-io/src/jsTest/kotlin/hep/dataforge/meta/JSMetaTest.kt deleted file mode 100644 index e925b92c..00000000 --- a/dataforge-meta-io/src/jsTest/kotlin/hep/dataforge/meta/JSMetaTest.kt +++ /dev/null @@ -1,18 +0,0 @@ -package hep.dataforge.meta - -import hep.dataforge.meta.io.JSMeta -import kotlin.js.json -import kotlin.test.Test -import kotlin.test.assertEquals - -class JSMetaTest{ - @Test - fun testConverstion(){ - val test = json( - "a" to 2, - "b" to "ddd" - ) - val meta = JSMeta(test) - assertEquals(2, meta["a"]!!.int) - } -} \ No newline at end of file diff --git a/dataforge-meta-io/src/jvmMain/kotlin/hep/dataforge/meta/io/JSONMetaFormat.kt b/dataforge-meta-io/src/jvmMain/kotlin/hep/dataforge/meta/io/JSONMetaFormat.kt deleted file mode 100644 index ae597b53..00000000 --- a/dataforge-meta-io/src/jvmMain/kotlin/hep/dataforge/meta/io/JSONMetaFormat.kt +++ /dev/null @@ -1,124 +0,0 @@ -package hep.dataforge.meta.io - -import com.github.cliftonlabs.json_simple.JsonArray -import com.github.cliftonlabs.json_simple.JsonObject -import com.github.cliftonlabs.json_simple.Jsoner -import hep.dataforge.meta.* -import hep.dataforge.names.toName -import hep.dataforge.values.* -import kotlinx.io.core.* -import java.io.ByteArrayInputStream -import java.io.InputStreamReader -import java.io.Reader -import java.nio.ByteBuffer -import java.text.ParseException - -internal actual fun writeJson(meta: Meta, out: Output) { - val json = meta.toJson() - val string = Jsoner.prettyPrint(Jsoner.serialize(json)) - out.writeText(string) -} - -private fun Value.toJson(): Any { - return if (list.size == 1) { - when (type) { - ValueType.NUMBER -> number - ValueType.BOOLEAN -> boolean - else -> string - } - } else { - JsonArray().apply { - list.forEach { add(it.toJson()) } - } - } -} - -fun Meta.toJson(): JsonObject { - val builder = JsonObject() - items.forEach { name, item -> - when (item) { - is MetaItem.ValueItem -> builder[name.toString()] = item.value.toJson() - is MetaItem.NodeItem -> builder[name.toString()] = item.node.toJson() - } - } - return builder -} - - -internal actual fun readJson(input: Input, length: Int): Meta { - return if (length == 0) { - EmptyMeta - } else { - val json = if (length > 0) { - //Read into intermediate buffer - val buffer = ByteArray(length) - input.readAvailable(buffer, length) - Jsoner.deserialize(InputStreamReader(ByteArrayInputStream(buffer), Charsets.UTF_8)) as JsonObject - } else { - //automatic - val reader = object : Reader() { - override fun close() { - input.close() - } - - override fun read(cbuf: CharArray, off: Int, len: Int): Int { - val buffer = ByteBuffer.allocate(len) - val res = input.readAvailable(buffer) - val chars = String(buffer.array()).toCharArray() - System.arraycopy(chars, 0, cbuf, off, chars.size) - return res - } - - } - Jsoner.deserialize(reader) as JsonObject - } - json.toMeta() - } -} - -@Throws(ParseException::class) -private fun JsonObject.toMeta(): Meta { - return buildMeta { - this@toMeta.forEach { key, value -> appendValue(key as String, value) } - } -} - -private fun JsonArray.toListValue(): Value { - val list: List = this.map { value -> - when (value) { - null -> Null - is JsonArray -> value.toListValue() - is Number -> NumberValue(value) - is Boolean -> if (value) True else False - is String -> LazyParsedValue(value) - is JsonObject -> error("Object values inside multidimensional arrays are not allowed") - else -> error("Unknown token $value in json") - } - } - return Value.of(list) -} - -private fun MetaBuilder.appendValue(key: String, value: Any?) { - when (value) { - is JsonObject -> this[key] = value.toMeta() - is JsonArray -> { - if (value.none { it is JsonObject }) { - //If all values are primitives or arrays - this[key] = value.toListValue() - } else { - val list = value.map { - when (it) { - is JsonObject -> it.toMeta() - is JsonArray -> it.toListValue().toMeta() - else -> Value.of(it).toMeta() - } - } - setIndexed(key.toName(), list) - } - } - is Number -> this[key] = NumberValue(value) - is Boolean -> this[key] = value - is String -> this[key] = LazyParsedValue(value) - //ignore anything else - } -}