kscience/dataforge-core
8
0
Fork 0
Code Issues Pull Requests 1 Packages Projects Releases 12 Wiki Activity

Switched to kotlinx serialization for json processing

Browse Source
This commit is contained in:
Alexander Nozik 2018-11-08 20:59:41 +03:00
parent d3ce88eb3f
commit 3d65a1b409
10 changed files with 89 additions and 211 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
dataforge-context/src/jvmMain/kotlin/hep/dataforge/context/Global.kt
View File

@ -15,6 +15,9 @@
*/
package hep.dataforge.context
import hep.dataforge.meta.Meta
import hep.dataforge.meta.buildMeta
import hep.dataforge.names.toName
import java.util.*
import kotlin.collections.HashMap

1
dataforge-context/src/jvmMain/kotlin/hep/dataforge/context/JVMContext.kt
View File

@ -15,6 +15,7 @@
*/
package hep.dataforge.context
import hep.dataforge.meta.*
import mu.KLogger
import mu.KotlinLogging
import java.lang.ref.WeakReference

9
dataforge-meta-io/build.gradle
View File

@ -1,6 +1,6 @@
plugins {
id 'kotlin-multiplatform'
//id 'kotlinx-serialization'
id 'kotlinx-serialization'
}
repositories {
@ -21,7 +21,7 @@ kotlin {
dependencies {
api project(":dataforge-meta")
//implementation 'org.jetbrains.kotlin:kotlin-reflect'
//implementation "org.jetbrains.kotlinx:kotlinx-serialization-runtime-common:$serialization_version"
implementation "org.jetbrains.kotlinx:kotlinx-serialization-runtime-common:$serialization_version"
implementation "org.jetbrains.kotlinx:kotlinx-io:$kotlinx_io_version"
}
}
@ -33,8 +33,7 @@ kotlin {
}
jvmMain {
dependencies {
implementation 'com.github.cliftonlabs:json-simple:3.0.2'
//implementation "org.jetbrains.kotlinx:kotlinx-serialization-runtime:$serialization_version"
implementation "org.jetbrains.kotlinx:kotlinx-serialization-runtime:$serialization_version"
implementation "org.jetbrains.kotlinx:kotlinx-io-jvm:$kotlinx_io_version"
}
}
@ -46,7 +45,7 @@ kotlin {
}
jsMain {
dependencies {
//implementation "org.jetbrains.kotlinx:kotlinx-serialization-runtime-js:$serialization_version"
implementation "org.jetbrains.kotlinx:kotlinx-serialization-runtime-js:$serialization_version"
implementation "org.jetbrains.kotlinx:kotlinx-io-js:$kotlinx_io_version"
}
}

77
dataforge-meta-io/src/commonMain/kotlin/hep/dataforge/meta/io/JsonMetaFormat.kt Normal file
View File

@ -0,0 +1,77 @@
package hep.dataforge.meta.io
import hep.dataforge.meta.Meta
import hep.dataforge.meta.MetaItem
import hep.dataforge.names.NameToken
import hep.dataforge.values.*
import kotlinx.io.core.Input
import kotlinx.io.core.Output
import kotlinx.io.core.readText
import kotlinx.io.core.writeText
import kotlinx.serialization.json.*
object JSONMetaFormat : MetaFormat {
override val name: String = "json"
override val key: Short = 0x4a53//"JS"
override fun write(meta: Meta, out: Output) {
val str = meta.toJson().toString()
out.writeText(str)
}
override fun read(input: Input): Meta {
val str = input.readText()
val json = JsonTreeParser.parse(str)
return json.toMeta()
}
}
fun Value.toJson(): JsonElement {
return when (type) {
ValueType.NUMBER -> JsonPrimitive(number)
ValueType.STRING -> JsonPrimitive(string)
ValueType.BOOLEAN -> JsonPrimitive(boolean)
ValueType.NULL -> JsonNull
}
}
fun Meta.toJson(): JsonObject {
val map = this.items.mapValues { entry ->
val value = entry.value
when (value) {
is MetaItem.ValueItem -> value.value.toJson()
is MetaItem.NodeItem -> value.node.toJson()
}
}.mapKeys { it.key.toString() }
return JsonObject(map)
}
fun JsonObject.toMeta() = JsonMeta(this)
private fun JsonPrimitive.toValue(): Value {
return when (this) {
JsonNull -> Null
else -> this.content.parseValue() // Optimize number and boolean parsing
}
}
class JsonMeta(val json: JsonObject) : Meta {
override val items: Map<NameToken, MetaItem<out Meta>> by lazy {
json.mapKeys { NameToken(it.key) }.mapValues { entry ->
val element = entry.value
when (element) {
is JsonPrimitive -> MetaItem.ValueItem<JsonMeta>(element.toValue())
is JsonObject -> MetaItem.NodeItem(element.toMeta())
is JsonArray -> {
if (element.all { it is JsonPrimitive }) {
val value = ListValue(element.map { (it as JsonPrimitive).toValue() })
MetaItem.ValueItem<JsonMeta>(value)
} else {
TODO("mixed nodes json")
}
}
}
}
}
}

16
dataforge-meta-io/src/commonMain/kotlin/hep/dataforge/meta/io/MetaFormat.kt
View File

@ -15,28 +15,16 @@ interface MetaFormat {
fun read(input: Input): Meta
}
fun MetaFormat.stringify(meta: Meta): String {
fun Meta.asString(format: MetaFormat = JSONMetaFormat): String{
val builder = BytePacketBuilder()
write(meta,builder)
format.write(this,builder)
return builder.build().readText()
}
fun Meta.asString() = JSONMetaFormat.stringify(this)
fun MetaFormat.parse(str: String): Meta{
return read(ByteReadPacket(str.toByteArray()))
}
internal expect fun writeJson(meta: Meta, out: Output)
internal expect fun readJson(input: Input, length: Int = -1): Meta
object JSONMetaFormat : MetaFormat {
override val name: String = "json"
override val key: Short = 0x4a53//"JS"
override fun write(meta: Meta, out: Output) = writeJson(meta, out)
override fun read(input: Input): Meta = readJson(input)
}
object BinaryMetaFormat : MetaFormat {
override val name: String = "bin"

4
dataforge-meta-io/src/commonTest/kotlin/hep/dataforge/meta/io/MetaFormatTest.kt
View File

@ -14,7 +14,7 @@ class MetaFormatTest{
"c" to 11.1
}
}
val string = BinaryMetaFormat.stringify(meta)
val string = meta.asString(BinaryMetaFormat)
val result = BinaryMetaFormat.parse(string)
assertEquals(meta,result)
}
@ -28,7 +28,7 @@ class MetaFormatTest{
"c" to 11.1
}
}
val string = JSONMetaFormat.stringify(meta)
val string = meta.asString(JSONMetaFormat)
val result = JSONMetaFormat.parse(string)
assertEquals(meta,result)
}

31
dataforge-meta-io/src/jsMain/kotlin/hep/dataforge/meta/io/JSMeta.kt
View File

@ -1,31 +0,0 @@
package hep.dataforge.meta.io
import hep.dataforge.meta.Meta
import hep.dataforge.meta.MetaItem
import hep.dataforge.names.NameToken
import hep.dataforge.values.Value
/**
* Represent any js object as meta
*/
class JSMeta(val obj: Any) : Meta {
override val items: Map<NameToken, MetaItem<out Meta>>
get() = listKeys(obj).map { NameToken(it) }.associateWith { convert(js("obj[it]")) }
private fun listKeys(obj: Any): List<String> = js("Object").keys(obj) as List<String>
private fun isList(obj: Any): Boolean = js("Array").isArray(obj) as Boolean
private fun isPrimitive(@Suppress("UNUSED_PARAMETER") obj: Any?): Boolean = js("obj !== Object(obj)") as Boolean
private fun convert(obj: Any?): MetaItem<out Meta> {
return when (obj) {
null, isPrimitive(obj), is Number, is String, is Boolean -> MetaItem.ValueItem<JSMeta>(Value.of(obj))
isList(obj) -> {
val list = obj as List<*>
MetaItem.ValueItem<JSMeta>(Value.of(list))
}
else -> MetaItem.NodeItem(JSMeta(obj))
}
}
}

17
dataforge-meta-io/src/jsMain/kotlin/hep/dataforge/meta/io/JSONMetaFormat.kt
View File

@ -1,17 +0,0 @@
package hep.dataforge.meta.io
import hep.dataforge.meta.Meta
import kotlinx.io.core.Input
import kotlinx.io.core.Output
import kotlinx.io.core.readText
import kotlinx.io.core.writeText
import kotlin.js.Json
internal actual fun writeJson(meta: Meta, out: Output) {
out.writeText(JSON.stringify(meta))
}
internal actual fun readJson(input: Input, length: Int): Meta {
val json: Json = JSON.parse(input.readText(max = if (length > 0) length else Int.MAX_VALUE))
return JSMeta(json)
}

18
dataforge-meta-io/src/jsTest/kotlin/hep/dataforge/meta/JSMetaTest.kt
View File

@ -1,18 +0,0 @@
package hep.dataforge.meta
import hep.dataforge.meta.io.JSMeta
import kotlin.js.json
import kotlin.test.Test
import kotlin.test.assertEquals
class JSMetaTest{
@Test
fun testConverstion(){
val test = json(
"a" to 2,
"b" to "ddd"
)
val meta = JSMeta(test)
assertEquals(2, meta["a"]!!.int)
}
}

124
dataforge-meta-io/src/jvmMain/kotlin/hep/dataforge/meta/io/JSONMetaFormat.kt
View File

@ -1,124 +0,0 @@
package hep.dataforge.meta.io
import com.github.cliftonlabs.json_simple.JsonArray
import com.github.cliftonlabs.json_simple.JsonObject
import com.github.cliftonlabs.json_simple.Jsoner
import hep.dataforge.meta.*
import hep.dataforge.names.toName
import hep.dataforge.values.*
import kotlinx.io.core.*
import java.io.ByteArrayInputStream
import java.io.InputStreamReader
import java.io.Reader
import java.nio.ByteBuffer
import java.text.ParseException
internal actual fun writeJson(meta: Meta, out: Output) {
val json = meta.toJson()
val string = Jsoner.prettyPrint(Jsoner.serialize(json))
out.writeText(string)
}
private fun Value.toJson(): Any {
return if (list.size == 1) {
when (type) {
ValueType.NUMBER -> number
ValueType.BOOLEAN -> boolean
else -> string
}
} else {
JsonArray().apply {
list.forEach { add(it.toJson()) }
}
}
}
fun Meta.toJson(): JsonObject {
val builder = JsonObject()
items.forEach { name, item ->
when (item) {
is MetaItem.ValueItem -> builder[name.toString()] = item.value.toJson()
is MetaItem.NodeItem -> builder[name.toString()] = item.node.toJson()
}
}
return builder
}
internal actual fun readJson(input: Input, length: Int): Meta {
return if (length == 0) {
EmptyMeta
} else {
val json = if (length > 0) {
//Read into intermediate buffer
val buffer = ByteArray(length)
input.readAvailable(buffer, length)
Jsoner.deserialize(InputStreamReader(ByteArrayInputStream(buffer), Charsets.UTF_8)) as JsonObject
} else {
//automatic
val reader = object : Reader() {
override fun close() {
input.close()
}
override fun read(cbuf: CharArray, off: Int, len: Int): Int {
val buffer = ByteBuffer.allocate(len)
val res = input.readAvailable(buffer)
val chars = String(buffer.array()).toCharArray()
System.arraycopy(chars, 0, cbuf, off, chars.size)
return res
}
}
Jsoner.deserialize(reader) as JsonObject
}
json.toMeta()
}
}
@Throws(ParseException::class)
private fun JsonObject.toMeta(): Meta {
return buildMeta {
this@toMeta.forEach { key, value -> appendValue(key as String, value) }
}
}
private fun JsonArray.toListValue(): Value {
val list: List<Value> = this.map { value ->
when (value) {
null -> Null
is JsonArray -> value.toListValue()
is Number -> NumberValue(value)
is Boolean -> if (value) True else False
is String -> LazyParsedValue(value)
is JsonObject -> error("Object values inside multidimensional arrays are not allowed")
else -> error("Unknown token $value in json")
}
}
return Value.of(list)
}
private fun MetaBuilder.appendValue(key: String, value: Any?) {
when (value) {
is JsonObject -> this[key] = value.toMeta()
is JsonArray -> {
if (value.none { it is JsonObject }) {
//If all values are primitives or arrays
this[key] = value.toListValue()
} else {
val list = value.map<Any, Meta> {
when (it) {
is JsonObject -> it.toMeta()
is JsonArray -> it.toListValue().toMeta()
else -> Value.of(it).toMeta()
}
}
setIndexed(key.toName(), list)
}
}
is Number -> this[key] = NumberValue(value)
is Boolean -> this[key] = value
is String -> this[key] = LazyParsedValue(value)
//ignore anything else
}
}