custom JSON serialization (no universal serialization yet)

This commit is contained in:
Alexander Nozik 2018-09-23 21:09:35 +03:00
parent 1926d157ee
commit a5a4b67c97
12 changed files with 411 additions and 181 deletions

2
.gitignore vendored
View File

@ -3,7 +3,7 @@
*.iws
out/
.gradle
/build/
**/build/
!gradle-wrapper.jar

View File

@ -1,26 +1,40 @@
buildscript {
ext.kotlin_version = '1.2.70'
ext.serialization_version = '0.6.2'
ext.kotlin_version = '1.3.0-rc-57'
ext.serialization_version = '0.8.0-rc13'
repositories {
jcenter()
maven { url "https://kotlin.bintray.com/kotlinx" }
maven {
url = "http://dl.bintray.com/kotlin/kotlin-eap"
}
}
dependencies {
classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlin_version"
classpath "org.jetbrains.kotlinx:kotlinx-gradle-serialization-plugin:$serialization_version"
classpath "org.jetbrains.kotlin:kotlin-serialization:$kotlin_version"
}
}
//plugins {
// id 'kotlin-platform-common' version "$kotlin_version" apply false
// id 'kotlin-platform-jvm' version "$kotlin_version" apply false
// id 'kotlin-platform-js' version "$kotlin_version" apply false
// id 'kotlinx-serialization' version "$kotlin_version" apply false
//}
description = "The basic interfaces for DataForge meta-data"
group 'hep.dataforge'
version '0.1.1-SNAPSHOT'
subprojects{
allprojects {
repositories {
jcenter()
maven { url = "http://dl.bintray.com/kotlin/kotlin-eap" }
maven { url "https://kotlin.bintray.com/kotlinx" }
//maven { url 'https://jitpack.io' }
}
}
subprojects {
apply plugin: 'kotlinx-serialization'
}

View File

@ -1,6 +1,5 @@
plugins {
plugins{
id 'kotlin-platform-common'
id 'kotlinx-serialization'
}
dependencies {
@ -8,10 +7,4 @@ dependencies {
compile "org.jetbrains.kotlinx:kotlinx-serialization-runtime-common:$serialization_version"
testCompile "org.jetbrains.kotlin:kotlin-test-annotations-common"
testCompile "org.jetbrains.kotlin:kotlin-test-common"
}
kotlin {
experimental {
coroutines "enable"
}
}

View File

@ -2,7 +2,6 @@ package hep.dataforge.meta
import hep.dataforge.names.Name
import hep.dataforge.names.toName
import kotlinx.serialization.Serializable
/**
* A member of the meta tree. Could be represented as one of following:
@ -11,9 +10,34 @@ import kotlinx.serialization.Serializable
* * a list of nodes
*/
sealed class MetaItem<M : Meta> {
class ValueItem<M : Meta>(val value: Value) : MetaItem<M>()
class SingleNodeItem<M : Meta>(val node: M) : MetaItem<M>()
class MultiNodeItem<M : Meta>(val nodes: List<M>) : MetaItem<M>()
class ValueItem<M : Meta>(val value: Value) : MetaItem<M>(){
override fun equals(other: Any?): Boolean {
return this.value == (other as? ValueItem<*>)?.value
}
override fun hashCode(): Int {
return value.hashCode()
}
}
class SingleNodeItem<M : Meta>(val node: M) : MetaItem<M>(){
override fun equals(other: Any?): Boolean {
return this.node == (other as? SingleNodeItem<*>)?.node
}
override fun hashCode(): Int {
return node.hashCode()
}
}
class MultiNodeItem<M : Meta>(val nodes: List<M>) : MetaItem<M>(){
override fun equals(other: Any?): Boolean {
return this.nodes == (other as? MultiNodeItem<*>)?.nodes
}
override fun hashCode(): Int {
return nodes.hashCode()
}
}
}
operator fun <M : Meta> List<M>.get(query: String): M? {
@ -31,7 +55,6 @@ operator fun <M : Meta> List<M>.get(query: String): M? {
* * [MetaItem.SingleNodeItem] single node
* * [MetaItem.MultiNodeItem] multi-value node
*/
@Serializable
interface Meta {
val items: Map<String, MetaItem<out Meta>>
}
@ -62,6 +85,22 @@ abstract class MetaNode<M : MetaNode<M>> : Meta {
}
operator fun get(key: String): MetaItem<M>? = get(key.toName())
override fun equals(other: Any?): Boolean {
if (this === other) return true
if (other !is Meta) return false
return this.items == other.items
}
override fun hashCode(): Int {
return items.hashCode()
}
override fun toString(): String {
return toJSON().toString()
}
}
/**
@ -69,17 +108,23 @@ abstract class MetaNode<M : MetaNode<M>> : Meta {
*
* If the argument is possibly mutable node, it is copied on creation
*/
class SealedMeta(meta: Meta) : MetaNode<SealedMeta>() {
override val items: Map<String, MetaItem<SealedMeta>> = if (meta is SealedMeta) {
meta.items
} else {
meta.items.mapValues { entry ->
val item = entry.value
when (item) {
is MetaItem.ValueItem -> MetaItem.ValueItem(item.value)
is MetaItem.SingleNodeItem -> MetaItem.SingleNodeItem(SealedMeta(item.node))
is MetaItem.MultiNodeItem -> MetaItem.MultiNodeItem(item.nodes.map { SealedMeta(it) })
class SealedMeta internal constructor(override val items: Map<String, MetaItem<SealedMeta>>) : MetaNode<SealedMeta>() {
companion object {
fun seal(meta: Meta): SealedMeta {
val items = if (meta is SealedMeta) {
meta.items
} else {
meta.items.mapValues { entry ->
val item = entry.value
when (item) {
is MetaItem.ValueItem -> MetaItem.ValueItem(item.value)
is MetaItem.SingleNodeItem -> MetaItem.SingleNodeItem(seal(item.node))
is MetaItem.MultiNodeItem -> MetaItem.MultiNodeItem(item.nodes.map { seal(it) })
}
}
}
return SealedMeta(items)
}
}
}
@ -87,12 +132,51 @@ class SealedMeta(meta: Meta) : MetaNode<SealedMeta>() {
/**
* Generate sealed node from [this]. If it is already sealed return it as is
*/
fun Meta.seal(): SealedMeta = this as? SealedMeta ?: SealedMeta(this)
fun Meta.seal(): SealedMeta = this as? SealedMeta ?: SealedMeta.seal(this)
object EmptyMeta : Meta {
override val items: Map<String, MetaItem<out Meta>> = emptyMap()
}
/**
* Unsafe methods to access values and nodes directly from [MetaItem]
*/
val MetaItem<*>.value
get() = (this as? MetaItem.ValueItem)?.value ?: error("Trying to interpret node meta item as value item")
val MetaItem<*>.string get() = value.string
val MetaItem<*>.boolean get() = value.boolean
val MetaItem<*>.number get() = value.number
val MetaItem<*>.double get() = number.toDouble()
val MetaItem<*>.int get() = number.toInt()
val MetaItem<*>.long get() = number.toLong()
val <M : Meta> MetaItem<M>.node: M
get() = when (this) {
is MetaItem.ValueItem -> error("Trying to interpret value meta item as node item")
is MetaItem.SingleNodeItem -> node
is MetaItem.MultiNodeItem -> nodes.first()
}
/**
* Utility method to access item content as list of nodes.
* Returns empty list if it is value item.
*/
val <M : Meta> MetaItem<M>.nodes: List<M>
get() = when (this) {
is MetaItem.ValueItem -> emptyList()//error("Trying to interpret value meta item as node item")
is MetaItem.SingleNodeItem -> listOf(node)
is MetaItem.MultiNodeItem -> nodes
}
fun <M : Meta> MetaItem<M>.indexOf(meta: M): Int {
return when (this) {
is MetaItem.ValueItem -> -1
is MetaItem.SingleNodeItem -> if (node == meta) 0 else -1
is MetaItem.MultiNodeItem -> nodes.indexOf(meta)
}
}
/**
* Generic meta-holder object
*/

View File

@ -10,9 +10,20 @@ class MetaBuilder : MutableMetaNode<MetaBuilder>() {
override fun empty(): MetaBuilder = MetaBuilder()
infix fun String.to(value: Any) {
if (value is Meta) {
this@MetaBuilder[this] = value
}
this@MetaBuilder[this] = Value.of(value)
}
infix fun String.to(meta: Meta) {
this@MetaBuilder[this] = meta
}
infix fun String.to(value: Iterable<Meta>) {
this@MetaBuilder[this] = value.toList()
}
infix fun String.to(metaBuilder: MetaBuilder.() -> Unit) {
this@MetaBuilder[this] = MetaBuilder().apply(metaBuilder)
}

View File

@ -1,40 +0,0 @@
package hep.dataforge.meta
/**
* Unsafe methods to access values and nodes directly from [MetaItem]
*/
val MetaItem<*>.value
get() = (this as? MetaItem.ValueItem)?.value ?: error("Trying to interpret node meta item as value item")
val MetaItem<*>.string get() = value.string
val MetaItem<*>.boolean get() = value.boolean
val MetaItem<*>.number get() = value.number
val MetaItem<*>.double get() = number.toDouble()
val MetaItem<*>.int get() = number.toInt()
val MetaItem<*>.long get() = number.toLong()
val <M : Meta> MetaItem<M>.node: M
get() = when (this) {
is MetaItem.ValueItem -> error("Trying to interpret value meta item as node item")
is MetaItem.SingleNodeItem -> node
is MetaItem.MultiNodeItem -> nodes.first()
}
/**
* Utility method to access item content as list of nodes.
* Returns empty list if it is value item.
*/
val <M : Meta> MetaItem<M>.nodes: List<M>
get() = when (this) {
is MetaItem.ValueItem -> emptyList()//error("Trying to interpret value meta item as node item")
is MetaItem.SingleNodeItem -> listOf(node)
is MetaItem.MultiNodeItem -> nodes
}
fun <M : Meta> MetaItem<M>.indexOf(meta: M): Int {
return when (this) {
is MetaItem.ValueItem -> -1
is MetaItem.SingleNodeItem -> if (node == meta) 0 else -1
is MetaItem.MultiNodeItem -> nodes.indexOf(meta)
}
}

View File

@ -1,95 +1,206 @@
package hep.dataforge.meta
import kotlinx.serialization.*
import kotlinx.serialization.internal.SerialClassDescImpl
import kotlinx.serialization.json.*
@Serializer(forClass = Value::class)
object ValueSerializer : KSerializer<Value> {
override val serialClassDesc: KSerialClassDesc = SerialClassDescImpl("Value")
override fun load(input: KInput): Value {
val key = input.readByteValue()
return when (key.toChar()) {
'S' -> StringValue(input.readStringValue())
'd' -> NumberValue(input.readDoubleValue())
'f' -> NumberValue(input.readFloatValue())
'i' -> NumberValue(input.readIntValue())
's' -> NumberValue(input.readShortValue())
'l' -> NumberValue(input.readLongValue())
'b' -> NumberValue(input.readByteValue())
'+' -> True
'-' -> False
'N' -> Null
'L' -> {
val size = input.readIntValue()
val list = (0 until size).map { load(input) }
ListValue(list)
}
else -> error("Unknown value deserialization ket '$key'")
/*Universal serialization*/
//sealed class MetaItemProxy {
//
// @Serializable
// class NumberValueProxy(val number: Number) : MetaItemProxy()
//
// @Serializable
// class StringValueProxy(val string: String) : MetaItemProxy()
//
// @Serializable
// class BooleanValueProxy(val boolean: Boolean) : MetaItemProxy()
//
// @Serializable
// object NullValueProxy : MetaItemProxy()
//
// @Serializable
// class MetaProxy(@Serializable val map: Map<String, MetaItemProxy>) : MetaItemProxy()
//
// @Serializable
// class MetaListProxy(@Serializable val nodes: List<MetaProxy>) : MetaItemProxy()
//}
//
//
//fun Meta.toMap(): Map<String, MetaItemProxy> {
// return this.items.mapValues { (_, value) ->
// when (value) {
// is MetaItem.ValueItem -> when (value.value.type) {
// ValueType.NUMBER -> MetaItemProxy.NumberValueProxy(value.value.number)
// ValueType.STRING -> MetaItemProxy.StringValueProxy(value.value.string)
// ValueType.BOOLEAN -> MetaItemProxy.BooleanValueProxy(value.value.boolean)
// ValueType.NULL -> MetaItemProxy.NullValueProxy
// }
// is MetaItem.SingleNodeItem -> MetaItemProxy.MetaProxy(value.node.toMap())
// is MetaItem.MultiNodeItem -> MetaItemProxy.MetaListProxy(value.nodes.map { MetaItemProxy.MetaProxy(it.toMap()) })
// }
// }
//}
/*Direct JSON serialization*/
fun Value.toJson(): JsonElement = if (isList()) {
JsonArray(list.map { it.toJson() })
} else {
when (type) {
ValueType.NUMBER -> JsonPrimitive(number)
ValueType.STRING -> JsonPrimitive(string)
ValueType.BOOLEAN -> JsonPrimitive(boolean)
ValueType.NULL -> JsonNull
}
}
fun Meta.toJSON(): JsonObject {
val map = this.items.mapValues { (_, value) ->
when (value) {
is MetaItem.ValueItem -> value.value.toJson()
is MetaItem.SingleNodeItem -> value.node.toJSON()
is MetaItem.MultiNodeItem -> JsonArray(value.nodes.map { it.toJSON() })
}
}
return JsonObject(map)
}
override fun save(output: KOutput, obj: Value) {
when (obj.type) {
ValueType.NUMBER -> {
val number = obj.number
when (number) {
is Float -> {
output.writeByteValue('f'.toByte())
output.writeFloatValue(number)
}
is Short -> {
output.writeByteValue('s'.toByte())
output.writeShortValue(number)
}
is Int -> {
output.writeByteValue('i'.toByte())
output.writeIntValue(number)
}
is Long -> {
output.writeByteValue('l'.toByte())
output.writeLongValue(number)
}
is Byte -> {
output.writeByteValue('b'.toByte())
output.writeByteValue(number)
}
is Double -> {
output.writeByteValue('d'.toByte())
output.writeDoubleValue(number)
}
else -> {
//TODO add warning
output.writeByteValue('d'.toByte())
output.writeDoubleValue(number.toDouble())
}
fun JsonPrimitive.toValue(): Value {
return when (this) {
is JsonLiteral -> LazyParsedValue(content)
is JsonNull -> Null
}
}
fun JsonObject.toMeta(): Meta {
return buildMeta {
this@toMeta.forEach { (key, value) ->
when (value) {
is JsonPrimitive -> set(key, value.toValue())
is JsonObject -> set(key, value.toMeta())
is JsonArray -> if (value.all { it is JsonPrimitive }) {
set(key, ListValue(value.map { (it as JsonPrimitive).toValue() }))
} else {
set(
key,
value.map {
if (it is JsonObject) {
it.toMeta()
} else {
buildMeta { "@value" to it.primitive.toValue() }
}
}
)
}
}
ValueType.STRING -> {
output.writeByteValue('S'.toByte())
output.writeStringValue(obj.string)
}
ValueType.BOOLEAN -> if (obj.boolean) {
output.writeByteValue('+'.toByte())
} else {
output.writeByteValue('-'.toByte())
}
ValueType.NULL -> output.writeByteValue('N'.toByte())
}
}
}
/*Direct CBOR serialization*/
//@Serializer(forClass = Meta::class)
//object MetaSerializer: KSerializer<Meta>{
// override val serialClassDesc: KSerialClassDesc = SerialClassDescImpl("Meta")
//
// override fun load(input: KInput): Meta {
//
//fun Meta.toBinary(out: OutputStream) {
// fun CBOR.CBOREncoder.encodeChar(char: Char) {
// encodeNumber(char.toByte().toLong())
// }
//
// override fun save(output: KOutput, obj: Meta) {
// NamedValueOutput()
// fun CBOR.CBOREncoder.encodeValue(value: Value) {
// if (value.isList()) {
// encodeChar('L')
// startArray()
// value.list.forEach {
// encodeValue(it)
// }
// end()
// } else when (value.type) {
// ValueType.NUMBER -> when (value.value) {
// is Int, is Short, is Long -> {
// encodeChar('i')
// encodeNumber(value.number.toLong())
// }
// is Float -> {
// encodeChar('f')
// encodeFloat(value.number.toFloat())
// }
// else -> {
// encodeChar('d')
// encodeDouble(value.number.toDouble())
// }
// }
// ValueType.STRING -> {
// encodeChar('S')
// encodeString(value.string)
// }
// ValueType.BOOLEAN -> {
// if (value.boolean) {
// encodeChar('+')
// } else {
// encodeChar('-')
// }
// }
// ValueType.NULL -> {
// encodeChar('N')
// }
// }
// }
//
// fun CBOR.CBOREncoder.encodeMeta(meta: Meta) {
// meta.items.forEach { (key, item) ->
// this.startMap()
// encodeString(key)
// when (item) {
// is MetaItem.ValueItem -> {
// encodeChar('V')
// encodeValue(item.value)
// }
// is MetaItem.SingleNodeItem -> {
// startArray()
// encodeMeta(item.node)
// }
// is MetaItem.MultiNodeItem -> {
// startArray()
// item.nodes.forEach {
// encodeMeta(it)
// }
// end()
// }
// }
// }
// }
//
//
// CBOR.CBOREncoder(out).apply {
// encodeMeta(this@toBinary)
// }
//}
//
//fun InputStream.readBinaryMeta(): Meta {
// fun CBOR.CBORDecoder.nextChar(): Char = nextNumber().toByte().toChar()
//
// fun CBOR.CBORDecoder.nextValue(): Value {
// val key = nextChar()
// return when(key){
// 'L' -> {
// val size = startArray()
// val res = (0 until size).map { nextValue() }
// end()
// ListValue(res)
// }
// 'S' -> StringValue(nextString())
// 'N' -> Null
// '+' -> True
// '-' -> False
// 'i' -> NumberValue(nextNumber())
// 'f' -> NumberValue(nextFloat())
// 'd' -> NumberValue(nextDouble())
// else -> error("Unknown binary key: $key")
// }
// }
//
// fun CBOR.CBORDecoder.nextMeta(): Meta{
//
// }
//
//}

View File

@ -1,7 +1,5 @@
package hep.dataforge.meta
import kotlinx.serialization.Serializable
/**
* The list of supported Value types.
@ -18,7 +16,6 @@ enum class ValueType {
*
* Value can represent a list of value objects.
*/
@Serializable(with = ValueSerializer::class)
interface Value {
/**
* Get raw value of this value
@ -69,10 +66,10 @@ interface Value {
* A singleton null value
*/
object Null : Value {
override val value: Any? = null
override val type: ValueType = ValueType.NULL
override val number: Number = Double.NaN
override val string: String = "@null"
override val value: Any? get() = null
override val type: ValueType get() = ValueType.NULL
override val number: Number get() = Double.NaN
override val string: String get() = "@null"
}
/**
@ -85,40 +82,60 @@ fun Value.isNull(): Boolean = this == Null
* Singleton true value
*/
object True : Value {
override val value: Any? = true
override val type: ValueType = ValueType.BOOLEAN
override val number: Number = 1.0
override val string: String = "+"
override val value: Any? get() = true
override val type: ValueType get() = ValueType.BOOLEAN
override val number: Number get() = 1.0
override val string: String get() = "+"
}
/**
* Singleton false value
*/
object False : Value {
override val value: Any? = false
override val type: ValueType = ValueType.BOOLEAN
override val number: Number = -1.0
override val string: String = "-"
override val value: Any? get() = false
override val type: ValueType get() = ValueType.BOOLEAN
override val number: Number get() = -1.0
override val string: String get() = "-"
}
val Value.boolean get() = this == True || this.list.firstOrNull() == True || (type == ValueType.STRING && string.toBoolean())
class NumberValue(override val number: Number) : Value {
override val value: Any? get() = number
override val type: ValueType = ValueType.NUMBER
override val type: ValueType get() = ValueType.NUMBER
override val string: String get() = number.toString()
override fun equals(other: Any?): Boolean {
return this.number == (other as? Value)?.number
}
override fun hashCode(): Int = number.hashCode()
}
class StringValue(override val string: String) : Value {
override val value: Any? get() = string
override val type: ValueType = ValueType.STRING
override val type: ValueType get() = ValueType.STRING
override val number: Number get() = string.toDouble()
override fun equals(other: Any?): Boolean {
return this.string == (other as? Value)?.string
}
override fun hashCode(): Int = string.hashCode()
}
class EnumValue<E : Enum<*>>(override val value: E) : Value {
override val type: ValueType = ValueType.STRING
override val number: Number = value.ordinal
override val string: String = value.name
override val type: ValueType get() = ValueType.STRING
override val number: Number get() = value.ordinal
override val string: String get() = value.name
override fun equals(other: Any?): Boolean {
return string == (other as? Value)?.string
}
override fun hashCode(): Int = value.hashCode()
}
class ListValue(override val list: List<Value>) : Value {
@ -148,6 +165,7 @@ fun String.asValue(): Value = StringValue(this)
fun Collection<Value>.asValue(): Value = ListValue(this.toList())
/**
* Create Value from String using closest match conversion
*/
@ -181,4 +199,15 @@ fun String.parseValue(): Value {
//Give up and return a StringValue
return StringValue(this)
}
class LazyParsedValue(override val string: String): Value{
private val parsedValue by lazy { string.parseValue() }
override val value: Any?
get() = parsedValue.value
override val type: ValueType
get() = parsedValue.type
override val number: Number
get() = parsedValue.number
}

View File

@ -1,6 +1,5 @@
package hep.dataforge.names
import kotlin.coroutines.experimental.buildSequence
/**
* The general interface for working with names.
@ -75,7 +74,7 @@ data class NameToken internal constructor(val body: String, val query: String) {
}
fun String.toName(): Name {
val tokens = buildSequence<NameToken> {
val tokens = sequence {
var bodyBuilder = StringBuilder()
var queryBuilder = StringBuilder()
var bracketCount: Int = 0

View File

@ -1,14 +1,12 @@
package scientifik.kplot.remote
package hep.dataforge.meta
import hep.dataforge.meta.buildMeta
import hep.dataforge.meta.get
import hep.dataforge.meta.value
import kotlinx.serialization.json.JSON
import kotlin.test.Test
import kotlin.test.assertEquals
class SerializationTest {
class SerializationTest{
@Test
fun testMetaSerialization(){
fun testJSONSerialization() {
val meta = buildMeta {
"a" to 2
"b" to {
@ -16,7 +14,39 @@ class SerializationTest{
"d" to 2.2
}
}
val json = JSON.stringify(meta["a"]?.value!!)
val json = meta.toJSON()
println(json)
val result = json.toMeta()
assertEquals(meta, result)
}
// @Test
// fun testIndirectSerialization() {
// val meta = buildMeta {
// "a" to 2
// "b" to {
// "c" to "ddd"
// "d" to 2.2
// }
// }
// val json = JSON.stringify(meta.toMap())
// println(json)
//// val result = json.toMeta()
//// assertEquals(meta, result)
// }
// @Test
// fun testWeirdSerialization() {
// val meta = buildMeta {
// "a" to 2
// "b" to {
// "c" to "ddd"
// "d" to 2.2
// }
// }
// val json = JSON.stringify(meta.toJSON())
// println(json)
// val result: JsonObject = JSON.parse(json)
// assertEquals(meta, result.toMeta())
// }
}

View File

@ -1,6 +1,5 @@
plugins {
plugins{
id 'kotlin-platform-js'
//id 'kotlinx-serialization'
}
dependencies {

View File

@ -1,15 +1,15 @@
plugins {
plugins{
id 'kotlin-platform-jvm'
id 'kotlinx-serialization'
}
dependencies {
expectedBy project(":dataforge-meta-common")
compile "org.jetbrains.kotlin:kotlin-stdlib-jdk8:$kotlin_version"
compile "org.jetbrains.kotlin:kotlin-reflect:$kotlin_version"
compile "org.jetbrains.kotlinx:kotlinx-serialization-runtime:$serialization_version"
testCompile "org.jetbrains.kotlin:kotlin-test"
testCompile "org.jetbrains.kotlin:kotlin-test-junit5"
testCompile "org.jetbrains.kotlin:kotlin-test-junit"
}
compileKotlin {