kscience/dataforge-core
7
0
Fork 0
Code Issues Pull Requests 1 Packages Projects Releases 12 Wiki Activity

Move Meta serialization to dataforge-meta

Browse Source
This commit is contained in:
Alexander Nozik 2020-03-11 22:30:29 +03:00
parent 93c806c3bf
commit e2845f4efc
26 changed files with 392 additions and 429 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
build.gradle.kts
View File

@ -1,12 +1,12 @@
plugins {
val toolsVersion = "0.3.1"
val toolsVersion = "0.4.0"
id("scientifik.mpp") version toolsVersion apply false
id("scientifik.jvm") version toolsVersion apply false
id("scientifik.publish") version toolsVersion apply false
}
val dataforgeVersion by extra("0.1.5-dev-9")
val dataforgeVersion by extra("0.1.5-dev-10")
val bintrayRepo by extra("dataforge")
val githubProject by extra("dataforge-core")

6
dataforge-context/build.gradle.kts
View File

@ -12,20 +12,20 @@ kotlin {
dependencies {
api(project(":dataforge-meta"))
api(kotlin("reflect"))
api("io.github.microutils:kotlin-logging-common:1.7.2")
api("io.github.microutils:kotlin-logging-common:1.7.8")
api("org.jetbrains.kotlinx:kotlinx-coroutines-core-common:$coroutinesVersion")
}
}
val jvmMain by getting {
dependencies {
api("io.github.microutils:kotlin-logging:1.7.2")
api("io.github.microutils:kotlin-logging:1.7.8")
api("ch.qos.logback:logback-classic:1.2.3")
api("org.jetbrains.kotlinx:kotlinx-coroutines-core:$coroutinesVersion")
}
}
val jsMain by getting {
dependencies {
api("io.github.microutils:kotlin-logging-js:1.7.2")
api("io.github.microutils:kotlin-logging-js:1.7.8")
api("org.jetbrains.kotlinx:kotlinx-coroutines-core-js:$coroutinesVersion")
}
}

2
dataforge-context/src/commonMain/kotlin/hep/dataforge/context/Context.kt
View File

@ -103,7 +103,7 @@ open class Context(
plugins.forEach { it.detach() }
}
override fun toMeta(): Meta = buildMeta {
override fun toMeta(): Meta = Meta {
"parent" to parent?.name
"properties" put properties.seal()
"plugins" put plugins.map { it.toMeta() }

5
dataforge-context/src/commonMain/kotlin/hep/dataforge/context/ContextBuilder.kt
View File

@ -1,6 +1,7 @@
package hep.dataforge.context
import hep.dataforge.meta.DFBuilder
import hep.dataforge.meta.Meta
import hep.dataforge.meta.MetaBuilder
import hep.dataforge.meta.buildMeta
import hep.dataforge.names.toName
@ -22,11 +23,11 @@ class ContextBuilder(var name: String = "@anonymous", val parent: Context = Glob
}
fun plugin(tag: PluginTag, action: MetaBuilder.() -> Unit = {}) {
plugins.add(PluginRepository.fetch(tag, buildMeta(action)))
plugins.add(PluginRepository.fetch(tag, Meta(action)))
}
fun plugin(builder: PluginFactory<*>, action: MetaBuilder.() -> Unit = {}) {
plugins.add(builder.invoke(buildMeta(action)))
plugins.add(builder.invoke(Meta(action)))
}
fun plugin(name: String, group: String = "", version: String = "", action: MetaBuilder.() -> Unit = {}) {

2
dataforge-context/src/commonMain/kotlin/hep/dataforge/context/Plugin.kt
View File

@ -65,7 +65,7 @@ interface Plugin : Named, ContextAware, Provider, MetaRepr {
*/
fun detach()
override fun toMeta(): Meta = buildMeta {
override fun toMeta(): Meta = Meta {
"context" put context.name.toString()
"type" to this::class.simpleName
"tag" put tag

4
dataforge-context/src/commonMain/kotlin/hep/dataforge/context/PluginManager.kt
View File

@ -102,7 +102,7 @@ class PluginManager(override val context: Context) : ContextAware, Iterable<Plug
load(factory(meta, context))
fun <T : Plugin> load(factory: PluginFactory<T>, metaBuilder: MetaBuilder.() -> Unit): T =
load(factory, buildMeta(metaBuilder))
load(factory, Meta(metaBuilder))
/**
* Remove a plugin from [PluginManager]
@ -134,7 +134,7 @@ class PluginManager(override val context: Context) : ContextAware, Iterable<Plug
factory: PluginFactory<T>,
recursive: Boolean = true,
metaBuilder: MetaBuilder.() -> Unit
): T = fetch(factory, recursive, buildMeta(metaBuilder))
): T = fetch(factory, recursive, Meta(metaBuilder))
override fun iterator(): Iterator<Plugin> = plugins.iterator()

2
dataforge-context/src/commonMain/kotlin/hep/dataforge/context/PluginTag.kt
View File

@ -36,7 +36,7 @@ data class PluginTag(
override fun toString(): String = listOf(group, name, version).joinToString(separator = ":")
override fun toMeta(): Meta = buildMeta {
override fun toMeta(): Meta = Meta {
"name" put name
"group" put group
"version" put version

2
dataforge-data/src/commonMain/kotlin/hep/dataforge/data/Data.kt
View File

@ -19,7 +19,7 @@ interface Data<out T : Any> : Goal<T>, MetaRepr{
*/
val meta: Meta
override fun toMeta(): Meta = buildMeta {
override fun toMeta(): Meta = Meta {
"type" put (type.simpleName?:"undefined")
if(!meta.isEmpty()) {
"meta" put meta

6
dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataNode.kt
View File

@ -47,7 +47,7 @@ interface DataNode<out T : Any> : MetaRepr {
val meta: Meta
override fun toMeta(): Meta = buildMeta {
override fun toMeta(): Meta = Meta {
"type" put (type.simpleName ?: "undefined")
"items" put {
this@DataNode.items.forEach {
@ -255,11 +255,11 @@ fun <T : Any> DataTreeBuilder<T>.static(name: Name, data: T, meta: Meta = EmptyM
}
fun <T : Any> DataTreeBuilder<T>.static(name: Name, data: T, block: MetaBuilder.() -> Unit = {}) {
this[name] = Data.static(data, buildMeta(block))
this[name] = Data.static(data, Meta(block))
}
fun <T : Any> DataTreeBuilder<T>.static(name: String, data: T, block: MetaBuilder.() -> Unit = {}) {
this[name.toName()] = Data.static(data, buildMeta(block))
this[name.toName()] = Data.static(data, Meta(block))
}
fun <T : Any> DataTreeBuilder<T>.node(name: Name, node: DataNode<T>) {

131
dataforge-io/src/commonMain/kotlin/hep/dataforge/io/JsonMetaFormat.kt
View File

@ -9,6 +9,8 @@ import hep.dataforge.meta.descriptors.ValueDescriptor
import hep.dataforge.meta.Meta
import hep.dataforge.meta.MetaBase
import hep.dataforge.meta.MetaItem
import hep.dataforge.meta.serialization.toJson
import hep.dataforge.meta.serialization.toMeta
import hep.dataforge.names.NameToken
import hep.dataforge.names.toName
import hep.dataforge.values.*
@ -16,6 +18,7 @@ import kotlinx.io.Input
import kotlinx.io.Output
import kotlinx.io.text.readUtf8String
import kotlinx.io.text.writeUtf8String
import kotlinx.serialization.UnstableDefault
import kotlinx.serialization.json.*
@ -23,8 +26,8 @@ import kotlin.collections.component1
import kotlin.collections.component2
import kotlin.collections.set
class JsonMetaFormat(private val json: Json = Json.indented) : MetaFormat {
@OptIn(UnstableDefault::class)
class JsonMetaFormat(private val json: Json = DEFAULT_JSON) : MetaFormat {
override fun Output.writeMeta(meta: Meta, descriptor: NodeDescriptor?) {
val jsonObject = meta.toJson(descriptor)
@ -38,6 +41,8 @@ class JsonMetaFormat(private val json: Json = Json.indented) : MetaFormat {
}
companion object : MetaFormatFactory {
val DEFAULT_JSON = Json{prettyPrint = true}
override fun invoke(meta: Meta, context: Context): MetaFormat = default
override val shortName = "json"
@ -52,125 +57,3 @@ class JsonMetaFormat(private val json: Json = Json.indented) : MetaFormat {
default.run { readMeta(descriptor) }
}
}
/**
* @param descriptor reserved for custom serialization in future
*/
fun Value.toJson(descriptor: ValueDescriptor? = null): JsonElement {
return if (isList()) {
JsonArray(list.map { it.toJson() })
} else {
when (type) {
ValueType.NUMBER -> JsonPrimitive(number)
ValueType.STRING -> JsonPrimitive(string)
ValueType.BOOLEAN -> JsonPrimitive(boolean)
ValueType.NULL -> JsonNull
}
}
}
//Use these methods to customize JSON key mapping
private fun NameToken.toJsonKey(descriptor: ItemDescriptor?) = toString()
//private fun NodeDescriptor?.getDescriptor(key: String) = this?.items?.get(key)
fun Meta.toJson(descriptor: NodeDescriptor? = null): JsonObject {
//TODO search for same name siblings and arrange them into arrays
val map = this.items.entries.associate { (name, item) ->
val itemDescriptor = descriptor?.items?.get(name.body)
val key = name.toJsonKey(itemDescriptor)
val value = when (item) {
is MetaItem.ValueItem -> {
item.value.toJson(itemDescriptor as? ValueDescriptor)
}
is MetaItem.NodeItem -> {
item.node.toJson(itemDescriptor as? NodeDescriptor)
}
}
key to value
}
return JsonObject(map)
}
fun JsonElement.toMeta(descriptor: NodeDescriptor? = null): Meta {
return when (val item = toMetaItem(descriptor)) {
is MetaItem.NodeItem<*> -> item.node
is MetaItem.ValueItem -> item.value.toMeta()
}
}
fun JsonPrimitive.toValue(descriptor: ValueDescriptor?): Value {
return when (this) {
JsonNull -> Null
else -> this.content.parseValue() // Optimize number and boolean parsing
}
}
fun JsonElement.toMetaItem(descriptor: ItemDescriptor? = null): MetaItem<JsonMeta> = when (this) {
is JsonPrimitive -> {
val value = this.toValue(descriptor as? ValueDescriptor)
MetaItem.ValueItem(value)
}
is JsonObject -> {
val meta = JsonMeta(this, descriptor as? NodeDescriptor)
MetaItem.NodeItem(meta)
}
is JsonArray -> {
if (this.all { it is JsonPrimitive }) {
val value = if (isEmpty()) {
Null
} else {
ListValue(
map<JsonElement, Value> {
//We already checked that all values are primitives
(it as JsonPrimitive).toValue(descriptor as? ValueDescriptor)
}
)
}
MetaItem.ValueItem(value)
} else {
json {
"@value" to this@toMetaItem
}.toMetaItem(descriptor)
}
}
}
class JsonMeta(val json: JsonObject, val descriptor: NodeDescriptor? = null) : MetaBase() {
@Suppress("UNCHECKED_CAST")
private operator fun MutableMap<String, MetaItem<JsonMeta>>.set(key: String, value: JsonElement): Unit {
val itemDescriptor = descriptor?.items?.get(key)
return when (value) {
is JsonPrimitive -> {
this[key] = MetaItem.ValueItem(value.toValue(itemDescriptor as? ValueDescriptor)) as MetaItem<JsonMeta>
}
is JsonObject -> {
this[key] = MetaItem.NodeItem(JsonMeta(value, itemDescriptor as? NodeDescriptor))
}
is JsonArray -> {
when {
value.all { it is JsonPrimitive } -> {
val listValue = ListValue(
value.map {
//We already checked that all values are primitives
(it as JsonPrimitive).toValue(itemDescriptor as? ValueDescriptor)
}
)
this[key] = MetaItem.ValueItem(listValue) as MetaItem<JsonMeta>
}
else -> value.forEachIndexed { index, jsonElement ->
this["$key[$index]"] = jsonElement.toMetaItem(itemDescriptor)
}
}
}
}
}
override val items: Map<NameToken, MetaItem<JsonMeta>> by lazy {
val map = HashMap<String, MetaItem<JsonMeta>>()
json.forEach { (key, value) -> map[key] = value }
map.mapKeys { it.key.toName().first()!! }
}
}

6
dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaglessEnvelopeFormat.kt
View File

@ -119,7 +119,7 @@ class TaglessEnvelopeFormat(
var line: String
do {
line = readUtf8Line()// ?: error("Input does not contain tagless envelope header")
offset += line.toUtf8Bytes().size.toUInt()
offset += line.encodeToByteArray().size.toUInt()
} while (!line.startsWith(TAGLESS_ENVELOPE_HEADER))
val properties = HashMap<String, String>()
@ -133,7 +133,7 @@ class TaglessEnvelopeFormat(
}
try {
line = readUtf8Line()
offset += line.toUtf8Bytes().size.toUInt()
offset += line.encodeToByteArray().size.toUInt()
} catch (ex: EOFException) {
return PartialEnvelope(Meta.EMPTY, offset.toUInt(), 0.toULong())
}
@ -156,7 +156,7 @@ class TaglessEnvelopeFormat(
do {
line = readUtf8Line() ?: return PartialEnvelope(Meta.EMPTY, offset.toUInt(), 0.toULong())
offset += line.toUtf8Bytes().size.toUInt()
offset += line.encodeToByteArray().size.toUInt()
//returning an Envelope without data if end of input is reached
} while (!line.startsWith(dataStart))

142
dataforge-io/src/commonMain/kotlin/hep/dataforge/io/serialization/MetaSerializer.kt
View File

@ -1,142 +0,0 @@
package hep.dataforge.io.serialization
import hep.dataforge.io.toJson
import hep.dataforge.io.toMeta
import hep.dataforge.meta.*
import hep.dataforge.names.NameToken
import hep.dataforge.values.*
import kotlinx.serialization.*
import kotlinx.serialization.internal.*
import kotlinx.serialization.json.JsonInput
import kotlinx.serialization.json.JsonObjectSerializer
import kotlinx.serialization.json.JsonOutput
@Serializer(Value::class)
@UseExperimental(InternalSerializationApi::class)
object ValueSerializer : KSerializer<Value> {
private val valueTypeSerializer = EnumSerializer(ValueType::class)
private val listSerializer by lazy { ArrayListSerializer(ValueSerializer) }
override val descriptor: SerialDescriptor = descriptor("Value") {
boolean("isList")
enum<ValueType>("valueType")
element("value", null)
}
private fun Decoder.decodeValue(): Value {
return when (decode(valueTypeSerializer)) {
ValueType.NULL -> Null
ValueType.NUMBER -> decodeDouble().asValue() //TODO differentiate?
ValueType.BOOLEAN -> decodeBoolean().asValue()
ValueType.STRING -> decodeString().asValue()
else -> decodeString().parseValue()
}
}
override fun deserialize(decoder: Decoder): Value {
val isList = decoder.decodeBoolean()
return if (isList) {
listSerializer.deserialize(decoder).asValue()
} else {
decoder.decodeValue()
}
}
private fun Encoder.encodeValue(value: Value) {
encode(valueTypeSerializer, value.type)
when (value.type) {
ValueType.NULL -> {
// do nothing
}
ValueType.NUMBER -> encodeDouble(value.double)
ValueType.BOOLEAN -> encodeBoolean(value.boolean)
ValueType.STRING -> encodeString(value.string)
else -> encodeString(value.string)
}
}
override fun serialize(encoder: Encoder, obj: Value) {
encoder.encodeBoolean(obj.isList())
if (obj.isList()) {
listSerializer.serialize(encoder, obj.list)
} else {
encoder.encodeValue(obj)
}
}
}
@Serializer(MetaItem::class)
object MetaItemSerializer : KSerializer<MetaItem<*>> {
override val descriptor: SerialDescriptor = descriptor("MetaItem") {
boolean("isNode")
element("value", null)
}
override fun deserialize(decoder: Decoder): MetaItem<*> {
val isNode = decoder.decodeBoolean()
return if (isNode) {
MetaItem.NodeItem(decoder.decode(MetaSerializer))
} else {
MetaItem.ValueItem(decoder.decode(ValueSerializer))
}
}
override fun serialize(encoder: Encoder, obj: MetaItem<*>) {
encoder.encodeBoolean(obj is MetaItem.NodeItem)
when (obj) {
is MetaItem.NodeItem -> MetaSerializer.serialize(encoder, obj.node)
is MetaItem.ValueItem -> ValueSerializer.serialize(encoder, obj.value)
}
}
}
private class DeserializedMeta(override val items: Map<NameToken, MetaItem<*>>) : MetaBase()
/**
* Serialized for meta
*/
@Serializer(Meta::class)
object MetaSerializer : KSerializer<Meta> {
private val mapSerializer = HashMapSerializer(
StringSerializer,
MetaItemSerializer
)
override val descriptor: SerialDescriptor = NamedMapClassDescriptor(
"hep.dataforge.meta.Meta",
StringSerializer.descriptor,
MetaItemSerializer.descriptor
)
override fun deserialize(decoder: Decoder): Meta {
return if (decoder is JsonInput) {
JsonObjectSerializer.deserialize(decoder).toMeta()
} else {
DeserializedMeta(mapSerializer.deserialize(decoder).mapKeys { NameToken(it.key) })
}
}
override fun serialize(encoder: Encoder, obj: Meta) {
if (encoder is JsonOutput) {
JsonObjectSerializer.serialize(encoder, obj.toJson())
} else {
mapSerializer.serialize(encoder, obj.items.mapKeys { it.key.toString() })
}
}
}
@Serializer(Config::class)
object ConfigSerializer : KSerializer<Config> {
override val descriptor: SerialDescriptor = MetaSerializer.descriptor
override fun deserialize(decoder: Decoder): Config {
return MetaSerializer.deserialize(decoder).asConfig()
}
override fun serialize(encoder: Encoder, obj: Config) {
MetaSerializer.serialize(encoder, obj)
}
}

33
dataforge-io/src/commonMain/kotlin/hep/dataforge/io/serialization/nameSerializers.kt
View File

@ -1,33 +0,0 @@
package hep.dataforge.io.serialization
import hep.dataforge.names.Name
import hep.dataforge.names.NameToken
import hep.dataforge.names.toName
import kotlinx.serialization.*
import kotlinx.serialization.internal.StringDescriptor
@Serializer(Name::class)
object NameSerializer : KSerializer<Name> {
override val descriptor: SerialDescriptor = StringDescriptor.withName("Name")
override fun deserialize(decoder: Decoder): Name {
return decoder.decodeString().toName()
}
override fun serialize(encoder: Encoder, obj: Name) {
encoder.encodeString(obj.toString())
}
}
@Serializer(NameToken::class)
object NameTokenSerializer : KSerializer<NameToken> {
override val descriptor: SerialDescriptor = StringDescriptor.withName("NameToken")
override fun deserialize(decoder: Decoder): NameToken {
return decoder.decodeString().toName().first()!!
}
override fun serialize(encoder: Encoder, obj: NameToken) {
encoder.encodeString(obj.toString())
}
}

93
dataforge-io/src/commonMain/kotlin/hep/dataforge/io/serialization/serializationUtils.kt
View File

@ -1,93 +0,0 @@
package hep.dataforge.io.serialization
import hep.dataforge.meta.DFExperimental
import kotlinx.serialization.*
import kotlinx.serialization.internal.*
/**
* A convenience builder for serial descriptors
*/
inline class SerialDescriptorBuilder(private val impl: SerialClassDescImpl) {
fun element(
name: String,
descriptor: SerialDescriptor?,
isOptional: Boolean = false,
vararg annotations: Annotation
) {
impl.addElement(name, isOptional)
descriptor?.let { impl.pushDescriptor(descriptor) }
annotations.forEach {
impl.pushAnnotation(it)
}
}
fun element(
name: String,
isOptional: Boolean = false,
vararg annotations: Annotation,
block: SerialDescriptorBuilder.() -> Unit
) {
impl.addElement(name, isOptional)
impl.pushDescriptor(SerialDescriptorBuilder(SerialClassDescImpl(name)).apply(block).build())
annotations.forEach {
impl.pushAnnotation(it)
}
}
fun boolean(name: String, isOptional: Boolean = false, vararg annotations: Annotation) =
element(name, BooleanDescriptor, isOptional, *annotations)
fun string(name: String, isOptional: Boolean = false, vararg annotations: Annotation) =
element(name, StringDescriptor, isOptional, *annotations)
fun int(name: String, isOptional: Boolean = false, vararg annotations: Annotation) =
element(name, IntDescriptor, isOptional, *annotations)
fun double(name: String, isOptional: Boolean = false, vararg annotations: Annotation) =
element(name, DoubleDescriptor, isOptional, *annotations)
fun float(name: String, isOptional: Boolean = false, vararg annotations: Annotation) =
element(name, FloatDescriptor, isOptional, *annotations)
fun long(name: String, isOptional: Boolean = false, vararg annotations: Annotation) =
element(name, LongDescriptor, isOptional, *annotations)
fun doubleArray(name: String, isOptional: Boolean = false, vararg annotations: Annotation) =
element(name, DoubleArraySerializer.descriptor, isOptional, *annotations)
@UseExperimental(InternalSerializationApi::class)
inline fun <reified E : Enum<E>> enum(name: String, isOptional: Boolean = false, vararg annotations: Annotation) =
element(name, EnumSerializer(E::class).descriptor, isOptional, *annotations)
fun classAnnotation(a: Annotation) = impl.pushClassAnnotation(a)
fun build(): SerialDescriptor = impl
}
inline fun <reified T : Any> KSerializer<T>.descriptor(
name: String,
block: SerialDescriptorBuilder.() -> Unit
): SerialDescriptor =
SerialDescriptorBuilder(SerialClassDescImpl(name)).apply(block).build()
@DFExperimental
inline fun <R> Decoder.decodeStructure(
desc: SerialDescriptor,
vararg typeParams: KSerializer<*> = emptyArray(),
crossinline block: CompositeDecoder.() -> R
): R {
val decoder = beginStructure(desc, *typeParams)
val res = decoder.block()
decoder.endStructure(desc)
return res
}
inline fun Encoder.encodeStructure(
desc: SerialDescriptor,
vararg typeParams: KSerializer<*> = emptyArray(),
block: CompositeEncoder.() -> Unit
) {
val encoder = beginStructure(desc, *typeParams)
encoder.block()
encoder.endStructure(desc)
}

4
dataforge-meta/build.gradle.kts
View File

@ -1,5 +1,9 @@
import scientifik.useSerialization
plugins {
id("scientifik.mpp")
}
useSerialization()
description = "Meta definition and basic operations on meta"

2
dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Laminate.kt
View File

@ -64,7 +64,7 @@ class Laminate(layers: List<Meta>) : MetaBase() {
}
else -> map {
when (it) {
is MetaItem.ValueItem -> MetaItem.NodeItem(buildMeta { Meta.VALUE_KEY put it.value })
is MetaItem.ValueItem -> MetaItem.NodeItem(Meta { Meta.VALUE_KEY put it.value })
is MetaItem.NodeItem -> it
}
}.merge()

7
dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Meta.kt
View File

@ -3,11 +3,13 @@ package hep.dataforge.meta
import hep.dataforge.meta.Meta.Companion.VALUE_KEY
import hep.dataforge.meta.MetaItem.NodeItem
import hep.dataforge.meta.MetaItem.ValueItem
import hep.dataforge.meta.serialization.toJson
import hep.dataforge.names.*
import hep.dataforge.values.EnumValue
import hep.dataforge.values.Null
import hep.dataforge.values.Value
import hep.dataforge.values.boolean
import kotlinx.serialization.Serializable
/**
@ -15,11 +17,14 @@ import hep.dataforge.values.boolean
* * a [ValueItem] (leaf)
* * a [NodeItem] (node)
*/
@Serializable
sealed class MetaItem<out M : Meta> {
@Serializable
data class ValueItem(val value: Value) : MetaItem<Nothing>() {
override fun toString(): String = value.toString()
}
@Serializable
data class NodeItem<M : Meta>(val node: M) : MetaItem<M>() {
override fun toString(): String = node.toString()
}
@ -161,7 +166,7 @@ abstract class MetaBase : Meta {
override fun hashCode(): Int = items.hashCode()
override fun toString(): String = items.toString()
override fun toString(): String = toJson().toString()
}
/**

7
dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MetaBuilder.kt
View File

@ -152,9 +152,4 @@ fun buildMeta(builder: MetaBuilder.() -> Unit): MetaBuilder = MetaBuilder().appl
* Build a [MetaBuilder] using given transformation
*/
@Suppress("FunctionName")
fun Meta(builder: MetaBuilder.() -> Unit): MetaBuilder = MetaBuilder().apply(builder)
/**
* Build meta using given source meta as a base
*/
fun buildMeta(source: Meta, builder: MetaBuilder.() -> Unit): MetaBuilder = source.builder().apply(builder)
fun Meta(builder: MetaBuilder.() -> Unit): MetaBuilder = MetaBuilder().apply(builder)

2
dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/annotations.kt
View File

@ -6,6 +6,6 @@ package hep.dataforge.meta
@DslMarker
annotation class DFBuilder
@Experimental(level = Experimental.Level.WARNING)
@RequiresOptIn(level = RequiresOptIn.Level.WARNING)
@Retention(AnnotationRetention.BINARY)
annotation class DFExperimental

8
dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/descriptors/ItemDescriptor.kt
View File

@ -255,10 +255,10 @@ class ValueDescriptor : ItemDescriptor() {
}
companion object : SchemeSpec<ValueDescriptor>(::ValueDescriptor) {
inline fun <reified E : Enum<E>> enum(name: String) = ValueDescriptor {
type(ValueType.STRING)
this.allowedValues = enumValues<E>().map { Value.of(it.name) }
}
// inline fun <reified E : Enum<E>> enum(name: String) = ValueDescriptor {
// type(ValueType.STRING)
// this.allowedValues = enumValues<E>().map { Value.of(it.name) }
// }
// /**
// * Build a value descriptor from annotation

135
dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/serialization/JsonMeta.kt Normal file
View File

@ -0,0 +1,135 @@
package hep.dataforge.meta.serialization
import hep.dataforge.meta.Meta
import hep.dataforge.meta.MetaBase
import hep.dataforge.meta.MetaItem
import hep.dataforge.meta.descriptors.ItemDescriptor
import hep.dataforge.meta.descriptors.NodeDescriptor
import hep.dataforge.meta.descriptors.ValueDescriptor
import hep.dataforge.names.NameToken
import hep.dataforge.names.toName
import hep.dataforge.values.*
import kotlinx.serialization.json.*
/**
* @param descriptor reserved for custom serialization in future
*/
fun Value.toJson(descriptor: ValueDescriptor? = null): JsonElement {
return if (isList()) {
JsonArray(list.map { it.toJson() })
} else {
when (type) {
ValueType.NUMBER -> JsonPrimitive(number)
ValueType.STRING -> JsonPrimitive(string)
ValueType.BOOLEAN -> JsonPrimitive(boolean)
ValueType.NULL -> JsonNull
}
}
}
//Use these methods to customize JSON key mapping
private fun NameToken.toJsonKey(descriptor: ItemDescriptor?) = toString()
//private fun NodeDescriptor?.getDescriptor(key: String) = this?.items?.get(key)
fun Meta.toJson(descriptor: NodeDescriptor? = null): JsonObject {
//TODO search for same name siblings and arrange them into arrays
val map = this.items.entries.associate { (name, item) ->
val itemDescriptor = descriptor?.items?.get(name.body)
val key = name.toJsonKey(itemDescriptor)
val value = when (item) {
is MetaItem.ValueItem -> {
item.value.toJson(itemDescriptor as? ValueDescriptor)
}
is MetaItem.NodeItem -> {
item.node.toJson(itemDescriptor as? NodeDescriptor)
}
}
key to value
}
return JsonObject(map)
}
fun JsonElement.toMeta(descriptor: NodeDescriptor? = null): Meta {
return when (val item = toMetaItem(descriptor)) {
is MetaItem.NodeItem<*> -> item.node
is MetaItem.ValueItem -> item.value.toMeta()
}
}
fun JsonPrimitive.toValue(descriptor: ValueDescriptor?): Value {
return when (this) {
JsonNull -> Null
else -> this.content.parseValue() // Optimize number and boolean parsing
}
}
fun JsonElement.toMetaItem(descriptor: ItemDescriptor? = null): MetaItem<JsonMeta> = when (this) {
is JsonPrimitive -> {
val value = this.toValue(descriptor as? ValueDescriptor)
MetaItem.ValueItem(value)
}
is JsonObject -> {
val meta = JsonMeta(this, descriptor as? NodeDescriptor)
MetaItem.NodeItem(meta)
}
is JsonArray -> {
if (this.all { it is JsonPrimitive }) {
val value = if (isEmpty()) {
Null
} else {
ListValue(
map<JsonElement, Value> {
//We already checked that all values are primitives
(it as JsonPrimitive).toValue(descriptor as? ValueDescriptor)
}
)
}
MetaItem.ValueItem(value)
} else {
json {
"@value" to this@toMetaItem
}.toMetaItem(descriptor)
}
}
}
class JsonMeta(val json: JsonObject, val descriptor: NodeDescriptor? = null) : MetaBase() {
@Suppress("UNCHECKED_CAST")
private operator fun MutableMap<String, MetaItem<JsonMeta>>.set(key: String, value: JsonElement): Unit {
val itemDescriptor = descriptor?.items?.get(key)
return when (value) {
is JsonPrimitive -> {
this[key] = MetaItem.ValueItem(value.toValue(itemDescriptor as? ValueDescriptor)) as MetaItem<JsonMeta>
}
is JsonObject -> {
this[key] = MetaItem.NodeItem(JsonMeta(value, itemDescriptor as? NodeDescriptor))
}
is JsonArray -> {
when {
value.all { it is JsonPrimitive } -> {
val listValue = ListValue(
value.map {
//We already checked that all values are primitives
(it as JsonPrimitive).toValue(itemDescriptor as? ValueDescriptor)
}
)
this[key] = MetaItem.ValueItem(listValue) as MetaItem<JsonMeta>
}
else -> value.forEachIndexed { index, jsonElement ->
this["$key[$index]"] = jsonElement.toMetaItem(itemDescriptor)
}
}
}
}
}
override val items: Map<NameToken, MetaItem<JsonMeta>> by lazy {
val map = HashMap<String, MetaItem<JsonMeta>>()
json.forEach { (key, value) -> map[key] = value }
map.mapKeys { it.key.toName().first()!! }
}
}

110
dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/serialization/MetaSerializer.kt Normal file
View File

@ -0,0 +1,110 @@
package hep.dataforge.meta.serialization
import hep.dataforge.meta.*
import hep.dataforge.names.NameToken
import hep.dataforge.values.*
import kotlinx.serialization.*
import kotlinx.serialization.builtins.MapSerializer
import kotlinx.serialization.builtins.list
import kotlinx.serialization.builtins.serializer
import kotlinx.serialization.json.JsonInput
import kotlinx.serialization.json.JsonObjectSerializer
import kotlinx.serialization.json.JsonOutput
@Serializer(Value::class)
@OptIn(InternalSerializationApi::class)
object ValueSerializer : KSerializer<Value> {
// private val valueTypeSerializer = EnumSerializer(ValueType::class)
private val listSerializer by lazy { ValueSerializer.list }
override val descriptor: SerialDescriptor = SerialDescriptor("Value") {
boolean("isList")
enum<ValueType>("valueType")
string("value")
}
private fun Decoder.decodeValue(): Value {
return when (decode(ValueType.serializer())) {
ValueType.NULL -> Null
ValueType.NUMBER -> decodeDouble().asValue() //TODO differentiate?
ValueType.BOOLEAN -> decodeBoolean().asValue()
ValueType.STRING -> decodeString().asValue()
}
}
override fun deserialize(decoder: Decoder): Value {
val isList = decoder.decodeBoolean()
return if (isList) {
listSerializer.deserialize(decoder).asValue()
} else {
decoder.decodeValue()
}
}
private fun Encoder.encodeValue(value: Value) {
encode(ValueType.serializer(), value.type)
when (value.type) {
ValueType.NULL -> {
// do nothing
}
ValueType.NUMBER -> encodeDouble(value.double)
ValueType.BOOLEAN -> encodeBoolean(value.boolean)
ValueType.STRING -> encodeString(value.string)
}
}
override fun serialize(encoder: Encoder, value: Value) {
encoder.encodeBoolean(value.isList())
if (value.isList()) {
listSerializer.serialize(encoder, value.list)
} else {
encoder.encodeValue(value)
}
}
}
private class DeserializedMeta(override val items: Map<NameToken, MetaItem<*>>) : MetaBase()
/**
* Serialized for meta
*/
@Serializer(Meta::class)
object MetaSerializer : KSerializer<Meta> {
private val mapSerializer = MapSerializer(
String.serializer(),
MetaItem.serializer(MetaSerializer)
)
override val descriptor: SerialDescriptor get() = mapSerializer.descriptor
override fun deserialize(decoder: Decoder): Meta {
return if (decoder is JsonInput) {
JsonObjectSerializer.deserialize(decoder).toMeta()
} else {
DeserializedMeta(mapSerializer.deserialize(decoder).mapKeys { NameToken(it.key) })
}
}
override fun serialize(encoder: Encoder, value: Meta) {
if (encoder is JsonOutput) {
JsonObjectSerializer.serialize(encoder, value.toJson())
} else {
mapSerializer.serialize(encoder, value.items.mapKeys { it.key.toString() })
}
}
}
@Serializer(Config::class)
object ConfigSerializer : KSerializer<Config> {
override val descriptor: SerialDescriptor = MetaSerializer.descriptor
override fun deserialize(decoder: Decoder): Config {
return MetaSerializer.deserialize(decoder).asConfig()
}
override fun serialize(encoder: Encoder, value: Config) {
MetaSerializer.serialize(encoder, value)
}
}

67
dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/serialization/serializationUtils.kt Normal file
View File

@ -0,0 +1,67 @@
package hep.dataforge.meta.serialization
import hep.dataforge.meta.DFExperimental
import kotlinx.serialization.*
import kotlinx.serialization.builtins.DoubleArraySerializer
import kotlinx.serialization.internal.*
fun SerialDescriptorBuilder.boolean(name: String, isOptional: Boolean = false, vararg annotations: Annotation) =
element(name, PrimitiveDescriptor(name, PrimitiveKind.BOOLEAN), isOptional = isOptional, annotations = annotations.toList())
fun SerialDescriptorBuilder.string(name: String, isOptional: Boolean = false, vararg annotations: Annotation) =
element(name, PrimitiveDescriptor(name, PrimitiveKind.STRING), isOptional = isOptional, annotations = annotations.toList())
fun SerialDescriptorBuilder.int(name: String, isOptional: Boolean = false, vararg annotations: Annotation) =
element(name, PrimitiveDescriptor(name, PrimitiveKind.INT), isOptional = isOptional, annotations = annotations.toList())
fun SerialDescriptorBuilder.double(name: String, isOptional: Boolean = false, vararg annotations: Annotation) =
element(name,PrimitiveDescriptor(name, PrimitiveKind.DOUBLE), isOptional = isOptional, annotations = annotations.toList())
fun SerialDescriptorBuilder.float(name: String, isOptional: Boolean = false, vararg annotations: Annotation) =
element(name, PrimitiveDescriptor(name, PrimitiveKind.FLOAT), isOptional = isOptional, annotations = annotations.toList())
fun SerialDescriptorBuilder.long(name: String, isOptional: Boolean = false, vararg annotations: Annotation) =
element(name, PrimitiveDescriptor(name, PrimitiveKind.LONG), isOptional = isOptional, annotations = annotations.toList())
fun SerialDescriptorBuilder.doubleArray(name: String, isOptional: Boolean = false, vararg annotations: Annotation) =
element(name, DoubleArraySerializer().descriptor, isOptional = isOptional, annotations = annotations.toList())
@OptIn(InternalSerializationApi::class)
inline fun <reified E : Enum<E>> SerialDescriptorBuilder.enum(name: String, isOptional: Boolean = false, vararg annotations: Annotation) {
val enumDescriptor = SerialDescriptor(serialName, UnionKind.ENUM_KIND) {
enumValues<E>().forEach {
val fqn = "$serialName.${it.name}"
val enumMemberDescriptor = SerialDescriptor(fqn, StructureKind.OBJECT)
element(it.name, enumMemberDescriptor)
}
}
element(name, enumDescriptor, isOptional = isOptional, annotations = annotations.toList())
}
//inline fun <reified T : Any> KSerializer<T>.descriptor(
// name: String,
// block: SerialDescriptorBuilder.() -> Unit
//): SerialDescriptor =
// SerialDescriptorBuilder(SerialClassDescImpl(name)).apply(block).build()
@DFExperimental
inline fun <R> Decoder.decodeStructure(
desc: SerialDescriptor,
vararg typeParams: KSerializer<*> = emptyArray(),
crossinline block: CompositeDecoder.() -> R
): R {
val decoder = beginStructure(desc, *typeParams)
val res = decoder.block()
decoder.endStructure(desc)
return res
}
inline fun Encoder.encodeStructure(
desc: SerialDescriptor,
vararg typeParams: KSerializer<*> = emptyArray(),
block: CompositeEncoder.() -> Unit
) {
val encoder = beginStructure(desc, *typeParams)
encoder.block()
encoder.endStructure(desc)
}

8
dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/transformations/MetaTransformation.kt
View File

@ -90,7 +90,7 @@ inline class MetaTransformation(val transformations: Collection<TransformationRu
* Produce new meta using only those items that match transformation rules
*/
fun transform(source: Meta): Meta =
buildMeta {
Meta {
transformations.forEach { rule ->
rule.selectItems(source).forEach { name ->
rule.transformItem(name, source[name], this)
@ -102,7 +102,7 @@ inline class MetaTransformation(val transformations: Collection<TransformationRu
* Transform a meta, replacing all elements found in rules with transformed entries
*/
fun apply(source: Meta): Meta =
buildMeta(source) {
source.edit {
transformations.forEach { rule ->
rule.selectItems(source).forEach { name ->
remove(name)
@ -156,8 +156,8 @@ class MetaTransformationBuilder {
fun keep(regex: String) {
transformations.add(
RegexItemTransformationRule(regex.toRegex()) { name, _, metaItem ->
setItem(name, metaItem)
})
setItem(name, metaItem)
})
}
/**

30
dataforge-meta/src/commonMain/kotlin/hep/dataforge/names/Name.kt
View File

@ -1,11 +1,14 @@
package hep.dataforge.names
import kotlinx.serialization.*
/**
* The general interface for working with names.
* The name is a dot separated list of strings like `token1.token2.token3`.
* Each token could contain additional index in square brackets.
*/
@Serializable
class Name(val tokens: List<NameToken>) {
val length get() = tokens.size
@ -51,10 +54,21 @@ class Name(val tokens: List<NameToken>) {
}
companion object {
@Serializer(Name::class)
companion object: KSerializer<Name> {
const val NAME_SEPARATOR = "."
val EMPTY = Name(emptyList())
override val descriptor: SerialDescriptor = PrimitiveDescriptor("Name", PrimitiveKind.STRING)
override fun deserialize(decoder: Decoder): Name {
return decoder.decodeString().toName()
}
override fun serialize(encoder: Encoder, value: Name) {
encoder.encodeString(value.toString())
}
}
}
@ -63,6 +77,7 @@ class Name(val tokens: List<NameToken>) {
* Following symbols are prohibited in name tokens: `{}.:\`.
* A name token could have appendix in square brackets called *index*
*/
@Serializable
data class NameToken(val body: String, val index: String = "") {
init {
@ -82,6 +97,19 @@ data class NameToken(val body: String, val index: String = "") {
}
fun hasIndex() = index.isNotEmpty()
@Serializer(NameToken::class)
companion object :KSerializer<NameToken>{
override val descriptor: SerialDescriptor = PrimitiveDescriptor("NameToken", PrimitiveKind.STRING)
override fun deserialize(decoder: Decoder): NameToken {
return decoder.decodeString().toName().first()!!
}
override fun serialize(encoder: Encoder, value: NameToken) {
encoder.encodeString(value.toString())
}
}
}
/**

3
dataforge-meta/src/commonMain/kotlin/hep/dataforge/values/Value.kt
View File

@ -1,5 +1,7 @@
package hep.dataforge.values
import kotlinx.serialization.Serializable
/**
* The list of supported Value types.
@ -7,6 +9,7 @@ package hep.dataforge.values
* Time value and binary value are represented by string
*
*/
@Serializable
enum class ValueType {
NUMBER, STRING, BOOLEAN, NULL
}