From 332d38df77895ea3f72e6ba2a5b2d41233c08d29 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Mon, 10 Jun 2024 20:28:57 +0300 Subject: [PATCH] First implementation for protobuf converter --- .../kscience/dataforge/data/ActionsTest.kt | 16 ++-- .../dataforge-io-proto/build.gradle.kts | 32 +++++++ .../src/commonMain/proto/meta.proto | 19 ++++ .../src/jvmMain/kotlin/ProtoMetaFormat.kt | 92 +++++++++++++++++++ .../src/jvmTest/kotlin/ProtoBufTest.kt | 39 ++++++++ .../space/kscience/dataforge/meta/JsonMeta.kt | 5 +- .../space/kscience/dataforge/meta/Scheme.kt | 5 +- .../kscience/dataforge/names/NameToken.kt | 27 +++++- .../kscience/dataforge/names/NameTest.kt | 14 ++- settings.gradle.kts | 1 + 10 files changed, 233 insertions(+), 17 deletions(-) create mode 100644 dataforge-io/dataforge-io-proto/build.gradle.kts create mode 100644 dataforge-io/dataforge-io-proto/src/commonMain/proto/meta.proto create mode 100644 dataforge-io/dataforge-io-proto/src/jvmMain/kotlin/ProtoMetaFormat.kt create mode 100644 dataforge-io/dataforge-io-proto/src/jvmTest/kotlin/ProtoBufTest.kt diff --git a/dataforge-data/src/jvmTest/kotlin/space/kscience/dataforge/data/ActionsTest.kt b/dataforge-data/src/jvmTest/kotlin/space/kscience/dataforge/data/ActionsTest.kt index 13660eee..d608cbd3 100644 --- a/dataforge-data/src/jvmTest/kotlin/space/kscience/dataforge/data/ActionsTest.kt +++ b/dataforge-data/src/jvmTest/kotlin/space/kscience/dataforge/data/ActionsTest.kt @@ -1,8 +1,5 @@ package space.kscience.dataforge.data -import kotlinx.coroutines.flow.collect -import kotlinx.coroutines.flow.onEach -import kotlinx.coroutines.flow.take import kotlinx.coroutines.test.runTest import space.kscience.dataforge.actions.Action import space.kscience.dataforge.actions.invoke @@ -16,27 +13,28 @@ import kotlin.time.Duration.Companion.milliseconds internal class ActionsTest { @Test fun testStaticMapAction() = runTest(timeout = 500.milliseconds) { + val plusOne = Action.mapping { + result { it + 1 } + } + val data: DataTree = DataTree { repeat(10) { putValue(it.toString(), it) } } - val plusOne = Action.mapping { - result { it + 1 } - } val result = plusOne(data) assertEquals(2, result["1"]?.await()) } @Test fun testDynamicMapAction() = runTest(timeout = 500.milliseconds) { - val source: MutableDataTree = MutableDataTree() - val plusOne = Action.mapping { result { it + 1 } } + val source: MutableDataTree = MutableDataTree() + val result = plusOne(source) @@ -44,7 +42,7 @@ internal class ActionsTest { source.updateValue(it.toString(), it) } - result.updates.take(10).onEach { println(it.name) }.collect() +// result.updates.take(10).onEach { println(it.name) }.collect() assertEquals(2, result["1"]?.await()) } diff --git a/dataforge-io/dataforge-io-proto/build.gradle.kts b/dataforge-io/dataforge-io-proto/build.gradle.kts new file mode 100644 index 00000000..2899a2b1 --- /dev/null +++ b/dataforge-io/dataforge-io-proto/build.gradle.kts @@ -0,0 +1,32 @@ +plugins { + id("space.kscience.gradle.mpp") + id("com.squareup.wire") version "4.9.9" +} + +description = "ProtoBuf meta IO" + +kscience { + jvm() + dependencies { + api(projects.dataforgeIo) + api("com.squareup.wire:wire-runtime:4.9.9") + } + useSerialization { + protobuf() + } +} + +wire { + kotlin { + sourcePath { + srcDir("src/commonMain/proto") + } + } +} + +readme { + maturity = space.kscience.gradle.Maturity.PROTOTYPE + description = """ + ProtoBuf Meta representation + """.trimIndent() +} diff --git a/dataforge-io/dataforge-io-proto/src/commonMain/proto/meta.proto b/dataforge-io/dataforge-io-proto/src/commonMain/proto/meta.proto new file mode 100644 index 00000000..2fd33eab --- /dev/null +++ b/dataforge-io/dataforge-io-proto/src/commonMain/proto/meta.proto @@ -0,0 +1,19 @@ +syntax = "proto3"; +package space.kscience.dataforge.io.proto; + +message ProtoMeta { + message ProtoValue { + oneof value { + string stringValue = 2; + bool booleanValue = 3; + double doubleValue = 4; + float floatValue = 5; + int32 int32Value = 6; + int64 int64Value = 7; + bytes bytesValue = 8; + } + } + repeated ProtoValue value = 1; + + map items = 2; +} \ No newline at end of file diff --git a/dataforge-io/dataforge-io-proto/src/jvmMain/kotlin/ProtoMetaFormat.kt b/dataforge-io/dataforge-io-proto/src/jvmMain/kotlin/ProtoMetaFormat.kt new file mode 100644 index 00000000..5e55099b --- /dev/null +++ b/dataforge-io/dataforge-io-proto/src/jvmMain/kotlin/ProtoMetaFormat.kt @@ -0,0 +1,92 @@ +package space.kscience.dataforge.io.proto + +import kotlinx.io.Sink +import kotlinx.io.Source +import kotlinx.io.asInputStream +import kotlinx.io.asOutputStream +import org.slf4j.LoggerFactory +import space.kscience.dataforge.io.MetaFormat +import space.kscience.dataforge.meta.* +import space.kscience.dataforge.meta.descriptors.MetaDescriptor +import space.kscience.dataforge.names.NameToken + +internal class ProtoMetaWrapper(private val proto: ProtoMeta) : Meta { + + private fun ProtoMeta.ProtoValue.toValue(): Value = when { + stringValue != null -> stringValue.asValue() + booleanValue != null -> booleanValue.asValue() + doubleValue != null -> doubleValue.asValue() + floatValue != null -> floatValue.asValue() + int32Value != null -> int32Value.asValue() + int64Value != null -> int64Value.asValue() + bytesValue != null -> bytesValue.toByteArray().asValue() + else -> Null + } + + override val value: Value? + get() = when (proto.value_.size) { + 0 -> null + 1 -> proto.value_[0].toValue() + else -> proto.value_.map { it.toValue() }.asValue() + } + + + override val items: Map + get() = proto.items.entries.associate { NameToken.parse(it.key) to ProtoMetaWrapper(it.value) } + + override fun toString(): String = Meta.toString(this) + + override fun equals(other: Any?): Boolean = Meta.equals(this, other as? Meta) + + override fun hashCode(): Int = Meta.hashCode(this) +} + +internal fun Meta.toProto(): ProtoMeta { + + + fun MutableList.appendProtoValues(value: Value): Unit { + when (value.type) { + ValueType.NULL -> { + //do nothing + } + + ValueType.NUMBER -> when (value.value) { + is Int, is Short, is Byte -> add(ProtoMeta.ProtoValue(int32Value = value.int)) + is Long -> add(ProtoMeta.ProtoValue(int64Value = value.long)) + is Float -> add(ProtoMeta.ProtoValue(floatValue = value.float)) + else -> { + LoggerFactory.getLogger(ProtoMeta::class.java) + .warn("Unknown number type ${value.value} encoded as Double") + add(ProtoMeta.ProtoValue(doubleValue = value.double)) + } + } + + ValueType.STRING -> add(ProtoMeta.ProtoValue(stringValue = value.string)) + ValueType.BOOLEAN -> add(ProtoMeta.ProtoValue(booleanValue = value.boolean)) + ValueType.LIST -> { + value.list.forEach { + if (it.type == ValueType.LIST) { + error("Nested lists are not supported") + } else { + appendProtoValues(it) + } + } + } + } + } + + return ProtoMeta( + value_ = buildList { value?.let { appendProtoValues(it) } }, + items.entries.associate { it.key.toString() to it.value.toProto() } + ) +} + + +public object ProtoMetaFormat : MetaFormat { + override fun writeMeta(sink: Sink, meta: Meta, descriptor: MetaDescriptor?) { + ProtoMeta.ADAPTER.encode(sink.asOutputStream(), meta.toProto()) + } + + override fun readMeta(source: Source, descriptor: MetaDescriptor?): Meta = + ProtoMetaWrapper(ProtoMeta.ADAPTER.decode(source.asInputStream())) +} \ No newline at end of file diff --git a/dataforge-io/dataforge-io-proto/src/jvmTest/kotlin/ProtoBufTest.kt b/dataforge-io/dataforge-io-proto/src/jvmTest/kotlin/ProtoBufTest.kt new file mode 100644 index 00000000..bf819b6e --- /dev/null +++ b/dataforge-io/dataforge-io-proto/src/jvmTest/kotlin/ProtoBufTest.kt @@ -0,0 +1,39 @@ +package space.kscience.dataforge.io.proto + +import space.kscience.dataforge.meta.Meta +import space.kscience.dataforge.meta.get +import kotlin.test.Test +import kotlin.test.assertEquals + +class ProtoBufTest { + + @Test + fun testProtoBufMetaFormat(){ + val meta = Meta { + "a" put 22 + "node" put { + "b" put "DDD" + "c" put 11.1 + "d" put { + "d1" put { + "d11" put "aaa" + "d12" put "bbb" + } + "d2" put 2 + } + "array" put doubleArrayOf(1.0, 2.0, 3.0) + } + } + val buffer = kotlinx.io.Buffer() + ProtoMetaFormat.writeTo(buffer,meta) + val result = ProtoMetaFormat.readFrom(buffer) + + println(result["a"]?.value) + + meta.items.keys.forEach { + assertEquals(meta[it],result[it],"${meta[it]} != ${result[it]}") + } + + assertEquals(meta, result) + } +} \ No newline at end of file diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/JsonMeta.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/JsonMeta.kt index 8da8b2d3..049c1733 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/JsonMeta.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/JsonMeta.kt @@ -117,8 +117,11 @@ private fun MutableMap.addJsonElement( } else { val indexKey = descriptor?.indexKey ?: Meta.INDEX_KEY element.forEachIndexed { serial, childElement -> - val index = (childElement as? JsonObject)?.get(indexKey)?.jsonPrimitive?.content + + val index = (childElement as? JsonObject) + ?.get(indexKey)?.jsonPrimitive?.content ?: serial.toString() + val child: SealedMeta = when (childElement) { is JsonObject -> childElement.toMeta(descriptor) is JsonArray -> { diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/Scheme.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/Scheme.kt index fe121c42..bc05cb5d 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/Scheme.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/Scheme.kt @@ -221,13 +221,14 @@ public fun Configurable.updateWith( /** * A delegate that uses a [MetaReader] to wrap a child of this provider */ -public fun MutableMeta.scheme( +public fun MutableMetaProvider.scheme( spec: SchemeSpec, key: Name? = null, ): ReadWriteProperty = object : ReadWriteProperty { override fun getValue(thisRef: Any?, property: KProperty<*>): T { val name = key ?: property.name.asName() - return spec.write(getOrCreate(name)) + val node = get(name)?: MutableMeta().also { set(name,it) } + return spec.write(node) } override fun setValue(thisRef: Any?, property: KProperty<*>, value: T) { diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/names/NameToken.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/names/NameToken.kt index 83752b9a..3994ef27 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/names/NameToken.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/names/NameToken.kt @@ -67,10 +67,29 @@ public class NameToken(public val body: String, public val index: String? = null * Parse name token from a string */ public fun parse(string: String): NameToken { - val body = string.substringBefore('[') - val index = string.substringAfter('[', "") - if (index.isNotEmpty() && !index.endsWith(']')) error("NameToken with index must end with ']'") - return NameToken(body, index.removeSuffix("]")) + var indexStart = -1 + var indexEnd = -1 + string.forEachIndexed { index, c -> + when (c) { + '[' -> when { + indexStart >= 0 -> error("Second opening bracket not allowed in NameToken: $string") + else -> indexStart = index + } + + ']' -> when { + indexStart < 0 -> error("Closing index bracket could not be used before opening bracket in NameToken: $string") + indexEnd >= 0 -> error("Second closing bracket not allowed in NameToken: $string") + else -> indexEnd = index + } + + else -> if(indexEnd>=0) error("Symbols not allowed after index in NameToken: $string") + } + } + if(indexStart>=0 && indexEnd<0) error("Opening bracket without closing bracket not allowed in NameToken: $string") + return NameToken( + if(indexStart>=0) string.substring(0, indexStart) else string, + if(indexStart>=0) string.substring(indexStart + 1, indexEnd) else null + ) } } } diff --git a/dataforge-meta/src/commonTest/kotlin/space/kscience/dataforge/names/NameTest.kt b/dataforge-meta/src/commonTest/kotlin/space/kscience/dataforge/names/NameTest.kt index db630487..25725333 100644 --- a/dataforge-meta/src/commonTest/kotlin/space/kscience/dataforge/names/NameTest.kt +++ b/dataforge-meta/src/commonTest/kotlin/space/kscience/dataforge/names/NameTest.kt @@ -56,10 +56,22 @@ class NameTest { val token2 = NameToken.parse("token-body") assertEquals("token-body", token2.body) - assertEquals("", token2.index) + assertEquals(null, token2.index) + +// val token3 = NameToken.parse("[token-index]") +// assertEquals("", token3.body) +// assertEquals("token-index", token3.index) + + assertFails{ + NameToken.parse("[token-index]") + } assertFails { NameToken.parse("token[22") } + + assertFails { + NameToken.parse("token[22]ddd") + } } } \ No newline at end of file diff --git a/settings.gradle.kts b/settings.gradle.kts index ca872038..35eae74e 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -43,6 +43,7 @@ include( ":dataforge-meta", ":dataforge-io", ":dataforge-io:dataforge-io-yaml", + ":dataforge-io:dataforge-io-proto", ":dataforge-context", ":dataforge-data", ":dataforge-workspace",