First implementation for protobuf converter

dataforge-data/src/jvmTest/kotlin/space/kscience/dataforge/data/ActionsTest.kt

@@ -1,8 +1,5 @@
 package space.kscience.dataforge.data
 
-import kotlinx.coroutines.flow.collect
-import kotlinx.coroutines.flow.onEach
-import kotlinx.coroutines.flow.take
 import kotlinx.coroutines.test.runTest
 import space.kscience.dataforge.actions.Action
 import space.kscience.dataforge.actions.invoke
@@ -16,27 +13,28 @@ import kotlin.time.Duration.Companion.milliseconds
 internal class ActionsTest {
     @Test
     fun testStaticMapAction() = runTest(timeout = 500.milliseconds) {
+        val plusOne = Action.mapping<Int, Int> {
+            result { it + 1 }
+        }
+
         val data: DataTree<Int> = DataTree {
             repeat(10) {
                 putValue(it.toString(), it)
             }
         }
 
-        val plusOne = Action.mapping<Int, Int> {
-            result { it + 1 }
-        }
         val result = plusOne(data)
         assertEquals(2, result["1"]?.await())
     }
 
     @Test
     fun testDynamicMapAction() = runTest(timeout = 500.milliseconds) {
-        val source: MutableDataTree<Int> = MutableDataTree()
-
         val plusOne = Action.mapping<Int, Int> {
             result { it + 1 }
         }
 
+        val source: MutableDataTree<Int> = MutableDataTree()
+
         val result = plusOne(source)
 
 
@@ -44,7 +42,7 @@ internal class ActionsTest {
             source.updateValue(it.toString(), it)
         }
 
-        result.updates.take(10).onEach { println(it.name) }.collect()
+//        result.updates.take(10).onEach { println(it.name) }.collect()
 
         assertEquals(2, result["1"]?.await())
     }

dataforge-io/dataforge-io-proto/build.gradle.kts

@@ -0,0 +1,32 @@
+plugins {
+    id("space.kscience.gradle.mpp")
+    id("com.squareup.wire") version "4.9.9"
+}
+
+description = "ProtoBuf meta IO"
+
+kscience {
+    jvm()
+    dependencies {
+        api(projects.dataforgeIo)
+        api("com.squareup.wire:wire-runtime:4.9.9")
+    }
+    useSerialization {
+        protobuf()
+    }
+}
+
+wire {
+    kotlin {
+        sourcePath {
+            srcDir("src/commonMain/proto")
+        }
+    }
+}
+
+readme {
+    maturity = space.kscience.gradle.Maturity.PROTOTYPE
+    description = """
+        ProtoBuf Meta representation
+    """.trimIndent()
+}

dataforge-io/dataforge-io-proto/src/commonMain/proto/meta.proto

@@ -0,0 +1,19 @@
+syntax = "proto3";
+package space.kscience.dataforge.io.proto;
+
+message ProtoMeta {
+  message ProtoValue {
+    oneof value {
+      string stringValue = 2;
+      bool booleanValue = 3;
+      double doubleValue = 4;
+      float floatValue = 5;
+      int32 int32Value = 6;
+      int64 int64Value = 7;
+      bytes bytesValue = 8;
+    }
+  }
+  repeated ProtoValue value = 1;
+
+  map<string, ProtoMeta> items = 2;
+}

dataforge-io/dataforge-io-proto/src/jvmMain/kotlin/ProtoMetaFormat.kt

@@ -0,0 +1,92 @@
+package space.kscience.dataforge.io.proto
+
+import kotlinx.io.Sink
+import kotlinx.io.Source
+import kotlinx.io.asInputStream
+import kotlinx.io.asOutputStream
+import org.slf4j.LoggerFactory
+import space.kscience.dataforge.io.MetaFormat
+import space.kscience.dataforge.meta.*
+import space.kscience.dataforge.meta.descriptors.MetaDescriptor
+import space.kscience.dataforge.names.NameToken
+
+internal class ProtoMetaWrapper(private val proto: ProtoMeta) : Meta {
+
+    private fun ProtoMeta.ProtoValue.toValue(): Value = when {
+        stringValue != null -> stringValue.asValue()
+        booleanValue != null -> booleanValue.asValue()
+        doubleValue != null -> doubleValue.asValue()
+        floatValue != null -> floatValue.asValue()
+        int32Value != null -> int32Value.asValue()
+        int64Value != null -> int64Value.asValue()
+        bytesValue != null -> bytesValue.toByteArray().asValue()
+        else -> Null
+    }
+
+    override val value: Value?
+        get() = when (proto.value_.size) {
+            0 -> null
+            1 -> proto.value_[0].toValue()
+            else -> proto.value_.map { it.toValue() }.asValue()
+        }
+
+
+    override val items: Map<NameToken, Meta>
+        get() = proto.items.entries.associate { NameToken.parse(it.key) to ProtoMetaWrapper(it.value) }
+
+    override fun toString(): String = Meta.toString(this)
+
+    override fun equals(other: Any?): Boolean = Meta.equals(this, other as? Meta)
+
+    override fun hashCode(): Int = Meta.hashCode(this)
+}
+
+internal fun Meta.toProto(): ProtoMeta {
+
+
+    fun MutableList<ProtoMeta.ProtoValue>.appendProtoValues(value: Value): Unit {
+        when (value.type) {
+            ValueType.NULL -> {
+                //do nothing
+            }
+
+            ValueType.NUMBER -> when (value.value) {
+                is Int, is Short, is Byte -> add(ProtoMeta.ProtoValue(int32Value = value.int))
+                is Long -> add(ProtoMeta.ProtoValue(int64Value = value.long))
+                is Float -> add(ProtoMeta.ProtoValue(floatValue = value.float))
+                else -> {
+                    LoggerFactory.getLogger(ProtoMeta::class.java)
+                        .warn("Unknown number type ${value.value} encoded as Double")
+                    add(ProtoMeta.ProtoValue(doubleValue = value.double))
+                }
+            }
+
+            ValueType.STRING -> add(ProtoMeta.ProtoValue(stringValue = value.string))
+            ValueType.BOOLEAN -> add(ProtoMeta.ProtoValue(booleanValue = value.boolean))
+            ValueType.LIST -> {
+                value.list.forEach {
+                    if (it.type == ValueType.LIST) {
+                        error("Nested lists are not supported")
+                    } else {
+                        appendProtoValues(it)
+                    }
+                }
+            }
+        }
+    }
+
+    return ProtoMeta(
+        value_ = buildList { value?.let { appendProtoValues(it) } },
+        items.entries.associate { it.key.toString() to it.value.toProto() }
+    )
+}
+
+
+public object ProtoMetaFormat : MetaFormat {
+    override fun writeMeta(sink: Sink, meta: Meta, descriptor: MetaDescriptor?) {
+        ProtoMeta.ADAPTER.encode(sink.asOutputStream(), meta.toProto())
+    }
+
+    override fun readMeta(source: Source, descriptor: MetaDescriptor?): Meta =
+        ProtoMetaWrapper(ProtoMeta.ADAPTER.decode(source.asInputStream()))
+}

dataforge-io/dataforge-io-proto/src/jvmTest/kotlin/ProtoBufTest.kt

@@ -0,0 +1,39 @@
+package space.kscience.dataforge.io.proto
+
+import space.kscience.dataforge.meta.Meta
+import space.kscience.dataforge.meta.get
+import kotlin.test.Test
+import kotlin.test.assertEquals
+
+class ProtoBufTest {
+
+    @Test
+    fun testProtoBufMetaFormat(){
+        val meta = Meta {
+            "a" put 22
+            "node" put {
+                "b" put "DDD"
+                "c" put 11.1
+                "d" put {
+                    "d1" put {
+                        "d11" put "aaa"
+                        "d12" put "bbb"
+                    }
+                    "d2" put 2
+                }
+                "array" put doubleArrayOf(1.0, 2.0, 3.0)
+            }
+        }
+        val buffer = kotlinx.io.Buffer()
+        ProtoMetaFormat.writeTo(buffer,meta)
+        val result = ProtoMetaFormat.readFrom(buffer)
+
+        println(result["a"]?.value)
+
+        meta.items.keys.forEach {
+            assertEquals(meta[it],result[it],"${meta[it]} != ${result[it]}")
+        }
+
+        assertEquals(meta, result)
+    }
+}

dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/JsonMeta.kt

@@ -117,8 +117,11 @@ private fun MutableMap<NameToken, SealedMeta>.addJsonElement(
             } else {
                 val indexKey = descriptor?.indexKey ?: Meta.INDEX_KEY
                 element.forEachIndexed { serial, childElement ->
-                    val index = (childElement as? JsonObject)?.get(indexKey)?.jsonPrimitive?.content
+
+                    val index = (childElement as? JsonObject)
+                        ?.get(indexKey)?.jsonPrimitive?.content
                         ?: serial.toString()
+
                     val child: SealedMeta = when (childElement) {
                         is JsonObject -> childElement.toMeta(descriptor)
                         is JsonArray -> {

dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/Scheme.kt

@@ -221,13 +221,14 @@ public fun <T : Scheme> Configurable.updateWith(
 /**
  * A delegate that uses a [MetaReader] to wrap a child of this provider
  */
-public fun <T : Scheme> MutableMeta.scheme(
+public fun <T : Scheme> MutableMetaProvider.scheme(
     spec: SchemeSpec<T>,
     key: Name? = null,
 ): ReadWriteProperty<Any?, T> = object : ReadWriteProperty<Any?, T> {
     override fun getValue(thisRef: Any?, property: KProperty<*>): T {
         val name = key ?: property.name.asName()
-        return spec.write(getOrCreate(name))
+        val node = get(name)?: MutableMeta().also { set(name,it) }
+        return spec.write(node)
     }
 
     override fun setValue(thisRef: Any?, property: KProperty<*>, value: T) {

dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/names/NameToken.kt

@@ -67,10 +67,29 @@ public class NameToken(public val body: String, public val index: String? = null
          * Parse name token from a string
          */
         public fun parse(string: String): NameToken {
-            val body = string.substringBefore('[')
+            var indexStart = -1
-            val index = string.substringAfter('[', "")
+            var indexEnd = -1
-            if (index.isNotEmpty() && !index.endsWith(']')) error("NameToken with index must end with ']'")
+            string.forEachIndexed { index, c ->
-            return NameToken(body, index.removeSuffix("]"))
+                when (c) {
+                    '[' -> when {
+                        indexStart >= 0 -> error("Second opening bracket not allowed in NameToken: $string")
+                        else -> indexStart = index
+                    }
+
+                    ']' -> when {
+                        indexStart < 0 -> error("Closing index bracket could not be used before opening bracket in NameToken: $string")
+                        indexEnd >= 0 -> error("Second closing bracket not allowed in NameToken: $string")
+                        else -> indexEnd = index
+                    }
+
+                    else -> if(indexEnd>=0) error("Symbols not allowed after index in NameToken: $string")
+                }
+            }
+            if(indexStart>=0 && indexEnd<0) error("Opening bracket without closing bracket not allowed in NameToken: $string")
+            return NameToken(
+                if(indexStart>=0) string.substring(0, indexStart) else string,
+                if(indexStart>=0) string.substring(indexStart + 1, indexEnd) else null
+            )
         }
     }
 }

dataforge-meta/src/commonTest/kotlin/space/kscience/dataforge/names/NameTest.kt

@@ -56,10 +56,22 @@ class NameTest {
 
         val token2 = NameToken.parse("token-body")
         assertEquals("token-body", token2.body)
-        assertEquals("", token2.index)
+        assertEquals(null, token2.index)
+
+//        val token3 = NameToken.parse("[token-index]")
+//        assertEquals("", token3.body)
+//        assertEquals("token-index", token3.index)
+
+        assertFails{
+            NameToken.parse("[token-index]")
+        }
 
         assertFails {
             NameToken.parse("token[22")
         }
+
+        assertFails {
+            NameToken.parse("token[22]ddd")
+        }
     }
 }

settings.gradle.kts

@@ -43,6 +43,7 @@ include(
     ":dataforge-meta",
     ":dataforge-io",
     ":dataforge-io:dataforge-io-yaml",
+    ":dataforge-io:dataforge-io-proto",
     ":dataforge-context",
     ":dataforge-data",
     ":dataforge-workspace",