First implementation for protobuf converter
This commit is contained in:
parent
f79b7faeaf
commit
332d38df77
@ -1,8 +1,5 @@
|
||||
package space.kscience.dataforge.data
|
||||
|
||||
import kotlinx.coroutines.flow.collect
|
||||
import kotlinx.coroutines.flow.onEach
|
||||
import kotlinx.coroutines.flow.take
|
||||
import kotlinx.coroutines.test.runTest
|
||||
import space.kscience.dataforge.actions.Action
|
||||
import space.kscience.dataforge.actions.invoke
|
||||
@ -16,27 +13,28 @@ import kotlin.time.Duration.Companion.milliseconds
|
||||
internal class ActionsTest {
|
||||
@Test
|
||||
fun testStaticMapAction() = runTest(timeout = 500.milliseconds) {
|
||||
val plusOne = Action.mapping<Int, Int> {
|
||||
result { it + 1 }
|
||||
}
|
||||
|
||||
val data: DataTree<Int> = DataTree {
|
||||
repeat(10) {
|
||||
putValue(it.toString(), it)
|
||||
}
|
||||
}
|
||||
|
||||
val plusOne = Action.mapping<Int, Int> {
|
||||
result { it + 1 }
|
||||
}
|
||||
val result = plusOne(data)
|
||||
assertEquals(2, result["1"]?.await())
|
||||
}
|
||||
|
||||
@Test
|
||||
fun testDynamicMapAction() = runTest(timeout = 500.milliseconds) {
|
||||
val source: MutableDataTree<Int> = MutableDataTree()
|
||||
|
||||
val plusOne = Action.mapping<Int, Int> {
|
||||
result { it + 1 }
|
||||
}
|
||||
|
||||
val source: MutableDataTree<Int> = MutableDataTree()
|
||||
|
||||
val result = plusOne(source)
|
||||
|
||||
|
||||
@ -44,7 +42,7 @@ internal class ActionsTest {
|
||||
source.updateValue(it.toString(), it)
|
||||
}
|
||||
|
||||
result.updates.take(10).onEach { println(it.name) }.collect()
|
||||
// result.updates.take(10).onEach { println(it.name) }.collect()
|
||||
|
||||
assertEquals(2, result["1"]?.await())
|
||||
}
|
||||
|
32
dataforge-io/dataforge-io-proto/build.gradle.kts
Normal file
32
dataforge-io/dataforge-io-proto/build.gradle.kts
Normal file
@ -0,0 +1,32 @@
|
||||
plugins {
|
||||
id("space.kscience.gradle.mpp")
|
||||
id("com.squareup.wire") version "4.9.9"
|
||||
}
|
||||
|
||||
description = "ProtoBuf meta IO"
|
||||
|
||||
kscience {
|
||||
jvm()
|
||||
dependencies {
|
||||
api(projects.dataforgeIo)
|
||||
api("com.squareup.wire:wire-runtime:4.9.9")
|
||||
}
|
||||
useSerialization {
|
||||
protobuf()
|
||||
}
|
||||
}
|
||||
|
||||
wire {
|
||||
kotlin {
|
||||
sourcePath {
|
||||
srcDir("src/commonMain/proto")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
readme {
|
||||
maturity = space.kscience.gradle.Maturity.PROTOTYPE
|
||||
description = """
|
||||
ProtoBuf Meta representation
|
||||
""".trimIndent()
|
||||
}
|
@ -0,0 +1,19 @@
|
||||
syntax = "proto3";
|
||||
package space.kscience.dataforge.io.proto;
|
||||
|
||||
message ProtoMeta {
|
||||
message ProtoValue {
|
||||
oneof value {
|
||||
string stringValue = 2;
|
||||
bool booleanValue = 3;
|
||||
double doubleValue = 4;
|
||||
float floatValue = 5;
|
||||
int32 int32Value = 6;
|
||||
int64 int64Value = 7;
|
||||
bytes bytesValue = 8;
|
||||
}
|
||||
}
|
||||
repeated ProtoValue value = 1;
|
||||
|
||||
map<string, ProtoMeta> items = 2;
|
||||
}
|
@ -0,0 +1,92 @@
|
||||
package space.kscience.dataforge.io.proto
|
||||
|
||||
import kotlinx.io.Sink
|
||||
import kotlinx.io.Source
|
||||
import kotlinx.io.asInputStream
|
||||
import kotlinx.io.asOutputStream
|
||||
import org.slf4j.LoggerFactory
|
||||
import space.kscience.dataforge.io.MetaFormat
|
||||
import space.kscience.dataforge.meta.*
|
||||
import space.kscience.dataforge.meta.descriptors.MetaDescriptor
|
||||
import space.kscience.dataforge.names.NameToken
|
||||
|
||||
internal class ProtoMetaWrapper(private val proto: ProtoMeta) : Meta {
|
||||
|
||||
private fun ProtoMeta.ProtoValue.toValue(): Value = when {
|
||||
stringValue != null -> stringValue.asValue()
|
||||
booleanValue != null -> booleanValue.asValue()
|
||||
doubleValue != null -> doubleValue.asValue()
|
||||
floatValue != null -> floatValue.asValue()
|
||||
int32Value != null -> int32Value.asValue()
|
||||
int64Value != null -> int64Value.asValue()
|
||||
bytesValue != null -> bytesValue.toByteArray().asValue()
|
||||
else -> Null
|
||||
}
|
||||
|
||||
override val value: Value?
|
||||
get() = when (proto.value_.size) {
|
||||
0 -> null
|
||||
1 -> proto.value_[0].toValue()
|
||||
else -> proto.value_.map { it.toValue() }.asValue()
|
||||
}
|
||||
|
||||
|
||||
override val items: Map<NameToken, Meta>
|
||||
get() = proto.items.entries.associate { NameToken.parse(it.key) to ProtoMetaWrapper(it.value) }
|
||||
|
||||
override fun toString(): String = Meta.toString(this)
|
||||
|
||||
override fun equals(other: Any?): Boolean = Meta.equals(this, other as? Meta)
|
||||
|
||||
override fun hashCode(): Int = Meta.hashCode(this)
|
||||
}
|
||||
|
||||
internal fun Meta.toProto(): ProtoMeta {
|
||||
|
||||
|
||||
fun MutableList<ProtoMeta.ProtoValue>.appendProtoValues(value: Value): Unit {
|
||||
when (value.type) {
|
||||
ValueType.NULL -> {
|
||||
//do nothing
|
||||
}
|
||||
|
||||
ValueType.NUMBER -> when (value.value) {
|
||||
is Int, is Short, is Byte -> add(ProtoMeta.ProtoValue(int32Value = value.int))
|
||||
is Long -> add(ProtoMeta.ProtoValue(int64Value = value.long))
|
||||
is Float -> add(ProtoMeta.ProtoValue(floatValue = value.float))
|
||||
else -> {
|
||||
LoggerFactory.getLogger(ProtoMeta::class.java)
|
||||
.warn("Unknown number type ${value.value} encoded as Double")
|
||||
add(ProtoMeta.ProtoValue(doubleValue = value.double))
|
||||
}
|
||||
}
|
||||
|
||||
ValueType.STRING -> add(ProtoMeta.ProtoValue(stringValue = value.string))
|
||||
ValueType.BOOLEAN -> add(ProtoMeta.ProtoValue(booleanValue = value.boolean))
|
||||
ValueType.LIST -> {
|
||||
value.list.forEach {
|
||||
if (it.type == ValueType.LIST) {
|
||||
error("Nested lists are not supported")
|
||||
} else {
|
||||
appendProtoValues(it)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return ProtoMeta(
|
||||
value_ = buildList { value?.let { appendProtoValues(it) } },
|
||||
items.entries.associate { it.key.toString() to it.value.toProto() }
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
public object ProtoMetaFormat : MetaFormat {
|
||||
override fun writeMeta(sink: Sink, meta: Meta, descriptor: MetaDescriptor?) {
|
||||
ProtoMeta.ADAPTER.encode(sink.asOutputStream(), meta.toProto())
|
||||
}
|
||||
|
||||
override fun readMeta(source: Source, descriptor: MetaDescriptor?): Meta =
|
||||
ProtoMetaWrapper(ProtoMeta.ADAPTER.decode(source.asInputStream()))
|
||||
}
|
@ -0,0 +1,39 @@
|
||||
package space.kscience.dataforge.io.proto
|
||||
|
||||
import space.kscience.dataforge.meta.Meta
|
||||
import space.kscience.dataforge.meta.get
|
||||
import kotlin.test.Test
|
||||
import kotlin.test.assertEquals
|
||||
|
||||
class ProtoBufTest {
|
||||
|
||||
@Test
|
||||
fun testProtoBufMetaFormat(){
|
||||
val meta = Meta {
|
||||
"a" put 22
|
||||
"node" put {
|
||||
"b" put "DDD"
|
||||
"c" put 11.1
|
||||
"d" put {
|
||||
"d1" put {
|
||||
"d11" put "aaa"
|
||||
"d12" put "bbb"
|
||||
}
|
||||
"d2" put 2
|
||||
}
|
||||
"array" put doubleArrayOf(1.0, 2.0, 3.0)
|
||||
}
|
||||
}
|
||||
val buffer = kotlinx.io.Buffer()
|
||||
ProtoMetaFormat.writeTo(buffer,meta)
|
||||
val result = ProtoMetaFormat.readFrom(buffer)
|
||||
|
||||
println(result["a"]?.value)
|
||||
|
||||
meta.items.keys.forEach {
|
||||
assertEquals(meta[it],result[it],"${meta[it]} != ${result[it]}")
|
||||
}
|
||||
|
||||
assertEquals(meta, result)
|
||||
}
|
||||
}
|
@ -117,8 +117,11 @@ private fun MutableMap<NameToken, SealedMeta>.addJsonElement(
|
||||
} else {
|
||||
val indexKey = descriptor?.indexKey ?: Meta.INDEX_KEY
|
||||
element.forEachIndexed { serial, childElement ->
|
||||
val index = (childElement as? JsonObject)?.get(indexKey)?.jsonPrimitive?.content
|
||||
|
||||
val index = (childElement as? JsonObject)
|
||||
?.get(indexKey)?.jsonPrimitive?.content
|
||||
?: serial.toString()
|
||||
|
||||
val child: SealedMeta = when (childElement) {
|
||||
is JsonObject -> childElement.toMeta(descriptor)
|
||||
is JsonArray -> {
|
||||
|
@ -221,13 +221,14 @@ public fun <T : Scheme> Configurable.updateWith(
|
||||
/**
|
||||
* A delegate that uses a [MetaReader] to wrap a child of this provider
|
||||
*/
|
||||
public fun <T : Scheme> MutableMeta.scheme(
|
||||
public fun <T : Scheme> MutableMetaProvider.scheme(
|
||||
spec: SchemeSpec<T>,
|
||||
key: Name? = null,
|
||||
): ReadWriteProperty<Any?, T> = object : ReadWriteProperty<Any?, T> {
|
||||
override fun getValue(thisRef: Any?, property: KProperty<*>): T {
|
||||
val name = key ?: property.name.asName()
|
||||
return spec.write(getOrCreate(name))
|
||||
val node = get(name)?: MutableMeta().also { set(name,it) }
|
||||
return spec.write(node)
|
||||
}
|
||||
|
||||
override fun setValue(thisRef: Any?, property: KProperty<*>, value: T) {
|
||||
|
@ -67,10 +67,29 @@ public class NameToken(public val body: String, public val index: String? = null
|
||||
* Parse name token from a string
|
||||
*/
|
||||
public fun parse(string: String): NameToken {
|
||||
val body = string.substringBefore('[')
|
||||
val index = string.substringAfter('[', "")
|
||||
if (index.isNotEmpty() && !index.endsWith(']')) error("NameToken with index must end with ']'")
|
||||
return NameToken(body, index.removeSuffix("]"))
|
||||
var indexStart = -1
|
||||
var indexEnd = -1
|
||||
string.forEachIndexed { index, c ->
|
||||
when (c) {
|
||||
'[' -> when {
|
||||
indexStart >= 0 -> error("Second opening bracket not allowed in NameToken: $string")
|
||||
else -> indexStart = index
|
||||
}
|
||||
|
||||
']' -> when {
|
||||
indexStart < 0 -> error("Closing index bracket could not be used before opening bracket in NameToken: $string")
|
||||
indexEnd >= 0 -> error("Second closing bracket not allowed in NameToken: $string")
|
||||
else -> indexEnd = index
|
||||
}
|
||||
|
||||
else -> if(indexEnd>=0) error("Symbols not allowed after index in NameToken: $string")
|
||||
}
|
||||
}
|
||||
if(indexStart>=0 && indexEnd<0) error("Opening bracket without closing bracket not allowed in NameToken: $string")
|
||||
return NameToken(
|
||||
if(indexStart>=0) string.substring(0, indexStart) else string,
|
||||
if(indexStart>=0) string.substring(indexStart + 1, indexEnd) else null
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -56,10 +56,22 @@ class NameTest {
|
||||
|
||||
val token2 = NameToken.parse("token-body")
|
||||
assertEquals("token-body", token2.body)
|
||||
assertEquals("", token2.index)
|
||||
assertEquals(null, token2.index)
|
||||
|
||||
// val token3 = NameToken.parse("[token-index]")
|
||||
// assertEquals("", token3.body)
|
||||
// assertEquals("token-index", token3.index)
|
||||
|
||||
assertFails{
|
||||
NameToken.parse("[token-index]")
|
||||
}
|
||||
|
||||
assertFails {
|
||||
NameToken.parse("token[22")
|
||||
}
|
||||
|
||||
assertFails {
|
||||
NameToken.parse("token[22]ddd")
|
||||
}
|
||||
}
|
||||
}
|
@ -43,6 +43,7 @@ include(
|
||||
":dataforge-meta",
|
||||
":dataforge-io",
|
||||
":dataforge-io:dataforge-io-yaml",
|
||||
":dataforge-io:dataforge-io-proto",
|
||||
":dataforge-context",
|
||||
":dataforge-data",
|
||||
":dataforge-workspace",
|
||||
|
Loading…
Reference in New Issue
Block a user