kscience/dataforge-core
8
0
Fork 0
Code Issues Pull Requests 1 Packages Projects Releases 12 Wiki Activity

Protobuf meta representation

Browse Source
This commit is contained in:
Alexander Nozik 2024-06-11 12:06:48 +03:00
parent 7fa6617e7e
commit e7f2f7e96f
3 changed files with 38 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
dataforge-io/dataforge-io-proto/src/commonMain/proto/meta.proto
View File

@ -11,9 +11,20 @@ message ProtoMeta {
int32 int32Value = 6; int32 int32Value = 6;
int64 int64Value = 7; int64 int64Value = 7;
bytes bytesValue = 8; bytes bytesValue = 8;
ProtoValueList listValue = 9;
Float64List float64ListValue = 10;
} }
} }
repeated ProtoValue value = 1;
message ProtoValueList{
repeated ProtoValue values = 1;
}
message Float64List{
repeated double values = 1 [packed=true];
}
ProtoValue protoValue = 1;
map<string, ProtoMeta> items = 2; map<string, ProtoMeta> items = 2;
} }

48
dataforge-io/dataforge-io-proto/src/jvmMain/kotlin/ProtoMetaFormat.kt
View File

@ -12,7 +12,7 @@ import space.kscience.dataforge.names.NameToken
internal class ProtoMetaWrapper(private val proto: ProtoMeta) : Meta { internal class ProtoMetaWrapper(private val proto: ProtoMeta) : Meta {
private fun ProtoMeta.ProtoValue.toValue(): Value = when { private fun ProtoMeta.ProtoValue.toValue(): Value? = when {
stringValue != null -> stringValue.asValue() stringValue != null -> stringValue.asValue()
booleanValue != null -> booleanValue.asValue() booleanValue != null -> booleanValue.asValue()
doubleValue != null -> doubleValue.asValue() doubleValue != null -> doubleValue.asValue()
@ -20,15 +20,13 @@ internal class ProtoMetaWrapper(private val proto: ProtoMeta) : Meta {
int32Value != null -> int32Value.asValue() int32Value != null -> int32Value.asValue()
int64Value != null -> int64Value.asValue() int64Value != null -> int64Value.asValue()
bytesValue != null -> bytesValue.toByteArray().asValue() bytesValue != null -> bytesValue.toByteArray().asValue()
else -> Null listValue != null -> listValue.values.mapNotNull { it.toValue() }.asValue()
float64ListValue != null -> float64ListValue.values.map { it.asValue() }.asValue()
else -> null
} }
override val value: Value? override val value: Value?
get() = when (proto.value_.size) { get() = proto.protoValue?.toValue()
0 -> null
1 -> proto.value_[0].toValue()
else -> proto.value_.map { it.toValue() }.asValue()
}
override val items: Map<NameToken, Meta> override val items: Map<NameToken, Meta>
@ -44,39 +42,27 @@ internal class ProtoMetaWrapper(private val proto: ProtoMeta) : Meta {
internal fun Meta.toProto(): ProtoMeta { internal fun Meta.toProto(): ProtoMeta {
fun MutableList<ProtoMeta.ProtoValue>.appendProtoValues(value: Value): Unit { fun Value.toProto(): ProtoMeta.ProtoValue = when (type) {
when (value.type) { ValueType.NULL -> ProtoMeta.ProtoValue()
ValueType.NULL -> {
//do nothing
}
ValueType.NUMBER -> when (value.value) { ValueType.NUMBER -> when (value) {
is Int, is Short, is Byte -> add(ProtoMeta.ProtoValue(int32Value = value.int)) is Int, is Short, is Byte -> ProtoMeta.ProtoValue(int32Value = int)
is Long -> add(ProtoMeta.ProtoValue(int64Value = value.long)) is Long -> ProtoMeta.ProtoValue(int64Value = long)
is Float -> add(ProtoMeta.ProtoValue(floatValue = value.float)) is Float -> ProtoMeta.ProtoValue(floatValue = float)
else -> { else -> {
LoggerFactory.getLogger(ProtoMeta::class.java) LoggerFactory.getLogger(ProtoMeta::class.java)
.warn("Unknown number type ${value.value} encoded as Double") .warn("Unknown number type ${value} encoded as Double")
add(ProtoMeta.ProtoValue(doubleValue = value.double)) ProtoMeta.ProtoValue(doubleValue = double)
} }
} }
ValueType.STRING -> add(ProtoMeta.ProtoValue(stringValue = value.string)) ValueType.STRING -> ProtoMeta.ProtoValue(stringValue = string)
ValueType.BOOLEAN -> add(ProtoMeta.ProtoValue(booleanValue = value.boolean)) ValueType.BOOLEAN -> ProtoMeta.ProtoValue(booleanValue = boolean)
ValueType.LIST -> { ValueType.LIST -> ProtoMeta.ProtoValue(listValue = ProtoMeta.ProtoValueList(list.map { it.toProto() }))
value.list.forEach {
if (it.type == ValueType.LIST) {
error("Nested lists are not supported")
} else {
appendProtoValues(it)
}
}
}
}
} }
return ProtoMeta( return ProtoMeta(
value_ = buildList { value?.let { appendProtoValues(it) } }, protoValue = value?.toProto(),
items.entries.associate { it.key.toString() to it.value.toProto() } items.entries.associate { it.key.toString() to it.value.toProto() }
) )
} }

7
dataforge-io/dataforge-io-proto/src/jvmTest/kotlin/ProtoBufTest.kt
View File

@ -1,6 +1,7 @@
package space.kscience.dataforge.io.proto package space.kscience.dataforge.io.proto
import space.kscience.dataforge.meta.Meta import space.kscience.dataforge.meta.Meta
import space.kscience.dataforge.meta.asValue
import space.kscience.dataforge.meta.get import space.kscience.dataforge.meta.get
import kotlin.test.Test import kotlin.test.Test
import kotlin.test.assertEquals import kotlin.test.assertEquals
@ -22,13 +23,17 @@ class ProtoBufTest {
"d2" put 2 "d2" put 2
} }
"array" put doubleArrayOf(1.0, 2.0, 3.0) "array" put doubleArrayOf(1.0, 2.0, 3.0)
"array2d" put listOf(
doubleArrayOf(1.0, 2.0, 3.0).asValue(),
doubleArrayOf(1.0, 2.0, 3.0).asValue()
).asValue()
} }
} }
val buffer = kotlinx.io.Buffer() val buffer = kotlinx.io.Buffer()
ProtoMetaFormat.writeTo(buffer,meta) ProtoMetaFormat.writeTo(buffer,meta)
val result = ProtoMetaFormat.readFrom(buffer) val result = ProtoMetaFormat.readFrom(buffer)
println(result["a"]?.value) // println(result["a"]?.value)
meta.items.keys.forEach { meta.items.keys.forEach {
assertEquals(meta[it],result[it],"${meta[it]} != ${result[it]}") assertEquals(meta[it],result[it],"${meta[it]} != ${result[it]}")