kscience/dataforge-core
8
0
Fork 0
Code Issues Pull Requests 1 Packages Projects Releases 12 Wiki Activity

Add Envelope encoding to protobuf

Browse Source
This commit is contained in:
Alexander Nozik 2024-06-11 16:24:15 +03:00
parent e7f2f7e96f
commit b77fc9a0d5
7 changed files with 131 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
dataforge-io/dataforge-io-proto/build.gradle.kts
View File

@ -7,6 +7,7 @@ description = "ProtoBuf meta IO"
kscience {
jvm()
// js()
dependencies {
api(projects.dataforgeIo)
api("com.squareup.wire:wire-runtime:4.9.9")

32
dataforge-io/dataforge-io-proto/src/commonMain/kotlin/ProtoEnvelopeFormat.kt Normal file
View File

@ -0,0 +1,32 @@
package pace.kscience.dataforge.io.proto
import kotlinx.io.Sink
import kotlinx.io.Source
import kotlinx.io.readByteArray
import okio.ByteString
import okio.ByteString.Companion.toByteString
import space.kscience.dataforge.io.Envelope
import space.kscience.dataforge.io.EnvelopeFormat
import space.kscience.dataforge.io.asBinary
import space.kscience.dataforge.io.proto.ProtoEnvelope
import space.kscience.dataforge.io.toByteArray
import space.kscience.dataforge.meta.Meta
public object ProtoEnvelopeFormat : EnvelopeFormat {
override fun readFrom(source: Source): Envelope {
val protoEnvelope = ProtoEnvelope.ADAPTER.decode(source.readByteArray())
return Envelope(
meta = protoEnvelope.meta?.let { ProtoMetaWrapper(it) } ?: Meta.EMPTY,
data = protoEnvelope.dataBytes.toByteArray().asBinary()
)
}
override fun writeTo(sink: Sink, obj: Envelope) {
val protoEnvelope = ProtoEnvelope(
obj.meta.toProto(),
obj.data?.toByteArray()?.toByteString() ?: ByteString.EMPTY
)
sink.write(ProtoEnvelope.ADAPTER.encode(protoEnvelope))
}
}

16
dataforge-io/dataforge-io-proto/src/jvmMain/kotlin/ProtoMetaFormat.kt → dataforge-io/dataforge-io-proto/src/commonMain/kotlin/ProtoMetaFormat.kt
View File

@ -1,11 +1,10 @@
package space.kscience.dataforge.io.proto
package pace.kscience.dataforge.io.proto
import kotlinx.io.Sink
import kotlinx.io.Source
import kotlinx.io.asInputStream
import kotlinx.io.asOutputStream
import org.slf4j.LoggerFactory
import kotlinx.io.readByteArray
import space.kscience.dataforge.io.MetaFormat
import space.kscience.dataforge.io.proto.ProtoMeta
import space.kscience.dataforge.meta.*
import space.kscience.dataforge.meta.descriptors.MetaDescriptor
import space.kscience.dataforge.names.NameToken
@ -50,8 +49,8 @@ internal fun Meta.toProto(): ProtoMeta {
is Long -> ProtoMeta.ProtoValue(int64Value = long)
is Float -> ProtoMeta.ProtoValue(floatValue = float)
else -> {
LoggerFactory.getLogger(ProtoMeta::class.java)
.warn("Unknown number type ${value} encoded as Double")
// LoggerFactory.getLogger(ProtoMeta::class.java)
// .warn("Unknown number type ${value} encoded as Double")
ProtoMeta.ProtoValue(doubleValue = double)
}
}
@ -67,12 +66,11 @@ internal fun Meta.toProto(): ProtoMeta {
)
}
public object ProtoMetaFormat : MetaFormat {
override fun writeMeta(sink: Sink, meta: Meta, descriptor: MetaDescriptor?) {
ProtoMeta.ADAPTER.encode(sink.asOutputStream(), meta.toProto())
sink.write(ProtoMeta.ADAPTER.encode(meta.toProto()))
}
override fun readMeta(source: Source, descriptor: MetaDescriptor?): Meta =
ProtoMetaWrapper(ProtoMeta.ADAPTER.decode(source.asInputStream()))
ProtoMetaWrapper(ProtoMeta.ADAPTER.decode(source.readByteArray()))
}

5
dataforge-io/dataforge-io-proto/src/commonMain/proto/meta.proto
View File

@ -28,3 +28,8 @@ message ProtoMeta {
map<string, ProtoMeta> items = 2;
}
message ProtoEnvelope{
ProtoMeta meta = 1;
bytes dataBytes = 2;
}

83
dataforge-io/dataforge-io-proto/src/commonTest/kotlin/ProtoBufTest.kt Normal file
View File

@ -0,0 +1,83 @@
package pace.kscience.dataforge.io.proto
import kotlinx.io.writeString
import space.kscience.dataforge.io.Envelope
import space.kscience.dataforge.io.toByteArray
import space.kscience.dataforge.meta.Meta
import space.kscience.dataforge.meta.asValue
import space.kscience.dataforge.meta.get
import kotlin.test.Test
import kotlin.test.assertContentEquals
import kotlin.test.assertEquals
class ProtoBufTest {
@Test
fun testProtoBufMetaFormat(){
val meta = Meta {
"a" put 22
"node" put {
"b" put "DDD"
"c" put 11.1
"d" put {
"d1" put {
"d11" put "aaa"
"d12" put "bbb"
}
"d2" put 2
}
"array" put doubleArrayOf(1.0, 2.0, 3.0)
"array2d" put listOf(
doubleArrayOf(1.0, 2.0, 3.0).asValue(),
doubleArrayOf(1.0, 2.0, 3.0).asValue()
).asValue()
}
}
val buffer = kotlinx.io.Buffer()
ProtoMetaFormat.writeTo(buffer,meta)
val result = ProtoMetaFormat.readFrom(buffer)
// println(result["a"]?.value)
meta.items.keys.forEach {
assertEquals(meta[it],result[it],"${meta[it]} != ${result[it]}")
}
assertEquals(meta, result)
}
@Test
fun testProtoBufEnvelopeFormat(){
val envelope = Envelope{
meta {
"a" put 22
"node" put {
"b" put "DDD"
"c" put 11.1
"d" put {
"d1" put {
"d11" put "aaa"
"d12" put "bbb"
}
"d2" put 2
}
"array" put doubleArrayOf(1.0, 2.0, 3.0)
"array2d" put listOf(
doubleArrayOf(1.0, 2.0, 3.0).asValue(),
doubleArrayOf(1.0, 2.0, 3.0).asValue()
).asValue()
}
}
data {
writeString("Hello world!")
}
}
val buffer = kotlinx.io.Buffer()
ProtoEnvelopeFormat.writeTo(buffer,envelope)
val result = ProtoEnvelopeFormat.readFrom(buffer)
assertEquals(envelope.meta, result.meta)
assertContentEquals(envelope.data?.toByteArray(), result.data?.toByteArray())
}
}

44
dataforge-io/dataforge-io-proto/src/jvmTest/kotlin/ProtoBufTest.kt
View File

@ -1,44 +0,0 @@
package space.kscience.dataforge.io.proto
import space.kscience.dataforge.meta.Meta
import space.kscience.dataforge.meta.asValue
import space.kscience.dataforge.meta.get
import kotlin.test.Test
import kotlin.test.assertEquals
class ProtoBufTest {
@Test
fun testProtoBufMetaFormat(){
val meta = Meta {
"a" put 22
"node" put {
"b" put "DDD"
"c" put 11.1
"d" put {
"d1" put {
"d11" put "aaa"
"d12" put "bbb"
}
"d2" put 2
}
"array" put doubleArrayOf(1.0, 2.0, 3.0)
"array2d" put listOf(
doubleArrayOf(1.0, 2.0, 3.0).asValue(),
doubleArrayOf(1.0, 2.0, 3.0).asValue()
).asValue()
}
}
val buffer = kotlinx.io.Buffer()
ProtoMetaFormat.writeTo(buffer,meta)
val result = ProtoMetaFormat.readFrom(buffer)
// println(result["a"]?.value)
meta.items.keys.forEach {
assertEquals(meta[it],result[it],"${meta[it]} != ${result[it]}")
}
assertEquals(meta, result)
}
}

5
dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/Envelope.kt
View File

@ -2,7 +2,6 @@ package space.kscience.dataforge.io
import space.kscience.dataforge.meta.Laminate
import space.kscience.dataforge.meta.Meta
import space.kscience.dataforge.meta.get
import space.kscience.dataforge.meta.string
import space.kscience.dataforge.names.Name
import space.kscience.dataforge.names.asName
@ -34,7 +33,9 @@ public interface Envelope {
}
}
internal class SimpleEnvelope(override val meta: Meta, override val data: Binary?) : Envelope
internal class SimpleEnvelope(override val meta: Meta, override val data: Binary?) : Envelope{
override fun toString(): String = "Envelope(meta=$meta, data=$data)"
}
public fun Envelope(meta: Meta, data: Binary?): Envelope = SimpleEnvelope(meta, data)