Add Envelope encoding to protobuf
This commit is contained in:
parent
e7f2f7e96f
commit
b77fc9a0d5
@ -7,6 +7,7 @@ description = "ProtoBuf meta IO"
|
||||
|
||||
kscience {
|
||||
jvm()
|
||||
// js()
|
||||
dependencies {
|
||||
api(projects.dataforgeIo)
|
||||
api("com.squareup.wire:wire-runtime:4.9.9")
|
||||
|
@ -0,0 +1,32 @@
|
||||
package pace.kscience.dataforge.io.proto
|
||||
|
||||
import kotlinx.io.Sink
|
||||
import kotlinx.io.Source
|
||||
import kotlinx.io.readByteArray
|
||||
import okio.ByteString
|
||||
import okio.ByteString.Companion.toByteString
|
||||
import space.kscience.dataforge.io.Envelope
|
||||
import space.kscience.dataforge.io.EnvelopeFormat
|
||||
import space.kscience.dataforge.io.asBinary
|
||||
import space.kscience.dataforge.io.proto.ProtoEnvelope
|
||||
import space.kscience.dataforge.io.toByteArray
|
||||
import space.kscience.dataforge.meta.Meta
|
||||
|
||||
|
||||
public object ProtoEnvelopeFormat : EnvelopeFormat {
|
||||
override fun readFrom(source: Source): Envelope {
|
||||
val protoEnvelope = ProtoEnvelope.ADAPTER.decode(source.readByteArray())
|
||||
return Envelope(
|
||||
meta = protoEnvelope.meta?.let { ProtoMetaWrapper(it) } ?: Meta.EMPTY,
|
||||
data = protoEnvelope.dataBytes.toByteArray().asBinary()
|
||||
)
|
||||
}
|
||||
|
||||
override fun writeTo(sink: Sink, obj: Envelope) {
|
||||
val protoEnvelope = ProtoEnvelope(
|
||||
obj.meta.toProto(),
|
||||
obj.data?.toByteArray()?.toByteString() ?: ByteString.EMPTY
|
||||
)
|
||||
sink.write(ProtoEnvelope.ADAPTER.encode(protoEnvelope))
|
||||
}
|
||||
}
|
@ -1,11 +1,10 @@
|
||||
package space.kscience.dataforge.io.proto
|
||||
package pace.kscience.dataforge.io.proto
|
||||
|
||||
import kotlinx.io.Sink
|
||||
import kotlinx.io.Source
|
||||
import kotlinx.io.asInputStream
|
||||
import kotlinx.io.asOutputStream
|
||||
import org.slf4j.LoggerFactory
|
||||
import kotlinx.io.readByteArray
|
||||
import space.kscience.dataforge.io.MetaFormat
|
||||
import space.kscience.dataforge.io.proto.ProtoMeta
|
||||
import space.kscience.dataforge.meta.*
|
||||
import space.kscience.dataforge.meta.descriptors.MetaDescriptor
|
||||
import space.kscience.dataforge.names.NameToken
|
||||
@ -50,8 +49,8 @@ internal fun Meta.toProto(): ProtoMeta {
|
||||
is Long -> ProtoMeta.ProtoValue(int64Value = long)
|
||||
is Float -> ProtoMeta.ProtoValue(floatValue = float)
|
||||
else -> {
|
||||
LoggerFactory.getLogger(ProtoMeta::class.java)
|
||||
.warn("Unknown number type ${value} encoded as Double")
|
||||
// LoggerFactory.getLogger(ProtoMeta::class.java)
|
||||
// .warn("Unknown number type ${value} encoded as Double")
|
||||
ProtoMeta.ProtoValue(doubleValue = double)
|
||||
}
|
||||
}
|
||||
@ -67,12 +66,11 @@ internal fun Meta.toProto(): ProtoMeta {
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
public object ProtoMetaFormat : MetaFormat {
|
||||
override fun writeMeta(sink: Sink, meta: Meta, descriptor: MetaDescriptor?) {
|
||||
ProtoMeta.ADAPTER.encode(sink.asOutputStream(), meta.toProto())
|
||||
sink.write(ProtoMeta.ADAPTER.encode(meta.toProto()))
|
||||
}
|
||||
|
||||
override fun readMeta(source: Source, descriptor: MetaDescriptor?): Meta =
|
||||
ProtoMetaWrapper(ProtoMeta.ADAPTER.decode(source.asInputStream()))
|
||||
ProtoMetaWrapper(ProtoMeta.ADAPTER.decode(source.readByteArray()))
|
||||
}
|
@ -27,4 +27,9 @@ message ProtoMeta {
|
||||
ProtoValue protoValue = 1;
|
||||
|
||||
map<string, ProtoMeta> items = 2;
|
||||
}
|
||||
|
||||
message ProtoEnvelope{
|
||||
ProtoMeta meta = 1;
|
||||
bytes dataBytes = 2;
|
||||
}
|
@ -0,0 +1,83 @@
|
||||
package pace.kscience.dataforge.io.proto
|
||||
|
||||
import kotlinx.io.writeString
|
||||
import space.kscience.dataforge.io.Envelope
|
||||
import space.kscience.dataforge.io.toByteArray
|
||||
import space.kscience.dataforge.meta.Meta
|
||||
import space.kscience.dataforge.meta.asValue
|
||||
import space.kscience.dataforge.meta.get
|
||||
import kotlin.test.Test
|
||||
import kotlin.test.assertContentEquals
|
||||
import kotlin.test.assertEquals
|
||||
|
||||
class ProtoBufTest {
|
||||
|
||||
@Test
|
||||
fun testProtoBufMetaFormat(){
|
||||
val meta = Meta {
|
||||
"a" put 22
|
||||
"node" put {
|
||||
"b" put "DDD"
|
||||
"c" put 11.1
|
||||
"d" put {
|
||||
"d1" put {
|
||||
"d11" put "aaa"
|
||||
"d12" put "bbb"
|
||||
}
|
||||
"d2" put 2
|
||||
}
|
||||
"array" put doubleArrayOf(1.0, 2.0, 3.0)
|
||||
"array2d" put listOf(
|
||||
doubleArrayOf(1.0, 2.0, 3.0).asValue(),
|
||||
doubleArrayOf(1.0, 2.0, 3.0).asValue()
|
||||
).asValue()
|
||||
}
|
||||
}
|
||||
val buffer = kotlinx.io.Buffer()
|
||||
ProtoMetaFormat.writeTo(buffer,meta)
|
||||
val result = ProtoMetaFormat.readFrom(buffer)
|
||||
|
||||
// println(result["a"]?.value)
|
||||
|
||||
meta.items.keys.forEach {
|
||||
assertEquals(meta[it],result[it],"${meta[it]} != ${result[it]}")
|
||||
}
|
||||
|
||||
assertEquals(meta, result)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun testProtoBufEnvelopeFormat(){
|
||||
val envelope = Envelope{
|
||||
meta {
|
||||
"a" put 22
|
||||
"node" put {
|
||||
"b" put "DDD"
|
||||
"c" put 11.1
|
||||
"d" put {
|
||||
"d1" put {
|
||||
"d11" put "aaa"
|
||||
"d12" put "bbb"
|
||||
}
|
||||
"d2" put 2
|
||||
}
|
||||
"array" put doubleArrayOf(1.0, 2.0, 3.0)
|
||||
"array2d" put listOf(
|
||||
doubleArrayOf(1.0, 2.0, 3.0).asValue(),
|
||||
doubleArrayOf(1.0, 2.0, 3.0).asValue()
|
||||
).asValue()
|
||||
}
|
||||
}
|
||||
data {
|
||||
writeString("Hello world!")
|
||||
}
|
||||
}
|
||||
|
||||
val buffer = kotlinx.io.Buffer()
|
||||
ProtoEnvelopeFormat.writeTo(buffer,envelope)
|
||||
val result = ProtoEnvelopeFormat.readFrom(buffer)
|
||||
|
||||
assertEquals(envelope.meta, result.meta)
|
||||
assertContentEquals(envelope.data?.toByteArray(), result.data?.toByteArray())
|
||||
}
|
||||
}
|
@ -1,44 +0,0 @@
|
||||
package space.kscience.dataforge.io.proto
|
||||
|
||||
import space.kscience.dataforge.meta.Meta
|
||||
import space.kscience.dataforge.meta.asValue
|
||||
import space.kscience.dataforge.meta.get
|
||||
import kotlin.test.Test
|
||||
import kotlin.test.assertEquals
|
||||
|
||||
class ProtoBufTest {
|
||||
|
||||
@Test
|
||||
fun testProtoBufMetaFormat(){
|
||||
val meta = Meta {
|
||||
"a" put 22
|
||||
"node" put {
|
||||
"b" put "DDD"
|
||||
"c" put 11.1
|
||||
"d" put {
|
||||
"d1" put {
|
||||
"d11" put "aaa"
|
||||
"d12" put "bbb"
|
||||
}
|
||||
"d2" put 2
|
||||
}
|
||||
"array" put doubleArrayOf(1.0, 2.0, 3.0)
|
||||
"array2d" put listOf(
|
||||
doubleArrayOf(1.0, 2.0, 3.0).asValue(),
|
||||
doubleArrayOf(1.0, 2.0, 3.0).asValue()
|
||||
).asValue()
|
||||
}
|
||||
}
|
||||
val buffer = kotlinx.io.Buffer()
|
||||
ProtoMetaFormat.writeTo(buffer,meta)
|
||||
val result = ProtoMetaFormat.readFrom(buffer)
|
||||
|
||||
// println(result["a"]?.value)
|
||||
|
||||
meta.items.keys.forEach {
|
||||
assertEquals(meta[it],result[it],"${meta[it]} != ${result[it]}")
|
||||
}
|
||||
|
||||
assertEquals(meta, result)
|
||||
}
|
||||
}
|
@ -2,7 +2,6 @@ package space.kscience.dataforge.io
|
||||
|
||||
import space.kscience.dataforge.meta.Laminate
|
||||
import space.kscience.dataforge.meta.Meta
|
||||
import space.kscience.dataforge.meta.get
|
||||
import space.kscience.dataforge.meta.string
|
||||
import space.kscience.dataforge.names.Name
|
||||
import space.kscience.dataforge.names.asName
|
||||
@ -34,7 +33,9 @@ public interface Envelope {
|
||||
}
|
||||
}
|
||||
|
||||
internal class SimpleEnvelope(override val meta: Meta, override val data: Binary?) : Envelope
|
||||
internal class SimpleEnvelope(override val meta: Meta, override val data: Binary?) : Envelope{
|
||||
override fun toString(): String = "Envelope(meta=$meta, data=$data)"
|
||||
}
|
||||
|
||||
public fun Envelope(meta: Meta, data: Binary?): Envelope = SimpleEnvelope(meta, data)
|
||||
|
||||
|
Loading…
Reference in New Issue
Block a user