From 7c38cadddd53e0c64498e5149892949574049820 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Thu, 23 May 2019 22:16:34 +0300 Subject: [PATCH] Envelope IO --- .../kotlin/hep/dataforge/io/Binary.kt | 60 ++++++++++++++ .../hep/dataforge/io/BinaryMetaFormat.kt | 20 ++--- .../kotlin/hep/dataforge/io/Envelope.kt | 9 +-- .../kotlin/hep/dataforge/io/IOFormat.kt | 16 ++-- .../kotlin/hep/dataforge/io/JsonMetaFormat.kt | 18 ++--- .../kotlin/hep/dataforge/io/MetaFormat.kt | 19 ++--- .../hep/dataforge/io/TaggedEnvelopeFormat.kt | 80 +++++++++++++++++++ .../kotlin/hep/dataforge/io/FileBinary.kt | 16 ++++ .../kotlin/hep/dataforge/meta/MutableMeta.kt | 5 +- 9 files changed, 191 insertions(+), 52 deletions(-) create mode 100644 dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Binary.kt create mode 100644 dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt create mode 100644 dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileBinary.kt diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Binary.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Binary.kt new file mode 100644 index 00000000..a0374968 --- /dev/null +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Binary.kt @@ -0,0 +1,60 @@ +package hep.dataforge.io + +import kotlinx.io.core.ByteReadPacket +import kotlinx.io.core.Input +import kotlinx.io.core.readBytes + +/** + * A source of binary data + */ +interface Binary { + /** + * The size of binary in bytes + */ + val size: ULong + + /** + * Read continuous [Input] from this binary stating from the beginning. + * The input is automatically closed on scope close. + * Some implementation may forbid this to be called twice. In this case second call will throw an exception. + */ + fun read(block: Input.() -> R): R +} + +/** + * A [Binary] with addition random access functionality. It by default allows multiple [read] operations. + */ +interface RandomAccessBinary : Binary { + /** + * Read at most [size] of bytes starting at [from] offset from the beginning of the binary. + * This method could be called multiple times simultaneously. + */ + fun read(from: UInt, size: UInt = UInt.MAX_VALUE, block: Input.() -> R): R + + override fun read(block: Input.() -> R): R = read(0.toUInt(), UInt.MAX_VALUE, block) +} + +fun Binary.readAll(): ByteReadPacket = read { + ByteReadPacket(this.readBytes()) +} + +fun RandomAccessBinary.readPacket(from: UInt, size: UInt): ByteReadPacket = read(from, size) { + ByteReadPacket(this.readBytes()) +} + +object EmptyBinary : RandomAccessBinary { + + override val size: ULong = 0.toULong() + + override fun read(from: UInt, size: UInt, block: Input.() -> R): R { + error("The binary is empty") + } +} + +class ArrayBinary(val array: ByteArray) : RandomAccessBinary { + override val size: ULong = array.size.toULong() + + override fun read(from: UInt, size: UInt, block: Input.() -> R): R { + return ByteReadPacket(array, from.toInt(), size.toInt()).block() + } +} \ No newline at end of file diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/BinaryMetaFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/BinaryMetaFormat.kt index 0e73934a..79145796 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/BinaryMetaFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/BinaryMetaFormat.kt @@ -8,12 +8,11 @@ import kotlinx.io.core.readText import kotlinx.io.core.writeText object BinaryMetaFormat : MetaFormat { - override fun write(obj: Meta, out: Output) { - out.writeMeta(obj) - } + override val name: String = "bin" + override val key: Short = 0x4249//BI - override fun read(input: Input): Meta { - return (input.readMetaItem() as MetaItem.NodeItem).node + override fun Input.readObject(): Meta { + return (readMetaItem() as MetaItem.NodeItem).node } private fun Output.writeChar(char: Char) = writeByte(char.toByte()) @@ -70,7 +69,7 @@ object BinaryMetaFormat : MetaFormat { } } - private fun Output.writeMeta(meta: Meta) { + override fun Output.writeObject(meta: Meta) { writeChar('M') writeInt(meta.items.size) meta.items.forEach { (key, item) -> @@ -80,7 +79,7 @@ object BinaryMetaFormat : MetaFormat { writeValue(item.value) } is MetaItem.NodeItem -> { - writeMeta(item.node) + writeObject(item.node) } } } @@ -122,11 +121,4 @@ object BinaryMetaFormat : MetaFormat { else -> error("Unknown serialization key character: $keyChar") } } -} - -class BinaryMetaFormatFactory : MetaFormatFactory { - override val name: String = "bin" - override val key: Short = 0x4249//BI - - override fun build(): MetaFormat = BinaryMetaFormat } \ No newline at end of file diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Envelope.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Envelope.kt index 1a9e58d7..fc3ae90d 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Envelope.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Envelope.kt @@ -7,7 +7,7 @@ import kotlinx.io.core.Input interface Envelope { val meta: Meta - val data: Input? + val data: Binary? companion object { @@ -23,11 +23,7 @@ interface Envelope { } } -class SimpleEnvelope(override val meta: Meta, val dataProvider: () -> Input?) : Envelope{ - override val data: Input? - get() = dataProvider() - -} +class SimpleEnvelope(override val meta: Meta, override val data: Binary?) : Envelope /** * The purpose of the envelope @@ -50,3 +46,4 @@ val Envelope.dataType: String? get() = meta[Envelope.ENVELOPE_DATA_TYPE_KEY].str */ val Envelope.description: String? get() = meta[Envelope.ENVELOPE_DESCRIPTION_KEY].string +typealias EnvelopeFormat = IOFormat \ No newline at end of file diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOFormat.kt index ce58c05d..9055b249 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOFormat.kt @@ -1,10 +1,14 @@ package hep.dataforge.io -import kotlinx.io.core.Input -import kotlinx.io.core.Output - +import kotlinx.io.core.* +/** + * And interface for serialization facilities + */ interface IOFormat { - fun write(obj: T, out: Output) - fun read(input: Input): T -} \ No newline at end of file + fun Output.writeObject(obj: T) + fun Input.readObject(): T +} + +fun IOFormat.writePacket(obj: T): ByteReadPacket = buildPacket { writeObject(obj) } +fun IOFormat.writeBytes(obj: T): ByteArray = buildPacket { writeObject(obj) }.readBytes() \ No newline at end of file diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/JsonMetaFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/JsonMetaFormat.kt index 9d7b5739..13cbc717 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/JsonMetaFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/JsonMetaFormat.kt @@ -14,13 +14,16 @@ import kotlinx.serialization.json.* object JsonMetaFormat : MetaFormat { - override fun write(obj: Meta, out: Output) { + override val name: String = "json" + override val key: Short = 0x4a53//"JS" + + override fun Output.writeObject(obj: Meta) { val str = obj.toJson().toString() - out.writeText(str) + writeText(str) } - override fun read(input: Input): Meta { - val str = input.readText() + override fun Input.readObject(): Meta { + val str = readText() val json = Json.plain.parseJson(str) if (json is JsonObject) { @@ -97,11 +100,4 @@ class JsonMeta(val json: JsonObject) : Meta { json.forEach { (key, value) -> map[key] = value } map.mapKeys { it.key.toName().first()!! } } -} - -class JsonMetaFormatFactory : MetaFormatFactory { - override val name: String = "json" - override val key: Short = 0x4a53//"JS" - - override fun build() = JsonMetaFormat } \ No newline at end of file diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/MetaFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/MetaFormat.kt index 61969536..52f01525 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/MetaFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/MetaFormat.kt @@ -1,33 +1,26 @@ package hep.dataforge.io import hep.dataforge.meta.Meta -import kotlinx.io.core.BytePacketBuilder import kotlinx.io.core.ByteReadPacket +import kotlinx.io.core.buildPacket import kotlinx.io.core.toByteArray /** * A format for meta serialization */ -interface MetaFormat: IOFormat - -/** - * ServiceLoader compatible factory - */ -interface MetaFormatFactory { +interface MetaFormat : IOFormat { val name: String val key: Short - - fun build(): MetaFormat } fun Meta.asString(format: MetaFormat = JsonMetaFormat): String { - val builder = BytePacketBuilder() - format.write(this, builder) - return builder.build().readText() + return buildPacket { + format.run { writeObject(this@asString) } + }.readText() } fun MetaFormat.parse(str: String): Meta { - return read(ByteReadPacket(str.toByteArray())) + return ByteReadPacket(str.toByteArray()).readObject() } diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt new file mode 100644 index 00000000..40363eed --- /dev/null +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt @@ -0,0 +1,80 @@ +package hep.dataforge.io + +import kotlinx.io.core.* + +class TaggedEnvelopeFormat( + val metaFormats: Collection, + val outputMetaFormat: MetaFormat = metaFormats.first() +) : EnvelopeFormat { + + override fun Output.writeObject(obj: Envelope) { + write(obj, this, outputMetaFormat) + } + + /** + * Read an envelope from input into memory + * + * @param input an input to read from + * @param metaFormats a collection of meta formats to resolve + */ + override fun Input.readObject(): Envelope = read(this, metaFormats) + + + private data class Tag( + val metaFormatKey: Short, + val metaSize: UInt, + val dataSize: ULong + ) + + companion object { + private const val VERSION = "DF03" + private const val START_SEQUENCE = "#~" + private const val END_SEQUENCE = "~#\r\n" + + private fun Tag.toBytes(): ByteReadPacket = buildPacket(24) { + writeText(START_SEQUENCE) + writeText(VERSION) + writeShort(metaFormatKey) + writeUInt(metaSize) + writeULong(dataSize) + writeText(END_SEQUENCE) + } + + private fun Input.readTag(): Tag { + val start = readTextExactBytes(2) + if (start != START_SEQUENCE) error("The input is not an envelope") + val version = readTextExactBytes(4) + if (version != VERSION) error("Wrong version of DataForge: expected $VERSION but found $version") + val metaFormatKey = readShort() + val metaLength = readUInt() + val dataLength = readULong() + return Tag(metaFormatKey, metaLength, dataLength) + } + + fun read(input: Input, metaFormats: Collection): Envelope { + val tag = input.readTag() + + val metaFormat = metaFormats.find { it.key == tag.metaFormatKey } + ?: error("Meta format with key ${tag.metaFormatKey} not found") + + val metaPacket = ByteReadPacket(input.readBytes(tag.metaSize.toInt())) + val meta = metaFormat.run { metaPacket.readObject() } + + val dataBytes = input.readBytes(tag.dataSize.toInt()) + + return SimpleEnvelope(meta, ArrayBinary(dataBytes)) + } + + fun write(obj: Envelope, out: Output, metaFormat: MetaFormat) { + val metaBytes = metaFormat.writeBytes(obj.meta) + val tag = Tag(metaFormat.key, metaBytes.size.toUInt(), obj.data?.size ?: 0.toULong()) + out.writePacket(tag.toBytes()) + out.writeFully(metaBytes) + obj.data?.read { + while (!endOfInput){ + out.writeByte(readByte()) + } + } + } + } +} \ No newline at end of file diff --git a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileBinary.kt b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileBinary.kt new file mode 100644 index 00000000..3154a2cf --- /dev/null +++ b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileBinary.kt @@ -0,0 +1,16 @@ +package hep.dataforge.io + +import kotlinx.io.core.ByteReadPacket +import kotlinx.io.core.Input +import java.nio.channels.FileChannel +import java.nio.file.Path +import java.nio.file.StandardOpenOption + +class FileBinary(val path: Path, private val offset: Int = 0) : RandomAccessBinary { + override fun read(from: Long, size: Long, block: Input.() -> R): R { + FileChannel.open(path, StandardOpenOption.READ).use { + val buffer = it.map(FileChannel.MapMode.READ_ONLY, from + offset, size) + return ByteReadPacket(buffer).block() + } + } +} \ No newline at end of file diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MutableMeta.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MutableMeta.kt index 2b3e9c7a..000bff7a 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MutableMeta.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MutableMeta.kt @@ -95,12 +95,13 @@ abstract class MutableMetaNode> : AbstractMetaNode(), fun > MutableMeta.remove(name: Name) = set(name, null) fun > MutableMeta.remove(name: String) = remove(name.toName()) -fun > MutableMeta.setValue(name: Name, value: Value) = set(name, MetaItem.ValueItem(value)) -//fun > MutableMeta.setItem(name: String, item: MetaItem) = set(name.toName(), item) +fun > MutableMeta.setValue(name: Name, value: Value) = + set(name, MetaItem.ValueItem(value)) fun > MutableMeta.setValue(name: String, value: Value) = set(name.toName(), MetaItem.ValueItem(value)) //fun > MutableMeta.setItem(token: NameToken, item: MetaItem?) = set(token.asName(), item) +//fun > MutableMeta.setItem(name: String, item: MetaItem) = set(name.toName(), item) fun > MutableMetaNode.setItem(name: Name, item: MetaItem<*>) { when (item) {