Envelope IO

dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Binary.kt

@@ -0,0 +1,60 @@
+package hep.dataforge.io
+
+import kotlinx.io.core.ByteReadPacket
+import kotlinx.io.core.Input
+import kotlinx.io.core.readBytes
+
+/**
+ * A source of binary data
+ */
+interface Binary {
+    /**
+     * The size of binary in bytes
+     */
+    val size: ULong
+
+    /**
+     * Read continuous [Input] from this binary stating from the beginning.
+     * The input is automatically closed on scope close.
+     * Some implementation may forbid this to be called twice. In this case second call will throw an exception.
+     */
+    fun <R> read(block: Input.() -> R): R
+}
+
+/**
+ * A [Binary] with addition random access functionality. It by default allows multiple [read] operations.
+ */
+interface RandomAccessBinary : Binary {
+    /**
+     * Read at most [size] of bytes starting at [from] offset from the beginning of the binary.
+     * This method could be called multiple times simultaneously.
+     */
+    fun <R> read(from: UInt, size: UInt = UInt.MAX_VALUE, block: Input.() -> R): R
+
+    override fun <R> read(block: Input.() -> R): R = read(0.toUInt(), UInt.MAX_VALUE, block)
+}
+
+fun Binary.readAll(): ByteReadPacket = read {
+    ByteReadPacket(this.readBytes())
+}
+
+fun RandomAccessBinary.readPacket(from: UInt, size: UInt): ByteReadPacket = read(from, size) {
+    ByteReadPacket(this.readBytes())
+}
+
+object EmptyBinary : RandomAccessBinary {
+
+    override val size: ULong = 0.toULong()
+
+    override fun <R> read(from: UInt, size: UInt, block: Input.() -> R): R {
+        error("The binary is empty")
+    }
+}
+
+class ArrayBinary(val array: ByteArray) : RandomAccessBinary {
+    override val size: ULong = array.size.toULong()
+
+    override fun <R> read(from: UInt, size: UInt, block: Input.() -> R): R {
+        return ByteReadPacket(array, from.toInt(), size.toInt()).block()
+    }
+}

dataforge-io/src/commonMain/kotlin/hep/dataforge/io/BinaryMetaFormat.kt

@@ -8,12 +8,11 @@ import kotlinx.io.core.readText
 import kotlinx.io.core.writeText
 
 object BinaryMetaFormat : MetaFormat {
-    override fun write(obj: Meta, out: Output) {
+    override val name: String = "bin"
-        out.writeMeta(obj)
+    override val key: Short = 0x4249//BI
-    }
 
-    override fun read(input: Input): Meta {
+    override fun Input.readObject(): Meta {
-        return (input.readMetaItem() as MetaItem.NodeItem).node
+        return (readMetaItem() as MetaItem.NodeItem).node
     }
 
     private fun Output.writeChar(char: Char) = writeByte(char.toByte())
@@ -70,7 +69,7 @@ object BinaryMetaFormat : MetaFormat {
         }
     }
 
-    private fun Output.writeMeta(meta: Meta) {
+    override fun Output.writeObject(meta: Meta) {
         writeChar('M')
         writeInt(meta.items.size)
         meta.items.forEach { (key, item) ->
@@ -80,7 +79,7 @@ object BinaryMetaFormat : MetaFormat {
                     writeValue(item.value)
                 }
                 is MetaItem.NodeItem -> {
-                    writeMeta(item.node)
+                    writeObject(item.node)
                 }
             }
         }
@@ -122,11 +121,4 @@ object BinaryMetaFormat : MetaFormat {
             else -> error("Unknown serialization key character: $keyChar")
         }
     }
-}
-
-class BinaryMetaFormatFactory : MetaFormatFactory {
-    override val name: String = "bin"
-    override val key: Short = 0x4249//BI
-
-    override fun build(): MetaFormat = BinaryMetaFormat
 }

dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Envelope.kt

@@ -7,7 +7,7 @@ import kotlinx.io.core.Input
 
 interface Envelope {
     val meta: Meta
-    val data: Input?
+    val data: Binary?
 
     companion object {
 
@@ -23,11 +23,7 @@ interface Envelope {
     }
 }
 
-class SimpleEnvelope(override val meta: Meta, val dataProvider: () -> Input?) : Envelope{
+class SimpleEnvelope(override val meta: Meta, override val data: Binary?) : Envelope
-    override val data: Input?
-        get() = dataProvider()
-
-}
 
 /**
  * The purpose of the envelope
@@ -50,3 +46,4 @@ val Envelope.dataType: String? get() = meta[Envelope.ENVELOPE_DATA_TYPE_KEY].str
  */
 val Envelope.description: String? get() = meta[Envelope.ENVELOPE_DESCRIPTION_KEY].string
 
+typealias EnvelopeFormat = IOFormat<Envelope>

dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOFormat.kt

@@ -1,10 +1,14 @@
 package hep.dataforge.io
 
-import kotlinx.io.core.Input
+import kotlinx.io.core.*
-import kotlinx.io.core.Output
-
 
+/**
+ * And interface for serialization facilities
+ */
 interface IOFormat<T : Any> {
-    fun write(obj: T, out: Output)
+    fun Output.writeObject(obj: T)
-    fun read(input: Input): T
+    fun Input.readObject(): T
-}
+}
+
+fun <T : Any> IOFormat<T>.writePacket(obj: T): ByteReadPacket = buildPacket { writeObject(obj) }
+fun <T : Any> IOFormat<T>.writeBytes(obj: T): ByteArray = buildPacket { writeObject(obj) }.readBytes()

dataforge-io/src/commonMain/kotlin/hep/dataforge/io/JsonMetaFormat.kt

@@ -14,13 +14,16 @@ import kotlinx.serialization.json.*
 
 object JsonMetaFormat : MetaFormat {
 
-    override fun write(obj: Meta, out: Output) {
+    override val name: String = "json"
+    override val key: Short = 0x4a53//"JS"
+
+    override fun Output.writeObject(obj: Meta) {
         val str = obj.toJson().toString()
-        out.writeText(str)
+        writeText(str)
     }
 
-    override fun read(input: Input): Meta {
+    override fun Input.readObject(): Meta {
-        val str = input.readText()
+        val str = readText()
         val json = Json.plain.parseJson(str)
 
         if (json is JsonObject) {
@@ -97,11 +100,4 @@ class JsonMeta(val json: JsonObject) : Meta {
         json.forEach { (key, value) -> map[key] = value }
         map.mapKeys { it.key.toName().first()!! }
     }
-}
-
-class JsonMetaFormatFactory : MetaFormatFactory {
-    override val name: String = "json"
-    override val key: Short = 0x4a53//"JS"
-
-    override fun build() = JsonMetaFormat
 }

dataforge-io/src/commonMain/kotlin/hep/dataforge/io/MetaFormat.kt

@@ -1,33 +1,26 @@
 package hep.dataforge.io
 
 import hep.dataforge.meta.Meta
-import kotlinx.io.core.BytePacketBuilder
 import kotlinx.io.core.ByteReadPacket
+import kotlinx.io.core.buildPacket
 import kotlinx.io.core.toByteArray
 
 /**
  * A format for meta serialization
  */
-interface MetaFormat: IOFormat<Meta>
+interface MetaFormat : IOFormat<Meta> {
-
-/**
- * ServiceLoader compatible factory
- */
-interface MetaFormatFactory {
     val name: String
     val key: Short
-
-    fun build(): MetaFormat
 }
 
 fun Meta.asString(format: MetaFormat = JsonMetaFormat): String {
-    val builder = BytePacketBuilder()
+    return buildPacket {
-    format.write(this, builder)
+        format.run { writeObject(this@asString) }
-    return builder.build().readText()
+    }.readText()
 }
 
 fun MetaFormat.parse(str: String): Meta {
-    return read(ByteReadPacket(str.toByteArray()))
+    return ByteReadPacket(str.toByteArray()).readObject()
 }
 
 

dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt

@@ -0,0 +1,80 @@
+package hep.dataforge.io
+
+import kotlinx.io.core.*
+
+class TaggedEnvelopeFormat(
+    val metaFormats: Collection<MetaFormat>,
+    val outputMetaFormat: MetaFormat = metaFormats.first()
+) : EnvelopeFormat {
+
+    override fun Output.writeObject(obj: Envelope) {
+        write(obj, this, outputMetaFormat)
+    }
+
+    /**
+     * Read an envelope from input into memory
+     *
+     * @param input an input to read from
+     * @param metaFormats a collection of meta formats to resolve
+     */
+    override fun Input.readObject(): Envelope = read(this, metaFormats)
+
+
+    private data class Tag(
+        val metaFormatKey: Short,
+        val metaSize: UInt,
+        val dataSize: ULong
+    )
+
+    companion object {
+        private const val VERSION = "DF03"
+        private const val START_SEQUENCE = "#~"
+        private const val END_SEQUENCE = "~#\r\n"
+
+        private fun Tag.toBytes(): ByteReadPacket = buildPacket(24) {
+            writeText(START_SEQUENCE)
+            writeText(VERSION)
+            writeShort(metaFormatKey)
+            writeUInt(metaSize)
+            writeULong(dataSize)
+            writeText(END_SEQUENCE)
+        }
+
+        private fun Input.readTag(): Tag {
+            val start = readTextExactBytes(2)
+            if (start != START_SEQUENCE) error("The input is not an envelope")
+            val version = readTextExactBytes(4)
+            if (version != VERSION) error("Wrong version of DataForge: expected $VERSION but found $version")
+            val metaFormatKey = readShort()
+            val metaLength = readUInt()
+            val dataLength = readULong()
+            return Tag(metaFormatKey, metaLength, dataLength)
+        }
+
+        fun read(input: Input, metaFormats: Collection<MetaFormat>): Envelope {
+            val tag = input.readTag()
+
+            val metaFormat = metaFormats.find { it.key == tag.metaFormatKey }
+                ?: error("Meta format with key ${tag.metaFormatKey} not found")
+
+            val metaPacket = ByteReadPacket(input.readBytes(tag.metaSize.toInt()))
+            val meta = metaFormat.run { metaPacket.readObject() }
+
+            val dataBytes = input.readBytes(tag.dataSize.toInt())
+
+            return SimpleEnvelope(meta, ArrayBinary(dataBytes))
+        }
+
+        fun write(obj: Envelope, out: Output, metaFormat: MetaFormat) {
+            val metaBytes = metaFormat.writeBytes(obj.meta)
+            val tag = Tag(metaFormat.key, metaBytes.size.toUInt(), obj.data?.size ?: 0.toULong())
+            out.writePacket(tag.toBytes())
+            out.writeFully(metaBytes)
+            obj.data?.read {
+                while (!endOfInput){
+                    out.writeByte(readByte())
+                }
+            }
+        }
+    }
+}

dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileBinary.kt

@@ -0,0 +1,16 @@
+package hep.dataforge.io
+
+import kotlinx.io.core.ByteReadPacket
+import kotlinx.io.core.Input
+import java.nio.channels.FileChannel
+import java.nio.file.Path
+import java.nio.file.StandardOpenOption
+
+class FileBinary(val path: Path, private val offset: Int = 0) : RandomAccessBinary {
+    override fun <R> read(from: Long, size: Long, block: Input.() -> R): R {
+        FileChannel.open(path, StandardOpenOption.READ).use {
+            val buffer = it.map(FileChannel.MapMode.READ_ONLY, from + offset, size)
+            return ByteReadPacket(buffer).block()
+        }
+    }
+}

dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MutableMeta.kt

@@ -95,12 +95,13 @@ abstract class MutableMetaNode<M : MutableMetaNode<M>> : AbstractMetaNode<M>(),
 fun <M : MutableMeta<M>> MutableMeta<M>.remove(name: Name) = set(name, null)
 fun <M : MutableMeta<M>> MutableMeta<M>.remove(name: String) = remove(name.toName())
 
-fun <M : MutableMeta<M>> MutableMeta<M>.setValue(name: Name, value: Value) = set(name, MetaItem.ValueItem(value))
+fun <M : MutableMeta<M>> MutableMeta<M>.setValue(name: Name, value: Value) =
-//fun <M : MutableMeta<M>> MutableMeta<M>.setItem(name: String, item: MetaItem<M>) = set(name.toName(), item)
+    set(name, MetaItem.ValueItem(value))
 fun <M : MutableMeta<M>> MutableMeta<M>.setValue(name: String, value: Value) =
     set(name.toName(), MetaItem.ValueItem(value))
 
 //fun <M : MutableMeta<M>> MutableMeta<M>.setItem(token: NameToken, item: MetaItem<M>?) = set(token.asName(), item)
+//fun <M : MutableMeta<M>> MutableMeta<M>.setItem(name: String, item: MetaItem<M>) = set(name.toName(), item)
 
 fun <M : MutableMetaNode<M>> MutableMetaNode<M>.setItem(name: Name, item: MetaItem<*>) {
     when (item) {