Fix Envelope IO with binaries

dataforge-context/src/commonMain/kotlin/hep/dataforge/context/AbstractPlugin.kt

@@ -1,13 +1,12 @@
 package hep.dataforge.context
 
-import hep.dataforge.meta.EmptyMeta
 import hep.dataforge.meta.Meta
 import hep.dataforge.names.Name
 import kotlin.properties.ReadOnlyProperty
 import kotlin.reflect.KClass
 import kotlin.reflect.KProperty
 
-abstract class AbstractPlugin(override val meta: Meta = EmptyMeta) : Plugin {
+abstract class AbstractPlugin(override val meta: Meta = Meta.EMPTY) : Plugin {
     private var _context: Context? = null
     private val dependencies = ArrayList<PluginFactory<*>>()
 

dataforge-context/src/commonMain/kotlin/hep/dataforge/context/Factory.kt

@@ -1,8 +1,7 @@
 package hep.dataforge.context
 
-import hep.dataforge.meta.EmptyMeta
 import hep.dataforge.meta.Meta
 
 interface Factory<out T : Any> {
-    operator fun invoke(meta: Meta = EmptyMeta, context: Context = Global): T
+    operator fun invoke(meta: Meta = Meta.EMPTY, context: Context = Global): T
 }

dataforge-context/src/commonMain/kotlin/hep/dataforge/context/PluginRepository.kt

@@ -1,6 +1,5 @@
 package hep.dataforge.context
 
-import hep.dataforge.meta.EmptyMeta
 import hep.dataforge.meta.Meta
 import kotlin.reflect.KClass
 
@@ -23,7 +22,7 @@ expect object PluginRepository {
 /**
  * Fetch specific plugin and instantiate it with given meta
  */
-fun PluginRepository.fetch(tag: PluginTag, meta: Meta = EmptyMeta): Plugin =
+fun PluginRepository.fetch(tag: PluginTag, meta: Meta = Meta.EMPTY): Plugin =
     list().find { it.tag.matches(tag) }?.invoke(meta = meta)
         ?: error("Plugin with tag $tag not found in the repository")
 

dataforge-io/dataforge-io-yaml/src/main/kotlin/hep/dataforge/io/yaml/FrontMatterEnvelopeFormat.kt

@@ -3,7 +3,6 @@ package hep.dataforge.io.yaml
 import hep.dataforge.context.Context
 import hep.dataforge.io.*
 import hep.dataforge.meta.DFExperimental
-import hep.dataforge.meta.EmptyMeta
 import hep.dataforge.meta.Meta
 import kotlinx.io.*
 import kotlinx.io.text.readUtf8Line
@@ -13,7 +12,7 @@ import kotlinx.serialization.toUtf8Bytes
 @DFExperimental
 class FrontMatterEnvelopeFormat(
     val io: IOPlugin,
-    meta: Meta = EmptyMeta
+    val meta: Meta = Meta.EMPTY
 ) : EnvelopeFormat {
 
     override fun Input.readPartial(): PartialEnvelope {
@@ -26,7 +25,7 @@ class FrontMatterEnvelopeFormat(
 
         val readMetaFormat =
             metaTypeRegex.matchEntire(line)?.groupValues?.first()
-                ?.let { io.metaFormat(it) } ?: YamlMetaFormat
+                ?.let { io.resolveMetaFormat(it) } ?: YamlMetaFormat
 
         //TODO replace by preview
         val meta = Binary {
@@ -51,11 +50,11 @@ class FrontMatterEnvelopeFormat(
 
         val readMetaFormat =
             metaTypeRegex.matchEntire(line)?.groupValues?.first()
-                ?.let { io.metaFormat(it) } ?: YamlMetaFormat
+                ?.let { io.resolveMetaFormat(it) } ?: YamlMetaFormat
 
         val meta = Binary {
             do {
-                writeUtf8String(readUtf8Line()  + "\r\n")
+                writeUtf8String(readUtf8Line() + "\r\n")
             } while (!line.startsWith(SEPARATOR))
         }.read {
             readMetaFormat.run {
@@ -78,6 +77,11 @@ class FrontMatterEnvelopeFormat(
         }
     }
 
+    override fun toMeta(): Meta = Meta {
+        IOPlugin.IO_FORMAT_NAME_KEY put name.toString()
+        IOPlugin.IO_FORMAT_META_KEY put meta
+    }
+
     companion object : EnvelopeFormatFactory {
         const val SEPARATOR = "---"
 
@@ -88,11 +92,13 @@ class FrontMatterEnvelopeFormat(
         }
 
         override fun peekFormat(io: IOPlugin, input: Input): EnvelopeFormat? {
-            val line = input.readUtf8Line()
+            return input.preview {
-            return if (line.startsWith("---")) {
+                val line = readUtf8Line()
-                invoke()
+                return@preview if (line.startsWith("---")) {
-            } else {
+                    invoke()
-                null
+                } else {
+                    null
+                }
             }
         }
 

dataforge-io/dataforge-io-yaml/src/main/kotlin/hep/dataforge/io/yaml/YamlMetaFormat.kt

@@ -1,6 +1,7 @@
 package hep.dataforge.io.yaml
 
 import hep.dataforge.context.Context
+import hep.dataforge.io.IOPlugin
 import hep.dataforge.io.MetaFormat
 import hep.dataforge.io.MetaFormatFactory
 import hep.dataforge.meta.DFExperimental
@@ -11,10 +12,8 @@ import hep.dataforge.meta.toMeta
 import kotlinx.io.Input
 import kotlinx.io.Output
 import kotlinx.io.asInputStream
-import kotlinx.io.readUByte
 import kotlinx.io.text.writeUtf8String
 import org.yaml.snakeyaml.Yaml
-import java.io.InputStream
 
 @DFExperimental
 class YamlMetaFormat(val meta: Meta) : MetaFormat {
@@ -30,6 +29,11 @@ class YamlMetaFormat(val meta: Meta) : MetaFormat {
         return map.toMeta(descriptor)
     }
 
+    override fun toMeta(): Meta  = Meta{
+        IOPlugin.IO_FORMAT_NAME_KEY put FrontMatterEnvelopeFormat.name.toString()
+        IOPlugin.IO_FORMAT_META_KEY put meta
+    }
+
     companion object : MetaFormatFactory {
         override fun invoke(meta: Meta, context: Context): MetaFormat = YamlMetaFormat(meta)
 

dataforge-io/src/commonMain/kotlin/hep/dataforge/io/EnvelopeFormat.kt

@@ -2,7 +2,6 @@ package hep.dataforge.io
 
 import hep.dataforge.context.Context
 import hep.dataforge.io.EnvelopeFormatFactory.Companion.ENVELOPE_FORMAT_TYPE
-import hep.dataforge.meta.EmptyMeta
 import hep.dataforge.meta.Meta
 import hep.dataforge.names.Name
 import hep.dataforge.names.asName
@@ -26,7 +25,7 @@ interface EnvelopeFormat : IOFormat<Envelope> {
     fun Output.writeEnvelope(
         envelope: Envelope,
         metaFormatFactory: MetaFormatFactory = defaultMetaFormat,
-        formatMeta: Meta = EmptyMeta
+        formatMeta: Meta = Meta.EMPTY
     )
 
     override fun Input.readObject(): Envelope

dataforge-io/src/commonMain/kotlin/hep/dataforge/io/EnvelopeParts.kt

@@ -1,123 +1,121 @@
 package hep.dataforge.io
 
-import hep.dataforge.context.Global
+import hep.dataforge.io.Envelope.Companion.ENVELOPE_NODE_KEY
-import hep.dataforge.io.EnvelopeParts.FORMAT_META_KEY
+import hep.dataforge.io.PartDescriptor.Companion.DEFAULT_MULTIPART_DATA_SEPARATOR
-import hep.dataforge.io.EnvelopeParts.FORMAT_NAME_KEY
+import hep.dataforge.io.PartDescriptor.Companion.MULTIPART_DATA_TYPE
-import hep.dataforge.io.EnvelopeParts.INDEX_KEY
+import hep.dataforge.io.PartDescriptor.Companion.MULTIPART_KEY
-import hep.dataforge.io.EnvelopeParts.MULTIPART_DATA_SEPARATOR
+import hep.dataforge.io.PartDescriptor.Companion.PARTS_KEY
-import hep.dataforge.io.EnvelopeParts.MULTIPART_DATA_TYPE
+import hep.dataforge.io.PartDescriptor.Companion.PART_FORMAT_KEY
-import hep.dataforge.io.EnvelopeParts.SIZE_KEY
+import hep.dataforge.io.PartDescriptor.Companion.SEPARATOR_KEY
 import hep.dataforge.meta.*
 import hep.dataforge.names.asName
 import hep.dataforge.names.plus
-import hep.dataforge.names.toName
+import kotlinx.io.Binary
+import kotlinx.io.writeBinary
 
-object EnvelopeParts {
+private class PartDescriptor : Scheme() {
-    val MULTIPART_KEY = "multipart".asName()
+    var offset by int(0)
-    val SIZE_KEY = Envelope.ENVELOPE_NODE_KEY + MULTIPART_KEY + "size"
+    var size by int(0)
-    val INDEX_KEY = Envelope.ENVELOPE_NODE_KEY + MULTIPART_KEY + "index"
+    var meta by node()
-    val FORMAT_NAME_KEY = Envelope.ENVELOPE_NODE_KEY + MULTIPART_KEY + "format"
-    val FORMAT_META_KEY = Envelope.ENVELOPE_NODE_KEY + MULTIPART_KEY + "meta"
 
-    const val MULTIPART_DATA_SEPARATOR = "\r\n#~PART~#\r\n"
+    companion object : SchemeSpec<PartDescriptor>(::PartDescriptor) {
+        val MULTIPART_KEY = ENVELOPE_NODE_KEY + "multipart"
+        val PARTS_KEY = MULTIPART_KEY + "parts"
+        val SEPARATOR_KEY = MULTIPART_KEY + "separator"
 
-    const val MULTIPART_DATA_TYPE = "envelope.multipart"
+        const val DEFAULT_MULTIPART_DATA_SEPARATOR = "\r\n#~PART~#\r\n"
+
+        val PART_FORMAT_KEY = "format".asName()
+
+        const val MULTIPART_DATA_TYPE = "envelope.multipart"
+    }
 }
 
-/**
+data class EnvelopePart(val binary: Binary, val description: Meta?)
- * Append multiple serialized envelopes to the data block. Previous data is erased if it was present
+
- */
+typealias EnvelopeParts = List<EnvelopePart>
-@DFExperimental
+
 fun EnvelopeBuilder.multipart(
-    envelopes: Collection<Envelope>,
+    parts: EnvelopeParts,
-    format: EnvelopeFormatFactory,
+    separator: String = DEFAULT_MULTIPART_DATA_SEPARATOR
-    formatMeta: Meta = EmptyMeta
 ) {
     dataType = MULTIPART_DATA_TYPE
-    meta {
+
-        SIZE_KEY put envelopes.size
+    var offsetCounter = 0
-        FORMAT_NAME_KEY put format.name.toString()
+    val separatorSize = separator.length
-        if (!formatMeta.isEmpty()) {
+    val partDescriptors = parts.map { (binary, description) ->
-            FORMAT_META_KEY put formatMeta
+        offsetCounter += separatorSize
+        PartDescriptor {
+            offset = offsetCounter
+            size = binary.size
+            meta = description
+        }.also {
+            offsetCounter += binary.size
         }
     }
+
+    meta {
+        if (separator != DEFAULT_MULTIPART_DATA_SEPARATOR) {
+            SEPARATOR_KEY put separator
+        }
+        setIndexed(PARTS_KEY, partDescriptors.map { it.toMeta() })
+    }
+
     data {
-        format(formatMeta).run {
+        parts.forEach {
-            envelopes.forEach {
+            writeRawString(separator)
-                writeRawString(MULTIPART_DATA_SEPARATOR)
+            writeBinary(it.binary)
-                writeEnvelope(it)
-            }
         }
     }
 }
 
-/**
+fun EnvelopeBuilder.envelopes(
- * Create a multipart partition in the envelope adding additional name-index mapping in meta
+    envelopes: List<Envelope>,
- */
+    format: EnvelopeFormat = TaggedEnvelopeFormat,
-@DFExperimental
+    separator: String = DEFAULT_MULTIPART_DATA_SEPARATOR
-fun EnvelopeBuilder.multipart(
-    envelopes: Map<String, Envelope>,
-    format: EnvelopeFormatFactory,
-    formatMeta: Meta = EmptyMeta
 ) {
-    dataType = MULTIPART_DATA_TYPE
+    val parts = envelopes.map {
-    meta {
+        val binary = format.toBinary(it)
-        SIZE_KEY put envelopes.size
+        EnvelopePart(binary, null)
-        FORMAT_NAME_KEY put format.name.toString()
-        if (!formatMeta.isEmpty()) {
-            FORMAT_META_KEY put formatMeta
-        }
     }
-    data {
+    meta{
-        format.run {
+        set(MULTIPART_KEY + PART_FORMAT_KEY, format.toMeta())
-            var counter = 0
+    }
-            envelopes.forEach { (key, envelope) ->
+    multipart(parts, separator)
-                writeRawString(MULTIPART_DATA_SEPARATOR)
+}
-                writeEnvelope(envelope)
+
-                meta {
+fun Envelope.parts(): EnvelopeParts {
-                    append(INDEX_KEY, Meta {
+    if (data == null) return emptyList()
-                        "key" put key
+    //TODO add zip folder reader
-                        "index" put counter
+    val parts = meta.getIndexed(PARTS_KEY).values.mapNotNull { it.node }.map {
-                    })
+        PartDescriptor.wrap(it)
-                }
+    }
-                counter++
+    return if (parts.isEmpty()) {
-            }
+        listOf(EnvelopePart(data!!, meta[MULTIPART_KEY].node))
+    } else {
+        parts.map {
+            val binary = data!!.view(it.offset, it.size)
+            val meta = Laminate(it.meta, meta[MULTIPART_KEY].node)
+            EnvelopePart(binary, meta)
         }
     }
 }
 
-@DFExperimental
+fun EnvelopePart.envelope(format: EnvelopeFormat): Envelope = binary.readWith(format)
-fun EnvelopeBuilder.multipart(
+
-    formatFactory: EnvelopeFormatFactory,
+val EnvelopePart.name: String? get() = description?.get("name").string
-    formatMeta: Meta = EmptyMeta,
-    builder: suspend SequenceScope<Envelope>.() -> Unit
-) = multipart(sequence(builder).toList(), formatFactory, formatMeta)
 
 /**
- * If given envelope supports multipart data, return a sequence of those parts (could be empty). Otherwise return null.
+ * Represent envelope part by an envelope
  */
-@DFExperimental
+fun EnvelopePart.envelope(plugin: IOPlugin): Envelope {
-fun Envelope.parts(io: IOPlugin = Global.plugins.fetch(IOPlugin)): Sequence<Envelope>? {
+    val formatItem = description?.get(PART_FORMAT_KEY)
-    return when (dataType) {
+    return if (formatItem != null) {
-        MULTIPART_DATA_TYPE -> {
+        val format: EnvelopeFormat = plugin.resolveEnvelopeFormat(formatItem)
-            val size = meta[SIZE_KEY].int ?: error("Unsized parts not supported yet")
+            ?: error("Envelope format for $formatItem is not resolved")
-            val formatName = meta[FORMAT_NAME_KEY].string?.toName()
+        binary.readWith(format)
-                ?: error("Inferring parts format is not supported at the moment")
+    } else {
-            val formatMeta = meta[FORMAT_META_KEY].node ?: EmptyMeta
+        error("Envelope description not found")
-            val format = io.envelopeFormat(formatName, formatMeta)
+        //SimpleEnvelope(description ?: Meta.EMPTY, binary)
-                ?: error("Format $formatName is not resolved by $io")
-            return format.run {
-                data?.read {
-                    sequence {
-                        repeat(size) {
-                            val separator = readRawString(MULTIPART_DATA_SEPARATOR.length)
-                            if(separator!= MULTIPART_DATA_SEPARATOR) error("Separator is expected, but $separator found")
-                            yield(readObject())
-                        }
-                    }
-                } ?: emptySequence()
-            }
-        }
-        else -> null
     }
 }

dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOFormat.kt

@@ -4,8 +4,10 @@ import hep.dataforge.context.Context
 import hep.dataforge.context.Factory
 import hep.dataforge.context.Named
 import hep.dataforge.io.IOFormatFactory.Companion.IO_FORMAT_TYPE
+import hep.dataforge.io.IOPlugin.Companion.IO_FORMAT_NAME_KEY
 import hep.dataforge.meta.Meta
 import hep.dataforge.meta.MetaItem
+import hep.dataforge.meta.MetaRepr
 import hep.dataforge.names.Name
 import hep.dataforge.names.asName
 import hep.dataforge.provider.Type
@@ -18,12 +20,20 @@ import kotlin.reflect.KClass
 /**
  * And interface for reading and writing objects into with IO streams
  */
-interface IOFormat<T : Any> {
+interface IOFormat<T : Any> : MetaRepr {
     fun Output.writeObject(obj: T)
     fun Input.readObject(): T
 }
 
 fun <T : Any> Input.readWith(format: IOFormat<T>): T = format.run { readObject() }
+
+/**
+ * Read given binary as object using given format
+ */
+fun <T : Any> Binary.readWith(format: IOFormat<T>): T = read {
+    readWith(format)
+}
+
 fun <T : Any> Output.writeWith(format: IOFormat<T>, obj: T) = format.run { writeObject(obj) }
 
 class ListIOFormat<T : Any>(val format: IOFormat<T>) : IOFormat<List<T>> {
@@ -42,6 +52,11 @@ class ListIOFormat<T : Any>(val format: IOFormat<T>) : IOFormat<List<T>> {
             List(size) { readObject() }
         }
     }
+
+    override fun toMeta(): Meta = Meta {
+        IO_FORMAT_NAME_KEY put "list"
+        "contentFormat" put format.toMeta()
+    }
 }
 
 val <T : Any> IOFormat<T>.list get() = ListIOFormat(this)
@@ -57,12 +72,16 @@ fun ObjectPool<Buffer>.fill(block: Buffer.() -> Unit): Buffer {
 }
 
 @Type(IO_FORMAT_TYPE)
-interface IOFormatFactory<T : Any> : Factory<IOFormat<T>>, Named {
+interface IOFormatFactory<T : Any> : Factory<IOFormat<T>>, Named, MetaRepr {
     /**
      * Explicit type for dynamic type checks
      */
     val type: KClass<out T>
 
+    override fun toMeta(): Meta = Meta {
+        IO_FORMAT_NAME_KEY put name.toString()
+    }
+
     companion object {
         const val IO_FORMAT_TYPE = "io.format"
     }
@@ -100,12 +119,3 @@ object ValueIOFormat : IOFormat<Value>, IOFormatFactory<Value> {
             ?: error("The item is not a value")
     }
 }
-
-/**
- * Read given binary as object using given format
- */
-fun <T : Any> Binary.readWith(format: IOFormat<T>): T = format.run {
-    read {
-        readObject()
-    }
-}

dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOPlugin.kt

@@ -6,29 +6,51 @@ import hep.dataforge.io.IOFormatFactory.Companion.IO_FORMAT_TYPE
 import hep.dataforge.io.MetaFormatFactory.Companion.META_FORMAT_TYPE
 import hep.dataforge.meta.*
 import hep.dataforge.names.Name
-import hep.dataforge.names.get
+import hep.dataforge.names.asName
+import hep.dataforge.names.toName
 import kotlin.reflect.KClass
 
 class IOPlugin(meta: Meta) : AbstractPlugin(meta) {
     override val tag: PluginTag get() = Companion.tag
 
+    val ioFormatFactories by lazy {
+        context.content<IOFormatFactory<*>>(IO_FORMAT_TYPE).values
+    }
+
+    fun <T : Any> resolveIOFormat(item: MetaItem<*>, type: KClass<out T>): IOFormat<T>? {
+        val key = item.string ?: item.node[IO_FORMAT_NAME_KEY]?.string ?: error("Format name not defined")
+        val name = key.toName()
+        return ioFormatFactories.find { it.name == name }?.let {
+            @Suppress("UNCHECKED_CAST")
+            if (it.type != type) error("Format type ${it.type} is not the same as requested type $type")
+            else it.invoke(item.node[IO_FORMAT_META_KEY].node ?: Meta.EMPTY, context) as IOFormat<T>
+        }
+    }
+
+
     val metaFormatFactories by lazy {
         context.content<MetaFormatFactory>(META_FORMAT_TYPE).values
     }
 
-    fun metaFormat(key: Short, meta: Meta = EmptyMeta): MetaFormat? =
+    fun resolveMetaFormat(key: Short, meta: Meta = Meta.EMPTY): MetaFormat? =
         metaFormatFactories.find { it.key == key }?.invoke(meta)
 
-    fun metaFormat(name: String, meta: Meta = EmptyMeta): MetaFormat? =
+    fun resolveMetaFormat(name: String, meta: Meta = Meta.EMPTY): MetaFormat? =
         metaFormatFactories.find { it.shortName == name }?.invoke(meta)
 
     val envelopeFormatFactories by lazy {
         context.content<EnvelopeFormatFactory>(ENVELOPE_FORMAT_TYPE).values
     }
 
-    fun envelopeFormat(name: Name, meta: Meta = EmptyMeta) =
+    fun resolveEnvelopeFormat(name: Name, meta: Meta = Meta.EMPTY): EnvelopeFormat? =
         envelopeFormatFactories.find { it.name == name }?.invoke(meta, context)
 
+    fun resolveEnvelopeFormat(item: MetaItem<*>): EnvelopeFormat? {
+        val name = item.string ?: item.node[IO_FORMAT_NAME_KEY]?.string ?: error("Envelope format name not defined")
+        val meta = item.node[IO_FORMAT_META_KEY].node ?: Meta.EMPTY
+        return resolveEnvelopeFormat(name.toName(), meta)
+    }
+
     override fun provideTop(target: String): Map<Name, Any> {
         return when (target) {
             META_FORMAT_TYPE -> defaultMetaFormats.toMap()
@@ -37,20 +59,10 @@ class IOPlugin(meta: Meta) : AbstractPlugin(meta) {
         }
     }
 
-    val ioFormats: Map<Name, IOFormatFactory<*>> by lazy {
-        context.content<IOFormatFactory<*>>(IO_FORMAT_TYPE)
-    }
-
-    fun <T : Any> resolveIOFormat(item: MetaItem<*>, type: KClass<out T>): IOFormat<T>? {
-        val key = item.string ?: item.node["name"]?.string ?: error("Format name not defined")
-        return ioFormats[key]?.let {
-            @Suppress("UNCHECKED_CAST")
-            if (it.type != type) error("Format type ${it.type} is not the same as requested type $type")
-            else it.invoke(item.node["meta"].node ?: EmptyMeta, context) as IOFormat<T>
-        }
-    }
-
     companion object : PluginFactory<IOPlugin> {
+        val IO_FORMAT_NAME_KEY = "name".asName()
+        val IO_FORMAT_META_KEY = "meta".asName()
+
         val defaultMetaFormats: List<MetaFormatFactory> = listOf(JsonMetaFormat, BinaryMetaFormat)
         val defaultEnvelopeFormats = listOf(TaggedEnvelopeFormat, TaglessEnvelopeFormat)
 

dataforge-io/src/commonMain/kotlin/hep/dataforge/io/JsonMetaFormat.kt

@@ -12,7 +12,6 @@ import hep.dataforge.meta.toMetaItem
 import kotlinx.io.Input
 import kotlinx.io.Output
 import kotlinx.io.readByteArray
-import kotlinx.io.text.readUtf8String
 import kotlinx.io.text.writeUtf8String
 import kotlinx.serialization.UnstableDefault
 import kotlinx.serialization.json.Json
@@ -26,6 +25,10 @@ class JsonMetaFormat(private val json: Json = DEFAULT_JSON) : MetaFormat {
         writeUtf8String(json.stringify(JsonObjectSerializer, jsonObject))
     }
 
+    override fun toMeta(): Meta  = Meta{
+        IOPlugin.IO_FORMAT_NAME_KEY put name.toString()
+    }
+
     override fun Input.readMeta(descriptor: NodeDescriptor?): Meta {
         val str = readByteArray().decodeToString()
         val jsonElement = json.parseJson(str)

dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt

@@ -2,6 +2,7 @@ package hep.dataforge.io
 
 import hep.dataforge.context.Context
 import hep.dataforge.meta.Meta
+import hep.dataforge.meta.enum
 import hep.dataforge.meta.get
 import hep.dataforge.meta.string
 import hep.dataforge.names.Name
@@ -9,7 +10,6 @@ import hep.dataforge.names.plus
 import hep.dataforge.names.toName
 import kotlinx.io.*
 
-@ExperimentalIoApi
 class TaggedEnvelopeFormat(
     val io: IOPlugin,
     val version: VERSION = VERSION.DF02
@@ -58,7 +58,7 @@ class TaggedEnvelopeFormat(
     override fun Input.readObject(): Envelope {
         val tag = readTag(version)
 
-        val metaFormat = io.metaFormat(tag.metaFormatKey)
+        val metaFormat = io.resolveMetaFormat(tag.metaFormatKey)
             ?: error("Meta format with key ${tag.metaFormatKey} not found")
 
         val meta: Meta = limit(tag.metaSize.toInt()).run {
@@ -67,7 +67,7 @@ class TaggedEnvelopeFormat(
             }
         }
 
-        val data = ByteArray(tag.dataSize.toInt()).also { readByteArray(it) }.asBinary()
+        val data = readBinary(tag.dataSize.toInt())
 
         return SimpleEnvelope(meta, data)
     }
@@ -75,7 +75,7 @@ class TaggedEnvelopeFormat(
     override fun Input.readPartial(): PartialEnvelope {
         val tag = readTag(version)
 
-        val metaFormat = io.metaFormat(tag.metaFormatKey)
+        val metaFormat = io.resolveMetaFormat(tag.metaFormatKey)
             ?: error("Meta format with key ${tag.metaFormatKey} not found")
 
         val meta: Meta = limit(tag.metaSize.toInt()).run {
@@ -98,6 +98,13 @@ class TaggedEnvelopeFormat(
         DF03(24u)
     }
 
+    override fun toMeta(): Meta = Meta {
+        IOPlugin.IO_FORMAT_NAME_KEY put name.toString()
+        IOPlugin.IO_FORMAT_META_KEY put {
+            "version" put version
+        }
+    }
+
     companion object : EnvelopeFormatFactory {
         private const val START_SEQUENCE = "#~"
         private const val END_SEQUENCE = "~#\r\n"
@@ -111,7 +118,9 @@ class TaggedEnvelopeFormat(
             //Check if appropriate factory exists
             io.metaFormatFactories.find { it.name == metaFormatName } ?: error("Meta format could not be resolved")
 
-            return TaggedEnvelopeFormat(io)
+            val version: VERSION = meta["version"].enum<VERSION>() ?: VERSION.DF02
+
+            return TaggedEnvelopeFormat(io, version)
         }
 
         private fun Input.readTag(version: VERSION): Tag {
@@ -132,11 +141,13 @@ class TaggedEnvelopeFormat(
 
         override fun peekFormat(io: IOPlugin, input: Input): EnvelopeFormat? {
             return try {
-                val header = input.readRawString(6)
+                input.preview {
-                when (header.substring(2..5)) {
+                    val header = readRawString(6)
-                    VERSION.DF02.name -> TaggedEnvelopeFormat(io, VERSION.DF02)
+                    return@preview when (header.substring(2..5)) {
-                    VERSION.DF03.name -> TaggedEnvelopeFormat(io, VERSION.DF03)
+                        VERSION.DF02.name -> TaggedEnvelopeFormat(io, VERSION.DF02)
-                    else -> null
+                        VERSION.DF03.name -> TaggedEnvelopeFormat(io, VERSION.DF03)
+                        else -> null
+                    }
                 }
             } catch (ex: Exception) {
                 null

dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaglessEnvelopeFormat.kt

@@ -1,16 +1,19 @@
 package hep.dataforge.io
 
 import hep.dataforge.context.Context
-import hep.dataforge.meta.*
+import hep.dataforge.meta.Meta
+import hep.dataforge.meta.get
+import hep.dataforge.meta.isEmpty
+import hep.dataforge.meta.string
 import hep.dataforge.names.asName
 import kotlinx.io.*
 import kotlinx.io.text.readUtf8Line
 import kotlinx.io.text.writeUtf8String
+import kotlin.collections.set
 
-@ExperimentalIoApi
 class TaglessEnvelopeFormat(
     val io: IOPlugin,
-    meta: Meta = EmptyMeta
+    val meta: Meta = Meta.EMPTY
 ) : EnvelopeFormat {
 
     private val metaStart = meta[META_START_PROPERTY].string ?: DEFAULT_META_START
@@ -69,10 +72,10 @@ class TaglessEnvelopeFormat(
             line = readUtf8Line()
         }
 
-        var meta: Meta = EmptyMeta
+        var meta: Meta = Meta.EMPTY
 
         if (line.startsWith(metaStart)) {
-            val metaFormat = properties[META_TYPE_PROPERTY]?.let { io.metaFormat(it) } ?: JsonMetaFormat
+            val metaFormat = properties[META_TYPE_PROPERTY]?.let { io.resolveMetaFormat(it) } ?: JsonMetaFormat
             val metaSize = properties[META_LENGTH_PROPERTY]?.toInt()
             meta = if (metaSize != null) {
                 limit(metaSize).run {
@@ -95,9 +98,10 @@ class TaglessEnvelopeFormat(
         } while (!line.startsWith(dataStart))
 
         val data: Binary? = if (properties.containsKey(DATA_LENGTH_PROPERTY)) {
-            val bytes = ByteArray(properties[DATA_LENGTH_PROPERTY]!!.toInt())
+            readBinary(properties[DATA_LENGTH_PROPERTY]!!.toInt())
-            readByteArray(bytes)
+//            val bytes = ByteArray(properties[DATA_LENGTH_PROPERTY]!!.toInt())
-            bytes.asBinary()
+//            readByteArray(bytes)
+//            bytes.asBinary()
         } else {
             Binary {
                 copyTo(this)
@@ -132,10 +136,10 @@ class TaglessEnvelopeFormat(
             }
         }
 
-        var meta: Meta = EmptyMeta
+        var meta: Meta = Meta.EMPTY
 
         if (line.startsWith(metaStart)) {
-            val metaFormat = properties[META_TYPE_PROPERTY]?.let { io.metaFormat(it) } ?: JsonMetaFormat
+            val metaFormat = properties[META_TYPE_PROPERTY]?.let { io.resolveMetaFormat(it) } ?: JsonMetaFormat
             val metaSize = properties[META_LENGTH_PROPERTY]?.toInt()
             meta = if (metaSize != null) {
                 offset += metaSize.toUInt()
@@ -157,6 +161,11 @@ class TaglessEnvelopeFormat(
         return PartialEnvelope(meta, offset, dataSize)
     }
 
+    override fun toMeta(): Meta = Meta {
+        IOPlugin.IO_FORMAT_NAME_KEY put name.toString()
+        IOPlugin.IO_FORMAT_META_KEY put meta
+    }
+
     companion object : EnvelopeFormatFactory {
 
         private val propertyPattern = "#\\?\\s*(?<key>[\\w.]*)\\s*:\\s*(?<value>[^;]*);?".toRegex()
@@ -195,11 +204,13 @@ class TaglessEnvelopeFormat(
 
         override fun peekFormat(io: IOPlugin, input: Input): EnvelopeFormat? {
             return try {
-                val string = input.readRawString(TAGLESS_ENVELOPE_HEADER.length)
+                input.preview {
-                return if (string == TAGLESS_ENVELOPE_HEADER) {
+                    val string = readRawString(TAGLESS_ENVELOPE_HEADER.length)
-                    TaglessEnvelopeFormat(io)
+                    return@preview if (string == TAGLESS_ENVELOPE_HEADER) {
-                } else {
+                        TaglessEnvelopeFormat(io)
-                    null
+                    } else {
+                        null
+                    }
                 }
             } catch (ex: Exception) {
                 null

dataforge-io/src/commonMain/kotlin/hep/dataforge/io/ioMisc.kt

@@ -1,6 +1,7 @@
 package hep.dataforge.io
 
 import kotlinx.io.*
+import kotlin.math.min
 
 fun Output.writeRawString(str: String) {
     str.forEach { writeByte(it.toByte()) }
@@ -18,5 +19,24 @@ inline fun buildByteArray(expectedSize: Int = 16, block: Output.() -> Unit): Byt
 inline fun Binary(expectedSize: Int = 16, block: Output.() -> Unit): Binary =
     buildByteArray(expectedSize, block).asBinary()
 
-@Deprecated("To be replaced by Binary.EMPTY",level = DeprecationLevel.WARNING)
+@Deprecated("To be replaced by Binary.EMPTY", level = DeprecationLevel.WARNING)
 val EmptyBinary = ByteArrayBinary(ByteArray(0))
+
+/**
+ * View section of a [Binary] as an independent binary
+ */
+class BinaryView(private val source: Binary, private val start: Int, override val size: Int) : Binary {
+
+    init {
+        require(start > 0)
+        require(start + size <= source.size) { "View boundary is outside source binary size" }
+    }
+
+    override fun <R> read(offset: Int, atMost: Int, block: Input.() -> R): R {
+        return source.read(start + offset, min(size, atMost), block)
+    }
+}
+
+fun Binary.view(start: Int, size: Int) = BinaryView(this, start, size)
+
+operator fun Binary.get(range: IntRange) = view(range.first, range.last - range.first)

dataforge-io/src/commonTest/kotlin/hep/dataforge/io/BinaryTest.kt

@@ -0,0 +1,20 @@
+package hep.dataforge.io
+
+import kotlinx.io.asBinary
+import kotlinx.io.readByte
+import kotlinx.io.readInt
+import kotlin.test.Test
+import kotlin.test.assertEquals
+
+class BinaryTest {
+    @Test
+    fun testBinaryAccess(){
+        val binary = ByteArray(128){it.toByte()}.asBinary()
+
+        binary[3..12].read {
+            readInt()
+            val res = readByte()
+            assertEquals(7, res)
+        }
+    }
+}

dataforge-io/src/commonTest/kotlin/hep/dataforge/io/MultipartTest.kt

@@ -1,17 +1,19 @@
 package hep.dataforge.io
 
+import hep.dataforge.context.Global
 import hep.dataforge.meta.DFExperimental
 import hep.dataforge.meta.get
 import hep.dataforge.meta.int
 import kotlinx.io.text.writeUtf8String
-
 import kotlin.test.Test
 import kotlin.test.assertEquals
 import kotlin.test.assertTrue
 
 @DFExperimental
 class MultipartTest {
-    val envelopes = (0..5).map {
+    val io: IOPlugin = Global.io
+
+    val envelopes = (0 until 5).map {
         Envelope {
             meta {
                 "value" put it
@@ -26,19 +28,21 @@ class MultipartTest {
     }
 
     val partsEnvelope = Envelope {
-        multipart(envelopes, TaggedEnvelopeFormat)
+        envelopes(envelopes, TaglessEnvelopeFormat)
     }
 
     @Test
     fun testParts() {
-        TaggedEnvelopeFormat.run {
+        TaglessEnvelopeFormat.run {
             val singleEnvelopeData = toBinary(envelopes[0])
             val singleEnvelopeSize = singleEnvelopeData.size
             val bytes = toBinary(partsEnvelope)
-            assertTrue(5*singleEnvelopeSize < bytes.size)
+            assertTrue(envelopes.size * singleEnvelopeSize < bytes.size)
             val reconstructed = bytes.readWith(this)
-            val parts = reconstructed.parts()?.toList() ?: emptyList()
+            println(reconstructed.meta)
-            assertEquals(2, parts[2].meta["value"].int)
+            val parts = reconstructed.parts()
+            val envelope = parts[2].envelope(io)
+            assertEquals(2, envelope.meta["value"].int)
             println(reconstructed.data!!.size)
         }
     }

dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/fileIO.kt

@@ -1,7 +1,6 @@
 package hep.dataforge.io
 
 import hep.dataforge.meta.DFExperimental
-import hep.dataforge.meta.EmptyMeta
 import hep.dataforge.meta.Meta
 import hep.dataforge.meta.descriptors.NodeDescriptor
 import hep.dataforge.meta.isEmpty
@@ -60,7 +59,7 @@ fun Path.readEnvelope(format: EnvelopeFormat): Envelope {
 @Suppress("UNCHECKED_CAST")
 @DFExperimental
 inline fun <reified T : Any> IOPlugin.resolveIOFormat(): IOFormat<T>? {
-    return ioFormats.values.find { it.type.isSuperclassOf(T::class) } as IOFormat<T>?
+    return ioFormatFactories.find { it.type.isSuperclassOf(T::class) } as IOFormat<T>?
 }
 
 /**
@@ -78,7 +77,7 @@ fun IOPlugin.readMetaFile(path: Path, formatOverride: MetaFormat? = null, descri
     }
     val extension = actualPath.fileName.toString().substringAfterLast('.')
 
-    val metaFormat = formatOverride ?: metaFormat(extension) ?: error("Can't resolve meta format $extension")
+    val metaFormat = formatOverride ?: resolveMetaFormat(extension) ?: error("Can't resolve meta format $extension")
     return metaFormat.run {
         actualPath.read {
             readMeta(descriptor)
@@ -157,7 +156,7 @@ fun IOPlugin.readEnvelopeFile(
             .singleOrNull { it.fileName.toString().startsWith(IOPlugin.META_FILE_NAME) }
 
         val meta = if (metaFile == null) {
-            EmptyMeta
+            Meta.EMPTY
         } else {
             readMetaFile(metaFile)
         }

dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/functionsJVM.kt

@@ -9,7 +9,7 @@ import kotlin.reflect.full.isSuperclassOf
 
 
 fun IOPlugin.resolveIOFormatName(type: KClass<*>): Name {
-    return ioFormats.entries.find { it.value.type.isSuperclassOf(type) }?.key
+    return ioFormatFactories.find { it.type.isSuperclassOf(type) }?.name
         ?: error("Can't resolve IOFormat for type $type")
 }
 

dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/FileEnvelopeTest.kt

@@ -1,12 +1,14 @@
 package hep.dataforge.io
 
 import hep.dataforge.context.Global
+import hep.dataforge.meta.DFExperimental
 import kotlinx.io.writeDouble
 import java.nio.file.Files
 import kotlin.test.Test
 import kotlin.test.assertTrue
 
 
+@DFExperimental
 class FileEnvelopeTest {
     val envelope = Envelope {
         meta {

dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/tcp/EnvelopeServerTest.kt

@@ -46,7 +46,7 @@ class EnvelopeServerTest {
 
     @Test(timeout = 1000)
     fun doEchoTest() {
-        val request = Envelope.invoke {
+        val request = Envelope {
             type = "test.echo"
             meta {
                 "test.value" put 22

dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Meta.kt

@@ -98,7 +98,7 @@ interface Meta : MetaRepr {
          */
         const val VALUE_KEY = "@value"
 
-        val EMPTY: EmptyMeta = EmptyMeta
+        val EMPTY = EmptyMeta
     }
 }
 
@@ -188,7 +188,7 @@ abstract class MetaBase : Meta {
 
     override fun hashCode(): Int = items.hashCode()
 
-    override fun toString(): String = toJson().toString()
+    override fun toString(): String = PRETTY_JSON.stringify(MetaSerializer, this)
 }
 
 /**
@@ -216,6 +216,7 @@ fun MetaItem<*>.seal(): MetaItem<SealedMeta> = when (this) {
     is NodeItem -> NodeItem(node.seal())
 }
 
+@Deprecated("Use Meta.EMPTY instead", replaceWith = ReplaceWith("Meta.EMPTY"))
 object EmptyMeta : MetaBase() {
     override val items: Map<NameToken, MetaItem<*>> = emptyMap()
 }
@@ -251,4 +252,4 @@ val <M : Meta> MetaItem<M>?.node: M?
         is NodeItem -> node
     }
 
-fun Meta.isEmpty() = this === EmptyMeta || this.items.isEmpty()
+fun Meta.isEmpty() = this === Meta.EMPTY || this.items.isEmpty()

dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MutableMetaDelegate.kt

@@ -34,9 +34,7 @@ class ReadWriteDelegateWrapper<T, R>(
     val reader: (T) -> R,
     val writer: (R) -> T
 ) : ReadWriteProperty<Any?, R> {
-    override fun getValue(thisRef: Any?, property: KProperty<*>): R {
+    override fun getValue(thisRef: Any?, property: KProperty<*>): R = reader(delegate.getValue(thisRef, property))
-        return reader(delegate.getValue(thisRef, property))
-    }
 
     override fun setValue(thisRef: Any?, property: KProperty<*>, value: R) {
         delegate.setValue(thisRef, property, writer(value))

dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Specification.kt

@@ -16,7 +16,7 @@ interface Specification<T : Configurable> {
      */
     fun wrap(config: Config = Config(), defaultProvider: (Name) -> MetaItem<*>? = { null }): T
 
-    operator fun invoke(action: T.() -> Unit) = empty().apply(action)
+    operator fun invoke(action: T.() -> Unit): T = empty().apply(action)
 }
 
 /**
@@ -27,12 +27,16 @@ fun <T : Configurable> Specification<T>.update(config: Config, action: T.() -> U
 /**
  * Wrap a configuration using static meta as default
  */
-fun <T : Configurable> Specification<T>.wrap(config: Config = Config(), default: Meta): T = wrap(config) { default[it] }
+fun <T : Configurable> Specification<T>.wrap(config: Config = Config(), default: Meta = Meta.EMPTY): T =
+    wrap(config) { default[it] }
 
 /**
  * Wrap a configuration using static meta as default
  */
-fun <T : Configurable> Specification<T>.wrap(default: Meta): T = wrap(Config()) { default[it] }
+fun <T : Configurable> Specification<T>.wrap(source: Meta): T {
+    val default = source.seal()
+    return wrap(source.asConfig(), default)
+}
 
 
 /**

dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/serializationUtils.kt

@@ -3,6 +3,8 @@ package hep.dataforge.meta
 import kotlinx.serialization.*
 import kotlinx.serialization.builtins.DoubleArraySerializer
 import kotlinx.serialization.builtins.serializer
+import kotlinx.serialization.json.Json
+import kotlinx.serialization.json.JsonConfiguration
 
 fun SerialDescriptorBuilder.boolean(name: String, isOptional: Boolean = false, vararg annotations: Annotation) =
     element(name, Boolean.serializer().descriptor, isOptional = isOptional, annotations = annotations.toList())
@@ -63,3 +65,5 @@ inline fun Encoder.encodeStructure(
     encoder.block()
     encoder.endStructure(desc)
 }
+
+val PRETTY_JSON = Json(JsonConfiguration(prettyPrint = true, useArrayPolymorphism = true))