Binary and file data update

dataforge-data/build.gradle.kts

@@ -5,8 +5,6 @@ plugins {
 val coroutinesVersion: String = Scientifik.coroutinesVersion
 
 kotlin {
-    jvm()
-    js()
     sourceSets {
         val commonMain by getting{
             dependencies {

dataforge-io/build.gradle.kts

@@ -12,12 +12,12 @@ scientifik{
 
 kotlin {
     sourceSets {
-        val commonMain by getting{
+        commonMain{
             dependencies {
                 api(project(":dataforge-context"))
             }
         }
-        val jsMain by getting{
+        jsMain{
             dependencies{
                 api(npm("text-encoding"))
             }

dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Binary.kt

@@ -2,6 +2,7 @@ package hep.dataforge.io
 
 import kotlinx.io.core.ByteReadPacket
 import kotlinx.io.core.Input
+import kotlinx.io.core.buildPacket
 import kotlinx.io.core.readBytes
 
 /**
@@ -52,9 +53,29 @@ object EmptyBinary : RandomAccessBinary {
 }
 
 class ArrayBinary(val array: ByteArray) : RandomAccessBinary {
-    override val size: ULong = array.size.toULong()
+    override val size: ULong get() = array.size.toULong()
 
     override fun <R> read(from: UInt, size: UInt, block: Input.() -> R): R {
         return ByteReadPacket(array, from.toInt(), size.toInt()).block()
     }
+}
+
+/**
+ * Read given binary as object using given format
+ */
+fun <T : Any> Binary.readWith(format: IOFormat<T>): T = format.run {
+    read {
+        readThis()
+    }
+}
+
+/**
+ * Write this object to a binary
+ * TODO make a lazy binary that does not use intermediate array
+ */
+fun <T: Any> T.writeWith(format: IOFormat<T>): Binary = format.run{
+    val packet = buildPacket {
+        writeThis(this@writeWith)
+    }
+    return@run ArrayBinary(packet.readBytes())
 }

dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Envelope.kt

@@ -1,14 +1,9 @@
 package hep.dataforge.io
 
-import hep.dataforge.context.Named
+import hep.dataforge.meta.Laminate
-import hep.dataforge.io.EnvelopeFormat.Companion.ENVELOPE_FORMAT_TYPE
-import hep.dataforge.io.IOPlugin.Companion.defaultMetaFormats
 import hep.dataforge.meta.Meta
 import hep.dataforge.meta.get
 import hep.dataforge.meta.string
-import hep.dataforge.provider.Type
-import kotlinx.io.core.Input
-import kotlinx.io.core.Output
 
 interface Envelope {
     val meta: Meta
@@ -52,24 +47,20 @@ val Envelope.dataType: String? get() = meta[Envelope.ENVELOPE_DATA_TYPE_KEY].str
 val Envelope.description: String? get() = meta[Envelope.ENVELOPE_DESCRIPTION_KEY].string
 
 /**
- * A partially read envelope with meta, but without data
+ * An envelope, which wraps existing envelope and adds one or several additional layers of meta
  */
-@ExperimentalUnsignedTypes
+class ProxyEnvelope(val source: Envelope, vararg meta: Meta) : Envelope {
-data class PartialEnvelope(val meta: Meta, val dataOffset: UInt, val dataSize: ULong?)
+    override val meta: Laminate = Laminate(*meta, source.meta)
+    override val data: Binary? get() = source.data
+}
 
-@Type(ENVELOPE_FORMAT_TYPE)
+/**
-interface EnvelopeFormat : IOFormat<Envelope>, Named {
+ * Add few meta layers to existing envelope
-    fun Input.readPartial(formats: Collection<MetaFormat> = defaultMetaFormats): PartialEnvelope
+ */
-
+fun Envelope.withMetaLayers(vararg layers: Meta): Envelope {
-    fun Input.readEnvelope(formats: Collection<MetaFormat> = defaultMetaFormats): Envelope
+    return when {
-
+        layers.isEmpty() -> this
-    override fun Input.readThis(): Envelope  = readEnvelope()
+        this is ProxyEnvelope -> ProxyEnvelope(source, *layers, *this.meta.layers.toTypedArray())
-
+        else -> ProxyEnvelope(this, *layers)
-    fun Output.writeEnvelope(envelope: Envelope, format: MetaFormat = JsonMetaFormat)
-
-    override fun Output.writeThis(obj: Envelope) = writeEnvelope(obj)
-
-    companion object {
-        const val ENVELOPE_FORMAT_TYPE = "envelopeFormat"
     }
 }

dataforge-io/src/commonMain/kotlin/hep/dataforge/io/EnvelopeFormat.kt

@@ -0,0 +1,31 @@
+package hep.dataforge.io
+
+import hep.dataforge.context.Named
+import hep.dataforge.io.EnvelopeFormat.Companion.ENVELOPE_FORMAT_TYPE
+import hep.dataforge.meta.Meta
+import hep.dataforge.provider.Type
+import kotlinx.io.core.Input
+import kotlinx.io.core.Output
+
+/**
+ * A partially read envelope with meta, but without data
+ */
+@ExperimentalUnsignedTypes
+data class PartialEnvelope(val meta: Meta, val dataOffset: UInt, val dataSize: ULong?)
+
+@Type(ENVELOPE_FORMAT_TYPE)
+interface EnvelopeFormat : IOFormat<Envelope>, Named {
+    fun Input.readPartial(formats: Collection<MetaFormat> = IOPlugin.defaultMetaFormats): PartialEnvelope
+
+    fun Input.readEnvelope(formats: Collection<MetaFormat> = IOPlugin.defaultMetaFormats): Envelope
+
+    override fun Input.readThis(): Envelope = readEnvelope()
+
+    fun Output.writeEnvelope(envelope: Envelope, format: MetaFormat = JsonMetaFormat)
+
+    override fun Output.writeThis(obj: Envelope) = writeEnvelope(obj)
+
+    companion object {
+        const val ENVELOPE_FORMAT_TYPE = "envelopeFormat"
+    }
+}

dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileEnvelope.kt

@@ -39,4 +39,5 @@ fun Path.writeEnvelope(
     with(format) {
         output.writeEnvelope(envelope, metaFormat)
     }
-}
+}
+

dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/dataUtils.kt

@@ -0,0 +1,14 @@
+package hep.dataforge.workspace
+
+import hep.dataforge.data.Data
+import hep.dataforge.io.Envelope
+import hep.dataforge.io.IOFormat
+import hep.dataforge.io.readWith
+import kotlin.reflect.KClass
+
+/**
+ * Convert an [Envelope] to a data via given format. The actual parsing is done lazily.
+ */
+fun <T : Any> Envelope.toData(type: KClass<out T>, format: IOFormat<T>): Data<T> = Data(type, meta) {
+    data?.readWith(format) ?: error("Can't convert envelope without data to Data")
+}

dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/fileData.kt

@@ -1,11 +1,8 @@
 package hep.dataforge.workspace
 
-import hep.dataforge.context.Context
 import hep.dataforge.data.Data
 import hep.dataforge.descriptors.NodeDescriptor
-import hep.dataforge.io.IOFormat
+import hep.dataforge.io.*
-import hep.dataforge.io.JsonMetaFormat
-import hep.dataforge.io.MetaFormat
 import hep.dataforge.meta.EmptyMeta
 import hep.dataforge.meta.Meta
 import kotlinx.coroutines.Dispatchers
@@ -44,11 +41,19 @@ suspend fun Meta.write(path: Path, format: MetaFormat, descriptor: NodeDescripto
     }
 }
 
-suspend fun <T : Any> Context.readData(
+/**
+ * Read data with supported envelope format and binary format. If envelope format is null, then read binary directly from file.
+ * @param type explicit type of data read
+ * @param format binary format
+ * @param envelopeFormat the format of envelope. If null, file is read directly
+ * @param metaFile the relative file for optional meta override
+ * @param metaFileFormat the meta format for override
+ */
+suspend fun <T : Any> Path.readData(
     type: KClass<out T>,
-    path: Path,
     format: IOFormat<T>,
-    metaFile: Path = path.resolveSibling("${path.fileName}.meta"),
+    envelopeFormat: EnvelopeFormat? = null,
+    metaFile: Path = resolveSibling("$fileName.meta"),
     metaFileFormat: MetaFormat = JsonMetaFormat
 ): Data<T> {
     return coroutineScope {
@@ -57,14 +62,31 @@ suspend fun <T : Any> Context.readData(
         } else {
             null
         }
-        Data(type, externalMeta ?: EmptyMeta){
+        if (envelopeFormat == null) {
-            withContext(Dispatchers.IO) {
+            Data(type, externalMeta ?: EmptyMeta) {
-                format.run {
+                withContext(Dispatchers.IO) {
-                    Files.newByteChannel(path, StandardOpenOption.READ)
+                    format.run {
-                        .asInput()
+                        Files.newByteChannel(this@readData, StandardOpenOption.READ)
-                        .readThis()
+                            .asInput()
+                            .readThis()
+                    }
                 }
             }
+        } else {
+            withContext(Dispatchers.IO) {
+                readEnvelope(envelopeFormat).let {
+                    if (externalMeta == null) {
+                        it
+                    } else {
+                        it.withMetaLayers(externalMeta)
+                    }
+                }.toData(type, format)
+            }
         }
     }
-}
+}
+
+//suspend fun <T : Any> Path.writeData(
+//    data: Data<T>,
+//    format: IOFormat<T>,
+//    )