Data and DataNode file io update

build.gradle.kts

@@ -1,9 +1,9 @@
 plugins {
-    id("scientifik.mpp") version "0.1.7-dev" apply false
+    id("scientifik.mpp") version "0.1.7" apply false
-    id("scientifik.publish") version "0.1.7-dev" apply false
+    id("scientifik.publish") version "0.1.7" apply false
 }
 
-val dataforgeVersion by extra("0.1.4-dev-2")
+val dataforgeVersion by extra("0.1.4-dev-3")
 
 val bintrayRepo by extra("dataforge")
 val githubProject by extra("dataforge-core")

dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataNode.kt

@@ -140,7 +140,7 @@ private sealed class DataTreeBuilderItem<out T : Any> {
 /**
  * A builder for a DataTree.
  */
-class DataTreeBuilder<T : Any>(private val type: KClass<out T>) {
+class DataTreeBuilder<T : Any>(val type: KClass<out T>) {
     private val map = HashMap<NameToken, DataTreeBuilderItem<T>>()
 
     operator fun set(token: NameToken, node: DataTreeBuilder<out T>) {

dataforge-io/build.gradle.kts

@@ -5,8 +5,8 @@ plugins {
 description = "IO module"
 
 scientifik{
-    serialization = true
+    withSerialization()
-    io = true
+    withIO()
 }
 
 

dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Envelope.kt

@@ -26,7 +26,7 @@ interface Envelope {
         /**
          * Build a static envelope using provided builder
          */
-        fun build(block: EnvelopeBuilder.() -> Unit) = EnvelopeBuilder().apply(block).build()
+        operator fun invoke(block: EnvelopeBuilder.() -> Unit) = EnvelopeBuilder().apply(block).build()
     }
 }
 
@@ -97,6 +97,7 @@ class EnvelopeBuilder {
 
     var type by metaBuilder.string(key = Envelope.ENVELOPE_TYPE_KEY)
     var dataType by metaBuilder.string(key = Envelope.ENVELOPE_DATA_TYPE_KEY)
+    var dataID by metaBuilder.string(key = Envelope.ENVELOPE_DATA_ID_KEY)
     var description by metaBuilder.string(key = Envelope.ENVELOPE_DESCRIPTION_KEY)
 
     /**

dataforge-io/src/commonMain/kotlin/hep/dataforge/io/functions/RemoteFunctionClient.kt

@@ -12,7 +12,7 @@ class RemoteFunctionClient(override val context: Context, val responder: Respond
     private fun <T : Any> IOPlugin.encodeOne(
         meta: Meta,
         value: T
-    ): Envelope = Envelope.build {
+    ): Envelope = Envelope.invoke {
         meta(meta)
         type = REQUEST_TYPE
         data {
@@ -26,7 +26,7 @@ class RemoteFunctionClient(override val context: Context, val responder: Respond
     private fun <T : Any> IOPlugin.encodeMany(
         meta: Meta,
         values: List<T>
-    ): Envelope = Envelope.build {
+    ): Envelope = Envelope.invoke {
         meta(meta)
         type = REQUEST_TYPE
         meta {

dataforge-io/src/commonMain/kotlin/hep/dataforge/io/functions/RemoteFunctionServer.kt

@@ -40,7 +40,7 @@ class RemoteFunctionServer(
             input
         )
 
-        return Envelope.build {
+        return Envelope.invoke {
             meta {
                 meta(request.meta)
             }

dataforge-io/src/commonTest/kotlin/hep/dataforge/io/EnvelopeFormatTest.kt

@@ -5,7 +5,7 @@ import kotlin.test.assertEquals
 
 
 class EnvelopeFormatTest {
-    val envelope = Envelope.build {
+    val envelope = Envelope.invoke {
         type = "test.format"
         meta{
             "d" to 22.2

dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileBinary.kt

@@ -20,4 +20,6 @@ class FileBinary(val path: Path, private val offset: UInt = 0u, size: ULong? = n
             return ByteReadPacket(buffer).block()
         }
     }
-}
+}
+
+fun Path.asBinary(offset: UInt = 0u, size: ULong? = null): FileBinary = FileBinary(this, offset, size)

dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileEnvelope.kt

@@ -22,7 +22,7 @@ class FileEnvelope internal constructor(val path: Path, val format: EnvelopeForm
     override val data: Binary? = FileBinary(path, partialEnvelope.dataOffset, partialEnvelope.dataSize)
 }
 
-fun Path.readEnvelope(format: EnvelopeFormat) = FileEnvelope(this, format)
+fun Path.readEnvelope(format: EnvelopeFormat = TaggedEnvelopeFormat) = FileEnvelope(this, format)
 
 fun Path.writeEnvelope(
     envelope: Envelope,

dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/ioFormatsJVM.kt

@@ -1,5 +1,6 @@
 package hep.dataforge.io
 
+import hep.dataforge.descriptors.NodeDescriptor
 import hep.dataforge.io.functions.FunctionServer
 import hep.dataforge.io.functions.FunctionServer.Companion.FUNCTION_NAME_KEY
 import hep.dataforge.io.functions.FunctionServer.Companion.INPUT_FORMAT_KEY
@@ -7,6 +8,10 @@ import hep.dataforge.io.functions.FunctionServer.Companion.OUTPUT_FORMAT_KEY
 import hep.dataforge.meta.Meta
 import hep.dataforge.meta.buildMeta
 import hep.dataforge.names.Name
+import kotlinx.io.nio.asOutput
+import java.nio.file.Files
+import java.nio.file.Path
+import java.nio.file.StandardOpenOption
 import kotlin.reflect.KClass
 import kotlin.reflect.full.isSuperclassOf
 
@@ -31,4 +36,15 @@ inline fun <reified T : Any, reified R : Any> FunctionServer.function(
     val plugin = context.plugins.get<IOPlugin>() ?: error("IO plugin not loaded")
     val meta = plugin.generateFunctionMeta<T, R>(functionName)
     return function(meta)
+}
+
+/**
+ * Write meta to file in a given [format]
+ */
+fun Meta.write(path: Path, format: MetaFormat, descriptor: NodeDescriptor? = null) {
+    format.run {
+        Files.newByteChannel(path, StandardOpenOption.WRITE, StandardOpenOption.CREATE_NEW)
+            .asOutput()
+            .writeMeta(this@write, descriptor)
+    }
 }

dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/tcp/EnvelopeClient.kt

@@ -34,7 +34,7 @@ class EnvelopeClient(
     suspend fun close() {
         try {
             respond(
-                Envelope.build {
+                Envelope.invoke {
                     type = EnvelopeServer.SHUTDOWN_ENVELOPE_TYPE
                 }
             )

dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/FileEnvelopeTest.kt

@@ -0,0 +1,30 @@
+package hep.dataforge.io
+
+import java.nio.file.Files
+import kotlin.test.Test
+import kotlin.test.assertTrue
+
+
+class FileEnvelopeTest {
+    val envelope = Envelope {
+        meta {
+            "a" to "AAA"
+            "b" to 22.2
+        }
+        dataType = "hep.dataforge.test"
+        dataID = "myData" // добавил только что
+        data {
+            writeDouble(16.7)
+
+        }
+    }
+
+    @Test
+    fun testFileWriteRead() {
+        val tmpPath = Files.createTempFile("dataforge_test", ".df")
+        tmpPath.writeEnvelope(envelope)
+        println(tmpPath.toUri())
+        val restored: Envelope = tmpPath.readEnvelope()
+        assertTrue { envelope.contentEquals(restored) }
+    }
+}

dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/tcp/EnvelopeServerTest.kt

@@ -46,7 +46,7 @@ class EnvelopeServerTest {
     @Test
     fun doEchoTest() {
 
-        val request = Envelope.build {
+        val request = Envelope.invoke {
             type = "test.echo"
             meta {
                 "test.value" to 22

dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/fileData.kt

@@ -1,15 +1,16 @@
 package hep.dataforge.workspace
 
 import hep.dataforge.data.Data
+import hep.dataforge.data.DataNode
+import hep.dataforge.data.DataTreeBuilder
+import hep.dataforge.data.datum
 import hep.dataforge.descriptors.NodeDescriptor
 import hep.dataforge.io.*
 import hep.dataforge.meta.EmptyMeta
 import hep.dataforge.meta.Meta
 import kotlinx.coroutines.Dispatchers
-import kotlinx.coroutines.coroutineScope
 import kotlinx.coroutines.withContext
 import kotlinx.io.nio.asInput
-import kotlinx.io.nio.asOutput
 import java.nio.file.Files
 import java.nio.file.Path
 import java.nio.file.StandardOpenOption
@@ -18,69 +19,75 @@ import kotlin.reflect.KClass
 /**
  * Read meta from file in a given [format]
  */
-suspend fun Path.readMeta(format: MetaFormat, descriptor: NodeDescriptor? = null): Meta {
+fun Path.readMeta(format: MetaFormat, descriptor: NodeDescriptor? = null): Meta {
-    return withContext(Dispatchers.IO) {
+    return format.run {
-        format.run {
+        Files.newByteChannel(this@readMeta, StandardOpenOption.READ)
-            Files.newByteChannel(this@readMeta, StandardOpenOption.READ)
+            .asInput()
-                .asInput()
+            .readMeta(descriptor)
-                .readMeta(descriptor)
-        }
-    }
-}
-
-/**
- * Write meta to file in a given [format]
- */
-suspend fun Meta.write(path: Path, format: MetaFormat, descriptor: NodeDescriptor? = null) {
-    withContext(Dispatchers.IO) {
-        format.run {
-            Files.newByteChannel(path, StandardOpenOption.WRITE, StandardOpenOption.CREATE_NEW)
-                .asOutput()
-                .writeMeta(this@write, descriptor)
-        }
     }
 }
 
 /**
  * Read data with supported envelope format and binary format. If envelope format is null, then read binary directly from file.
+ * The operation is blocking since it must read meta header. The reading of envelope body is lazy
  * @param type explicit type of data read
  * @param format binary format
  * @param envelopeFormat the format of envelope. If null, file is read directly
  * @param metaFile the relative file for optional meta override
  * @param metaFileFormat the meta format for override
  */
-suspend fun <T : Any> Path.readData(
+fun <T : Any> Path.readData(
     type: KClass<out T>,
     format: IOFormat<T>,
     envelopeFormat: EnvelopeFormat? = null,
     metaFile: Path = resolveSibling("$fileName.meta"),
     metaFileFormat: MetaFormat = JsonMetaFormat
 ): Data<T> {
-    return coroutineScope {
+    val externalMeta = if (Files.exists(metaFile)) {
-        val externalMeta = if (Files.exists(metaFile)) {
+        metaFile.readMeta(metaFileFormat)
-            metaFile.readMeta(metaFileFormat)
+    } else {
-        } else {
+        null
-            null
+    }
-        }
+    return if (envelopeFormat == null) {
-        if (envelopeFormat == null) {
+        Data(type, externalMeta ?: EmptyMeta) {
-            Data(type, externalMeta ?: EmptyMeta) {
+            withContext(Dispatchers.IO) {
-                withContext(Dispatchers.IO) {
+                format.run {
-                    format.run {
+                    Files.newByteChannel(this@readData, StandardOpenOption.READ)
-                        Files.newByteChannel(this@readData, StandardOpenOption.READ)
+                        .asInput()
-                            .asInput()
+                        .readThis()
-                            .readThis()
-                    }
                 }
             }
-        } else {
+        }
-            withContext(Dispatchers.IO) {
+    } else {
-                readEnvelope(envelopeFormat).let {
+        readEnvelope(envelopeFormat).let {
-                    if (externalMeta == null) {
+            if (externalMeta == null) {
-                        it
+                it
-                    } else {
+            } else {
-                        it.withMetaLayers(externalMeta)
+                it.withMetaLayers(externalMeta)
-                    }
+            }
-                }.toData(type, format)
+        }.toData(type, format)
+    }
+}
+
+fun <T : Any> DataTreeBuilder<T>.file(path: Path, format: IOFormat<T>, envelopeFormat: EnvelopeFormat? = null) {
+    val data = path.readData(type, format, envelopeFormat)
+    val name = path.fileName.toString().replace(".df", "")
+    datum(name, data)
+}
+
+/**
+ * Read the directory as a data node
+ */
+fun <T : Any> Path.readDataNode(
+    type: KClass<out T>,
+    format: IOFormat<T>,
+    envelopeFormat: EnvelopeFormat? = null
+): DataNode<T> {
+    if (!Files.isDirectory(this)) error("Provided path $this is not a directory")
+    return DataNode(type) {
+        Files.list(this@readDataNode).forEach { path ->
+            if (!path.fileName.toString().endsWith(".meta")) {
+                file(path, format, envelopeFormat)
             }
         }
     }