A lot of minor fixes

dataforge-context/build.gradle.kts

@@ -12,20 +12,20 @@ kotlin {
             dependencies {
                 api(project(":dataforge-meta"))
                 api(kotlin("reflect"))
-                api("io.github.microutils:kotlin-logging-common:1.6.10")
+                api("io.github.microutils:kotlin-logging-common:1.7.2")
                 api("org.jetbrains.kotlinx:kotlinx-coroutines-core-common:$coroutinesVersion")
             }
         }
         val jvmMain by getting {
             dependencies {
-                api("io.github.microutils:kotlin-logging:1.6.10")
+                api("io.github.microutils:kotlin-logging:1.7.2")
                 api("ch.qos.logback:logback-classic:1.2.3")
                 api("org.jetbrains.kotlinx:kotlinx-coroutines-core:$coroutinesVersion")
             }
         }
         val jsMain by getting {
             dependencies {
-                api("io.github.microutils:kotlin-logging-js:1.6.10")
+                api("io.github.microutils:kotlin-logging-js:1.7.2")
                 api("org.jetbrains.kotlinx:kotlinx-coroutines-core-js:$coroutinesVersion")
             }
         }

dataforge-context/src/commonMain/kotlin/hep/dataforge/context/AbstractPlugin.kt

@@ -3,6 +3,7 @@ package hep.dataforge.context
 import hep.dataforge.meta.EmptyMeta
 import hep.dataforge.meta.Meta
 import hep.dataforge.names.Name
+import hep.dataforge.names.toName
 
 abstract class AbstractPlugin(override val meta: Meta = EmptyMeta) : Plugin {
     private var _context: Context? = null
@@ -19,4 +20,8 @@ abstract class AbstractPlugin(override val meta: Meta = EmptyMeta) : Plugin {
     }
 
     override fun provideTop(target: String): Map<Name, Any>  = emptyMap()
+
+    companion object{
+        fun <T: Named> Collection<T>.toMap(): Map<Name, T> = associate { it.name.toName() to it }
+    }
 }

dataforge-context/src/commonMain/kotlin/hep/dataforge/context/Context.kt

@@ -26,8 +26,10 @@ import kotlin.jvm.JvmName
  * Since plugins could contain mutable state, context has two states: active and inactive. No changes are allowed to active context.
  * @author Alexander Nozik
  */
-open class Context(final override val name: String, val parent: Context? = Global) : Named, MetaRepr, Provider,
+open class Context(
-    CoroutineScope {
+    final override val name: String,
+    val parent: Context? = Global
+) : Named, MetaRepr, Provider, CoroutineScope {
 
     private val config = Config()
 
@@ -60,10 +62,10 @@ open class Context(final override val name: String, val parent: Context? = Globa
     override val defaultTarget: String get() = Plugin.PLUGIN_TARGET
 
     override fun provideTop(target: String): Map<Name, Any> {
-        return when(target){
+        return when (target) {
             Value.TYPE -> properties.sequence().toMap()
             Plugin.PLUGIN_TARGET -> plugins.sequence(true).associateBy { it.name.toName() }
-            else-> emptyMap()
+            else -> emptyMap()
         }
     }
 
@@ -111,7 +113,7 @@ open class Context(final override val name: String, val parent: Context? = Globa
 fun Context.content(target: String): Map<Name, Any> = content<Any>(target)
 
 /**
- * A sequences of all objects provided by plugins with given target and type
+ * A map of all objects provided by plugins with given target and type
  */
 @JvmName("typedContent")
 inline fun <reified T : Any> Context.content(target: String): Map<Name, T> =

dataforge-data/src/commonMain/kotlin/hep/dataforge/data/Goal.kt

@@ -77,8 +77,8 @@ private class StaticGoalImpl<T>(override val scope: CoroutineScope, deferred: Co
  *
  * **Important:** Unlike regular deferred, the [Goal] is started lazily, so the actual calculation is called only when result is requested.
  */
-fun <R> CoroutineScope.createGoal(
+fun <R> CoroutineScope.goal(
-    dependencies: Collection<Goal<*>>,
+    dependencies: Collection<Goal<*>> = emptyList(),
     context: CoroutineContext = EmptyCoroutineContext,
     block: suspend CoroutineScope.() -> R
 ): Goal<R> {
@@ -102,7 +102,7 @@ fun <R> CoroutineScope.createGoal(
 fun <T, R> Goal<T>.pipe(
     context: CoroutineContext = EmptyCoroutineContext,
     block: suspend CoroutineScope.(T) -> R
-): Goal<R> = createGoal(listOf(this), context) { block(await()) }
+): Goal<R> = goal(listOf(this), context) { block(await()) }
 
 /**
  * Create a joining goal.
@@ -112,7 +112,7 @@ fun <T, R> Collection<Goal<T>>.join(
     scope: CoroutineScope = first(),
     context: CoroutineContext = EmptyCoroutineContext,
     block: suspend CoroutineScope.(Collection<T>) -> R
-): Goal<R> = scope.createGoal(this, context) {
+): Goal<R> = scope.goal(this, context) {
     block(map { it.await() })
 }
 
@@ -126,6 +126,6 @@ fun <K, T, R> Map<K, Goal<T>>.join(
     scope: CoroutineScope = values.first(),
     context: CoroutineContext = EmptyCoroutineContext,
     block: suspend CoroutineScope.(Map<K, T>) -> R
-): Goal<R> = scope.createGoal(this.values, context) {
+): Goal<R> = scope.goal(this.values, context) {
     block(mapValues { it.value.await() })
 }

dataforge-io/build.gradle.kts

@@ -14,7 +14,7 @@ kotlin {
     sourceSets {
         val commonMain by getting{
             dependencies {
-                api(project(":dataforge-meta"))
+                api(project(":dataforge-context"))
             }
         }
         val jsMain by getting{

dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Envelope.kt

@@ -1,9 +1,12 @@
 package hep.dataforge.io
 
+import hep.dataforge.io.EnvelopeFormat.Companion.ENVELOPE_FORMAT_TYPE
 import hep.dataforge.meta.Meta
 import hep.dataforge.meta.get
 import hep.dataforge.meta.string
+import hep.dataforge.provider.Type
 import kotlinx.io.core.Input
+import kotlinx.io.core.Output
 
 interface Envelope {
     val meta: Meta
@@ -46,4 +49,17 @@ val Envelope.dataType: String? get() = meta[Envelope.ENVELOPE_DATA_TYPE_KEY].str
  */
 val Envelope.description: String? get() = meta[Envelope.ENVELOPE_DESCRIPTION_KEY].string
 
-typealias EnvelopeFormat = IOFormat<Envelope>
+data class PartialEnvelope(val meta: Meta, val dataOffset: UInt, val dataSize: ULong?)
+
+@Type(ENVELOPE_FORMAT_TYPE)
+interface EnvelopeFormat : IOFormat<Envelope>{
+    fun readPartial(input: Input): PartialEnvelope
+
+    fun Output.writeEnvelope(envelope: Envelope, format: MetaFormat)
+
+    override fun Output.writeObject(obj: Envelope) = writeEnvelope(obj, JsonMetaFormat)
+
+    companion object {
+        const val ENVELOPE_FORMAT_TYPE = "envelopeFormat"
+    }
+}

dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOPlugin.kt

@@ -0,0 +1,39 @@
+package hep.dataforge.io
+
+import hep.dataforge.context.AbstractPlugin
+import hep.dataforge.context.PluginFactory
+import hep.dataforge.context.PluginTag
+import hep.dataforge.context.content
+import hep.dataforge.meta.Meta
+import hep.dataforge.names.Name
+import hep.dataforge.names.asName
+import kotlin.reflect.KClass
+
+class IOPlugin(meta: Meta) : AbstractPlugin(meta) {
+    override val tag: PluginTag get() = Companion.tag
+
+    val metaFormats by lazy {
+        context.content<MetaFormat>(MetaFormat.META_FORMAT_TYPE).values
+    }
+
+    fun metaFormat(key: Short): MetaFormat? = metaFormats.find { it.key == key }
+    fun metaFormat(name: String): MetaFormat? = metaFormats.find { it.name == name }
+
+    override fun provideTop(target: String): Map<Name, Any> {
+        return when (target) {
+            MetaFormat.META_FORMAT_TYPE -> internalMetaFormats
+            EnvelopeFormat.ENVELOPE_FORMAT_TYPE -> mapOf(
+                TaggedEnvelopeFormat.VERSION.asName() to TaggedEnvelopeFormat(metaFormats)
+            )
+            else -> super.provideTop(target)
+        }
+    }
+
+    companion object : PluginFactory<IOPlugin> {
+        private val internalMetaFormats = listOf(JsonMetaFormat, BinaryMetaFormat).toMap()
+
+        override val tag: PluginTag = PluginTag("io", group = PluginTag.DATAFORGE_GROUP)
+        override val type: KClass<out IOPlugin> = IOPlugin::class
+        override fun invoke(meta: Meta): IOPlugin = IOPlugin(meta)
+    }
+}

dataforge-io/src/commonMain/kotlin/hep/dataforge/io/MetaFormat.kt

@@ -1,14 +1,18 @@
 package hep.dataforge.io
 
+import hep.dataforge.context.Named
 import hep.dataforge.descriptors.NodeDescriptor
+import hep.dataforge.io.MetaFormat.Companion.META_FORMAT_TYPE
 import hep.dataforge.meta.Meta
+import hep.dataforge.provider.Type
 import kotlinx.io.core.*
 
 /**
  * A format for meta serialization
  */
-interface MetaFormat : IOFormat<Meta> {
+@Type(META_FORMAT_TYPE)
-    val name: String
+interface MetaFormat : IOFormat<Meta>, Named {
+    override val name: String
     val key: Short
 
     override fun Output.writeObject(obj: Meta) {
@@ -17,8 +21,12 @@ interface MetaFormat : IOFormat<Meta> {
 
     override fun Input.readObject(): Meta = readMeta(null)
 
-    fun Output.writeMeta(meta: Meta, descriptor: NodeDescriptor?)
+    fun Output.writeMeta(meta: Meta, descriptor: NodeDescriptor? = null)
-    fun Input.readMeta(descriptor: NodeDescriptor?): Meta
+    fun Input.readMeta(descriptor: NodeDescriptor? = null): Meta
+
+    companion object{
+        const val META_FORMAT_TYPE = "metaFormat"
+    }
 }
 
 fun Meta.toString(format: MetaFormat = JsonMetaFormat): String = buildPacket {

dataforge-io/src/commonMain/kotlin/hep/dataforge/io/MetaSerializer.kt

@@ -3,15 +3,29 @@ package hep.dataforge.io
 import hep.dataforge.meta.Config
 import hep.dataforge.meta.Meta
 import hep.dataforge.meta.toConfig
-import kotlinx.serialization.Decoder
+import hep.dataforge.names.Name
-import kotlinx.serialization.Encoder
+import hep.dataforge.names.toName
-import kotlinx.serialization.KSerializer
+import kotlinx.serialization.*
-import kotlinx.serialization.SerialDescriptor
+import kotlinx.serialization.internal.StringDescriptor
 import kotlinx.serialization.json.JsonObjectSerializer
 
+@Serializer(Name::class)
+object NameSerializer : KSerializer<Name> {
+    override val descriptor: SerialDescriptor = StringDescriptor
+
+    override fun deserialize(decoder: Decoder): Name {
+        return decoder.decodeString().toName()
+    }
+
+    override fun serialize(encoder: Encoder, obj: Name) {
+        encoder.encodeString(obj.toString())
+    }
+}
+
 /**
  * Serialized for meta
  */
+@Serializer(Meta::class)
 object MetaSerializer : KSerializer<Meta> {
     override val descriptor: SerialDescriptor = JsonObjectSerializer.descriptor
 
@@ -25,7 +39,8 @@ object MetaSerializer : KSerializer<Meta> {
     }
 }
 
-object ConfigSerializer: KSerializer<Config>{
+@Serializer(Config::class)
+object ConfigSerializer : KSerializer<Config> {
     override val descriptor: SerialDescriptor = JsonObjectSerializer.descriptor
 
     override fun deserialize(decoder: Decoder): Config {

dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt

@@ -2,14 +2,10 @@ package hep.dataforge.io
 
 import kotlinx.io.core.*
 
-@ExperimentalUnsignedTypes
+class TaggedEnvelopeFormat(val metaFormats: Collection<MetaFormat>) : EnvelopeFormat {
-class TaggedEnvelopeFormat(
-    val metaFormats: Collection<MetaFormat>,
-    val outputMetaFormat: MetaFormat = metaFormats.first()
-) : EnvelopeFormat {
 
-    override fun Output.writeObject(obj: Envelope) {
+    override fun Output.writeEnvelope(envelope: Envelope, format: MetaFormat) {
-        write(obj, this, outputMetaFormat)
+        write(this, envelope, format)
     }
 
     /**
@@ -20,6 +16,7 @@ class TaggedEnvelopeFormat(
      */
     override fun Input.readObject(): Envelope = read(this, metaFormats)
 
+    override fun readPartial(input: Input): PartialEnvelope = Companion.readPartial(input, metaFormats)
 
     private data class Tag(
         val metaFormatKey: Short,
@@ -28,9 +25,10 @@ class TaggedEnvelopeFormat(
     )
 
     companion object {
-        private const val VERSION = "DF03"
+        const val VERSION = "DF03"
         private const val START_SEQUENCE = "#~"
         private const val END_SEQUENCE = "~#\r\n"
+        private const val TAG_SIZE = 26u
 
         private fun Tag.toBytes(): ByteReadPacket = buildPacket(24) {
             writeText(START_SEQUENCE)
@@ -66,12 +64,24 @@ class TaggedEnvelopeFormat(
             return SimpleEnvelope(meta, ArrayBinary(dataBytes))
         }
 
-        fun write(obj: Envelope, out: Output, metaFormat: MetaFormat) {
+        fun readPartial(input: Input, metaFormats: Collection<MetaFormat>): PartialEnvelope {
-            val metaBytes = metaFormat.writeBytes(obj.meta)
+            val tag = input.readTag()
-            val tag = Tag(metaFormat.key, metaBytes.size.toUInt(), obj.data?.size ?: 0.toULong())
+
+            val metaFormat = metaFormats.find { it.key == tag.metaFormatKey }
+                ?: error("Meta format with key ${tag.metaFormatKey} not found")
+
+            val metaPacket = ByteReadPacket(input.readBytes(tag.metaSize.toInt()))
+            val meta = metaFormat.run { metaPacket.readObject() }
+
+            return PartialEnvelope(meta, TAG_SIZE + tag.metaSize, tag.dataSize)
+        }
+
+        fun write(out: Output, envelope: Envelope, metaFormat: MetaFormat) {
+            val metaBytes = metaFormat.writeBytes(envelope.meta)
+            val tag = Tag(metaFormat.key, metaBytes.size.toUInt(), envelope.data?.size ?: 0.toULong())
             out.writePacket(tag.toBytes())
             out.writeFully(metaBytes)
-            obj.data?.read { copyTo(out) }
+            envelope.data?.read { copyTo(out) }
         }
     }
 }

dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileBinary.kt

@@ -7,15 +7,13 @@ import java.nio.file.Files
 import java.nio.file.Path
 import java.nio.file.StandardOpenOption
 
-class FileBinary(val path: Path, private val offset: Int = 0) : RandomAccessBinary {
+class FileBinary(val path: Path, private val offset: UInt = 0u, size: ULong? = null) : RandomAccessBinary {
-
-    override val size: ULong
-        get() = (Files.size(path) - offset).toULong()
 
+    override val size: ULong = size ?: (Files.size(path).toULong() - offset).toULong()
 
     override fun <R> read(from: UInt, size: UInt, block: Input.() -> R): R {
         FileChannel.open(path, StandardOpenOption.READ).use {
-            val buffer = it.map(FileChannel.MapMode.READ_ONLY, (from.toLong() + offset), size.toLong())
+            val buffer = it.map(FileChannel.MapMode.READ_ONLY, (from + offset).toLong(), size.toLong())
             return ByteReadPacket(buffer).block()
         }
     }

dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileEnvelope.kt

@@ -0,0 +1,24 @@
+package hep.dataforge.io
+
+import hep.dataforge.meta.Meta
+import kotlinx.io.nio.asInput
+import java.nio.file.Files
+import java.nio.file.Path
+import java.nio.file.StandardOpenOption
+
+class FileEnvelope internal constructor(val path: Path, val format: EnvelopeFormat) : Envelope {
+    //TODO do not like this constructor. Hope to replace it later
+
+    private val partialEnvelope: PartialEnvelope
+
+    init {
+        val input = Files.newByteChannel(path, StandardOpenOption.READ).asInput()
+        partialEnvelope = format.readPartial(input)
+    }
+
+    override val meta: Meta get() = partialEnvelope.meta
+
+    override val data: Binary? = FileBinary(path, partialEnvelope.dataOffset, partialEnvelope.dataSize)
+}
+
+fun Path.readEnvelope(format: EnvelopeFormat) = FileEnvelope(this,format)

dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Laminate.kt

@@ -17,7 +17,7 @@ class Laminate(layers: List<Meta>) : Meta {
         }
     }
 
-    constructor(vararg layers: Meta) : this(layers.asList())
+    constructor(vararg layers: Meta?) : this(layers.filterNotNull())
 
     override val items: Map<NameToken, MetaItem<Meta>>
         get() = layers.map { it.items.keys }.flatten().associateWith { key ->

dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MetaBuilder.kt

@@ -8,7 +8,7 @@ import hep.dataforge.values.Value
  * DSL builder for meta. Is not intended to store mutable state
  */
 class MetaBuilder : AbstractMutableMeta<MetaBuilder>() {
-    override fun wrapNode(meta: Meta): MetaBuilder = meta.builder()
+    override fun wrapNode(meta: Meta): MetaBuilder = if (meta is MetaBuilder) meta else meta.builder()
     override fun empty(): MetaBuilder = MetaBuilder()
 
     infix fun String.to(value: Any) {

dataforge-meta/src/commonMain/kotlin/hep/dataforge/names/Name.kt

@@ -51,32 +51,55 @@ data class NameToken(val body: String, val index: String = "") {
         if (body.isEmpty()) error("Syntax error: Name token body is empty")
     }
 
+    private fun String.escape() =
+        replace("\\", "\\\\")
+            .replace(".", "\\.")
+            .replace("[", "\\[")
+            .replace("]", "\\]")
+
     override fun toString(): String = if (hasIndex()) {
-        "$body[$index]"
+        "${body.escape()}[$index]"
     } else {
-        body
+        body.escape()
     }
 
     fun hasIndex() = index.isNotEmpty()
 }
 
+/**
+ * Convert a [String] to name parsing it and extracting name tokens and index syntax.
+ * This operation is rather heavy so it should be used with care in high performance code.
+ */
 fun String.toName(): Name {
     if (isBlank()) return EmptyName
     val tokens = sequence {
         var bodyBuilder = StringBuilder()
         var queryBuilder = StringBuilder()
         var bracketCount: Int = 0
+        var escape: Boolean = false
         fun queryOn() = bracketCount > 0
 
-        asSequence().forEach {
+        for (it in this@toName) {
-            if (queryOn()) {
+            when {
-                when (it) {
+                escape -> {
-                    '[' -> bracketCount++
+                    if (queryOn()) {
-                    ']' -> bracketCount--
+                        queryBuilder.append(it)
+                    } else {
+                        bodyBuilder.append(it)
+                    }
+                    escape = false
                 }
-                if (queryOn()) queryBuilder.append(it)
+                it == '\\' -> {
-            } else {
+                    escape = true
-                when (it) {
+                }
+                queryOn() -> {
+                    when (it) {
+                        '[' -> bracketCount++
+                        ']' -> bracketCount--
+                    }
+                    if (queryOn()) queryBuilder.append(it)
+                }
+                else -> when (it) {
                     '.' -> {
                         yield(NameToken(bodyBuilder.toString(), queryBuilder.toString()))
                         bodyBuilder = StringBuilder()
@@ -96,6 +119,14 @@ fun String.toName(): Name {
     return Name(tokens.toList())
 }
 
+/**
+ * Convert the [String] to a [Name] by simply wrapping it in a single name token without parsing.
+ * The input string could contain dots and braces, but they are just escaped, not parsed.
+ */
+fun String.asName(): Name {
+    return NameToken(this).asName()
+}
+
 operator fun NameToken.plus(other: Name): Name = Name(listOf(this) + other.tokens)
 
 operator fun Name.plus(other: Name): Name = Name(this.tokens + other.tokens)

dataforge-meta/src/commonTest/kotlin/hep/dataforge/names/NameTest.kt

@@ -1,6 +1,9 @@
 package hep.dataforge.names
 
-import kotlin.test.*
+import kotlin.test.Test
+import kotlin.test.assertEquals
+import kotlin.test.assertFalse
+import kotlin.test.assertTrue
 
 class NameTest {
     @Test
@@ -25,4 +28,14 @@ class NameTest {
         assertTrue { name1.endsWith(name3) }
         assertFalse { name1.startsWith(name3) }
     }
+
+    @Test
+    fun escapeTest(){
+        val escapedName = "token\\.one.token2".toName()
+        val unescapedName = "token\\.one.token2".asName()
+
+        assertEquals(2, escapedName.length)
+        assertEquals(1, unescapedName.length)
+        assertEquals(escapedName, escapedName.toString().toName())
+    }
 }

dataforge-workspace/build.gradle.kts

@@ -10,6 +10,7 @@ kotlin {
             dependencies {
                 api(project(":dataforge-context"))
                 api(project(":dataforge-data"))
+                api(project(":dataforge-output"))
             }
         }
     }

dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/fileData.kt

@@ -0,0 +1,72 @@
+package hep.dataforge.workspace
+
+import hep.dataforge.context.Context
+import hep.dataforge.data.Data
+import hep.dataforge.data.goal
+import hep.dataforge.descriptors.NodeDescriptor
+import hep.dataforge.io.IOFormat
+import hep.dataforge.io.JsonMetaFormat
+import hep.dataforge.io.MetaFormat
+import hep.dataforge.meta.EmptyMeta
+import hep.dataforge.meta.Meta
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.coroutineScope
+import kotlinx.coroutines.withContext
+import kotlinx.io.nio.asInput
+import kotlinx.io.nio.asOutput
+import java.nio.file.Files
+import java.nio.file.Path
+import java.nio.file.StandardOpenOption
+import kotlin.reflect.KClass
+
+/**
+ * Read meta from file in a given [format]
+ */
+suspend fun Path.readMeta(format: MetaFormat, descriptor: NodeDescriptor? = null): Meta {
+    return withContext(Dispatchers.IO) {
+        format.run {
+            Files.newByteChannel(this@readMeta, StandardOpenOption.READ)
+                .asInput()
+                .readMeta(descriptor)
+        }
+    }
+}
+
+/**
+ * Write meta to file in a given [format]
+ */
+suspend fun Meta.write(path: Path, format: MetaFormat, descriptor: NodeDescriptor? = null) {
+    withContext(Dispatchers.IO) {
+        format.run {
+            Files.newByteChannel(path, StandardOpenOption.WRITE, StandardOpenOption.CREATE_NEW)
+                .asOutput()
+                .writeMeta(this@write, descriptor)
+        }
+    }
+}
+
+suspend fun <T : Any> Context.readData(
+    type: KClass<out T>,
+    path: Path,
+    format: IOFormat<T>,
+    metaFile: Path = path.resolveSibling("${path.fileName}.meta"),
+    metaFileFormat: MetaFormat = JsonMetaFormat
+): Data<T> {
+    return coroutineScope {
+        val externalMeta = if (Files.exists(metaFile)) {
+            metaFile.readMeta(metaFileFormat)
+        } else {
+            null
+        }
+        val goal = goal {
+            withContext(Dispatchers.IO) {
+                format.run {
+                    Files.newByteChannel(path, StandardOpenOption.READ)
+                        .asInput()
+                        .readObject()
+                }
+            }
+        }
+        Data.of(type, goal, externalMeta ?: EmptyMeta)
+    }
+}