From 3c6bc157165368ad865a193c7fb87b58912ef8de Mon Sep 17 00:00:00 2001
From: Alexander Nozik <altavir@gmail.com>
Date: Fri, 15 Apr 2022 18:56:00 +0300
Subject: [PATCH 01/19] move to Kotlin 1.6.20 and KTor 2.0

---
 CHANGELOG.md                                         |  2 ++
 dataforge-io/build.gradle.kts                        |  4 +++-
 .../kotlin/space/kscience/dataforge/io/Binary.kt     |  2 +-
 .../space/kscience/dataforge/io/EnvelopeBuilder.kt   |  2 +-
 .../kotlin/space/kscience/dataforge/io/MetaFormat.kt |  4 ++--
 .../kscience/dataforge/io/TaggedEnvelopeFormat.kt    |  4 ++--
 .../kotlin/space/kscience/dataforge/io/ioMisc.kt     | 12 ++++--------
 .../space/kscience/dataforge/io/MetaFormatTest.kt    |  4 ++--
 .../space/kscience/dataforge/io/ioTestUtils.kt       |  2 +-
 .../space/kscience/dataforge/scripting/Builders.kt   |  2 +-
 gradle.properties                                    |  8 +-------
 11 files changed, 20 insertions(+), 26 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 599556c1..67e4521f 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -4,9 +4,11 @@
 ### Added
 - Add `specOrNull` delegate to meta and Scheme
 - Suspended read methods to the `Binary`
+- Static `Meta` to all `DataSet`s
 
 ### Changed
 - `Factory` is now `fun interface` and uses `build` instead of `invoke`. `invoke moved to an extension.
+- KTor 2.0
 
 ### Deprecated
 
diff --git a/dataforge-io/build.gradle.kts b/dataforge-io/build.gradle.kts
index 4bc7f276..f4bdd54a 100644
--- a/dataforge-io/build.gradle.kts
+++ b/dataforge-io/build.gradle.kts
@@ -1,3 +1,5 @@
+import ru.mipt.npm.gradle.KScienceVersions
+
 plugins {
     id("ru.mipt.npm.gradle.mpp")
     id("ru.mipt.npm.gradle.native")
@@ -18,7 +20,7 @@ kotlin {
         commonMain {
             dependencies {
                 api(project(":dataforge-context"))
-                api(npmlibs.ktor.io)
+                api("io.ktor:ktor-io:${KScienceVersions.ktorVersion}")
             }
         }
     }
diff --git a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/Binary.kt b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/Binary.kt
index 7473b783..e8b2ae16 100644
--- a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/Binary.kt
+++ b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/Binary.kt
@@ -62,7 +62,7 @@ internal class ByteArrayBinary(
 public fun ByteArray.asBinary(): Binary = ByteArrayBinary(this)
 
 /**
- * Produce a [buildByteArray] representing an exact copy of this [Binary]
+ * Produce a [ByteArray] representing an exact copy of this [Binary]
  */
 public fun Binary.toByteArray(): ByteArray = if (this is ByteArrayBinary) {
     array.copyOf() // TODO do we need to ensure data safety here?
diff --git a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/EnvelopeBuilder.kt b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/EnvelopeBuilder.kt
index a1afd15e..eedd2075 100644
--- a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/EnvelopeBuilder.kt
+++ b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/EnvelopeBuilder.kt
@@ -34,7 +34,7 @@ public class EnvelopeBuilder : Envelope {
      * Construct a data binary from given builder
      */
     public inline fun data(block: Output.() -> Unit) {
-        data = buildByteArray { block() }.asBinary()
+        data = ByteArray { block() }.asBinary()
     }
 
     public fun seal(): Envelope = SimpleEnvelope(metaBuilder.seal(), data)
diff --git a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/MetaFormat.kt b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/MetaFormat.kt
index 37b42704..46d7b540 100644
--- a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/MetaFormat.kt
+++ b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/MetaFormat.kt
@@ -54,9 +54,9 @@ public interface MetaFormatFactory : IOFormatFactory<Meta>, MetaFormat {
     }
 }
 
-public fun Meta.toString(format: MetaFormat): String = buildByteArray {
+public fun Meta.toString(format: MetaFormat): String = ByteArray {
     format.run {
-        writeObject(this@buildByteArray, this@toString)
+        writeObject(this@ByteArray, this@toString)
     }
 }.decodeToString()
 
diff --git a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/TaggedEnvelopeFormat.kt b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/TaggedEnvelopeFormat.kt
index c4d281b4..96eaddb6 100644
--- a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/TaggedEnvelopeFormat.kt
+++ b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/TaggedEnvelopeFormat.kt
@@ -26,7 +26,7 @@ public class TaggedEnvelopeFormat(
 //        ?: error("Meta format with key $metaFormatKey could not be resolved in $io")
 
 
-    private fun Tag.toBinary() = Binary(24) {
+    private fun Tag.toBinary() = Binary {
         writeRawString(START_SEQUENCE)
         writeRawString(version.name)
         writeShort(metaFormatKey)
@@ -149,7 +149,7 @@ public class TaggedEnvelopeFormat(
 
         override fun peekFormat(io: IOPlugin, binary: Binary): EnvelopeFormat? {
             return try {
-                binary.read{
+                binary.read {
                     val header = readRawString(6)
                     return@read when (header.substring(2..5)) {
                         VERSION.DF02.name -> TaggedEnvelopeFormat(io, VERSION.DF02)
diff --git a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/ioMisc.kt b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/ioMisc.kt
index 5895380e..acb16916 100644
--- a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/ioMisc.kt
+++ b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/ioMisc.kt
@@ -15,7 +15,6 @@ public fun Output.writeUtf8String(str: String) {
     writeFully(str.encodeToByteArray())
 }
 
-@OptIn(ExperimentalIoApi::class)
 public fun Input.readRawString(size: Int): String {
     return Charsets.ISO_8859_1.newDecoder().decodeExactBytes(this, size)
 }
@@ -24,14 +23,11 @@ public fun Input.readUtf8String(): String = readBytes().decodeToString()
 
 public fun Input.readSafeUtf8Line(): String = readUTF8Line() ?: error("Line not found")
 
-public inline fun buildByteArray(expectedSize: Int = 16, block: Output.() -> Unit): ByteArray {
-    val builder = BytePacketBuilder(expectedSize)
-    builder.block()
-    return builder.build().readBytes()
-}
+public inline fun ByteArray(block: Output.() -> Unit): ByteArray =
+    buildPacket(block).readBytes()
 
-public inline fun Binary(expectedSize: Int = 16, block: Output.() -> Unit): Binary =
-    buildByteArray(expectedSize, block).asBinary()
+public inline fun Binary(block: Output.() -> Unit): Binary =
+    ByteArray(block).asBinary()
 
 /**
  * View section of a [Binary] as an independent binary
diff --git a/dataforge-io/src/commonTest/kotlin/space/kscience/dataforge/io/MetaFormatTest.kt b/dataforge-io/src/commonTest/kotlin/space/kscience/dataforge/io/MetaFormatTest.kt
index d142e746..f231801f 100644
--- a/dataforge-io/src/commonTest/kotlin/space/kscience/dataforge/io/MetaFormatTest.kt
+++ b/dataforge-io/src/commonTest/kotlin/space/kscience/dataforge/io/MetaFormatTest.kt
@@ -8,8 +8,8 @@ import kotlin.test.Test
 import kotlin.test.assertEquals
 
 
-fun Meta.toByteArray(format: MetaFormat = JsonMetaFormat) = buildByteArray {
-    format.writeObject(this@buildByteArray, this@toByteArray)
+fun Meta.toByteArray(format: MetaFormat = JsonMetaFormat) = ByteArray {
+    format.writeObject(this@ByteArray, this@toByteArray)
 }
 
 fun MetaFormat.fromByteArray(packet: ByteArray): Meta {
diff --git a/dataforge-io/src/commonTest/kotlin/space/kscience/dataforge/io/ioTestUtils.kt b/dataforge-io/src/commonTest/kotlin/space/kscience/dataforge/io/ioTestUtils.kt
index e99dcbd5..8eeb526b 100644
--- a/dataforge-io/src/commonTest/kotlin/space/kscience/dataforge/io/ioTestUtils.kt
+++ b/dataforge-io/src/commonTest/kotlin/space/kscience/dataforge/io/ioTestUtils.kt
@@ -4,7 +4,7 @@ import io.ktor.utils.io.core.ByteReadPacket
 import io.ktor.utils.io.core.use
 
 
-fun <T : Any> IOFormat<T>.writeToByteArray(obj: T): ByteArray = buildByteArray {
+fun <T : Any> IOFormat<T>.writeToByteArray(obj: T): ByteArray = ByteArray {
     writeObject(this, obj)
 }
 fun <T : Any> IOFormat<T>.readFromByteArray(array: ByteArray): T = ByteReadPacket(array).use {
diff --git a/dataforge-scripting/src/jvmMain/kotlin/space/kscience/dataforge/scripting/Builders.kt b/dataforge-scripting/src/jvmMain/kotlin/space/kscience/dataforge/scripting/Builders.kt
index 19c6e640..89750159 100644
--- a/dataforge-scripting/src/jvmMain/kotlin/space/kscience/dataforge/scripting/Builders.kt
+++ b/dataforge-scripting/src/jvmMain/kotlin/space/kscience/dataforge/scripting/Builders.kt
@@ -27,7 +27,7 @@ public object Builders {
                 dependenciesFromCurrentContext(wholeClasspath = true)
             }
             hostConfiguration(defaultJvmScriptingHostConfiguration)
-            compilerOptions("-jvm-target", Runtime.version().feature().toString())
+            compilerOptions("-jvm-target", Runtime.version().feature().toString(),"-Xcontext-receivers")
         }
 
         val evaluationConfiguration = ScriptEvaluationConfiguration {
diff --git a/gradle.properties b/gradle.properties
index 9b925f8e..0de6ab36 100644
--- a/gradle.properties
+++ b/gradle.properties
@@ -1,13 +1,7 @@
-org.gradle.jvmargs=-XX:MaxMetaspaceSize=1G
 org.gradle.parallel=true
 
 kotlin.code.style=official
 
-#kotlin.mpp.enableGranularSourceSetsMetadata=true
-#kotlin.native.enableDependencyPropagation=false
 kotlin.mpp.stability.nowarn=true
 
-publishing.github=false
-publishing.sonatype=false
-
-toolsVersion=0.11.1-kotlin-1.6.10
+toolsVersion=0.11.4-kotlin-1.6.20

From e5000171f1eee4baaee349d13e47434afcf649cb Mon Sep 17 00:00:00 2001
From: Alexander Nozik <altavir@gmail.com>
Date: Sun, 17 Apr 2022 21:59:09 +0300
Subject: [PATCH 02/19] move to Kotlin 1.6.20 and KTor 2.0

---
 CHANGELOG.md                                  |  3 +-
 build.gradle.kts                              |  2 +-
 .../kscience/dataforge/actions/MapAction.kt   |  5 +-
 .../kscience/dataforge/actions/SplitAction.kt |  9 +-
 .../kscience/dataforge/data/ActiveDataTree.kt | 53 ++++-------
 .../kscience/dataforge/data/CachingAction.kt  |  5 +-
 .../space/kscience/dataforge/data/DataSet.kt  | 14 ++-
 .../kscience/dataforge/data/DataSetBuilder.kt | 95 ++++++++++---------
 .../space/kscience/dataforge/data/DataTree.kt | 50 +++++++---
 .../kscience/dataforge/data/GroupRule.kt      |  5 +-
 .../kscience/dataforge/data/StaticDataTree.kt | 33 ++++---
 .../kscience/dataforge/data/dataFilter.kt     | 10 ++
 .../kscience/dataforge/data/dataSetMeta.kt    | 20 ----
 .../kscience/dataforge/data/dataTransform.kt  |  7 +-
 .../dataforge/data/dataSetBuilderInContext.kt | 40 ++++++++
 .../kscience/dataforge/data/ActionsTest.kt    |  8 +-
 .../dataforge/data/DataTreeBuilderTest.kt     |  7 +-
 .../dataforge/workspace/taskBuilders.kt       |  2 +-
 .../kscience/dataforge/workspace/fileData.kt  | 16 ++--
 .../workspace/DataPropagationTest.kt          |  4 +-
 .../dataforge/workspace/FileDataTest.kt       |  2 +-
 .../workspace/SimpleWorkspaceTest.kt          | 17 ++--
 22 files changed, 234 insertions(+), 173 deletions(-)
 delete mode 100644 dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataSetMeta.kt
 create mode 100644 dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/dataSetBuilderInContext.kt

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 67e4521f..b677019b 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -4,11 +4,12 @@
 ### Added
 - Add `specOrNull` delegate to meta and Scheme
 - Suspended read methods to the `Binary`
-- Static `Meta` to all `DataSet`s
+- Synchronously accessed `meta` to all `DataSet`s
 
 ### Changed
 - `Factory` is now `fun interface` and uses `build` instead of `invoke`. `invoke moved to an extension.
 - KTor 2.0
+- DataTree `items` call is blocking.
 
 ### Deprecated
 
diff --git a/build.gradle.kts b/build.gradle.kts
index 93588123..78ca749b 100644
--- a/build.gradle.kts
+++ b/build.gradle.kts
@@ -4,7 +4,7 @@ plugins {
 
 allprojects {
     group = "space.kscience"
-    version = "0.5.3-dev-4"
+    version = "0.6.0-dev-1"
     repositories{
         mavenCentral()
     }
diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/MapAction.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/MapAction.kt
index 962d4b4d..e7001d07 100644
--- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/MapAction.kt
+++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/MapAction.kt
@@ -1,7 +1,6 @@
 package space.kscience.dataforge.actions
 
 import kotlinx.coroutines.CoroutineScope
-import kotlinx.coroutines.flow.collect
 import kotlinx.coroutines.flow.map
 import kotlinx.coroutines.launch
 import space.kscience.dataforge.data.*
@@ -79,13 +78,13 @@ internal class MapAction<in T : Any, out R : Any>(
         val flow = dataSet.flowData().map(::mapOne)
 
         return ActiveDataTree(outputType) {
-            populate(flow)
+            populateWith(flow)
             scope?.launch {
                 dataSet.updates.collect { name ->
                     //clear old nodes
                     remove(name)
                     //collect new items
-                    populate(dataSet.flowChildren(name).map(::mapOne))
+                    populateWith(dataSet.flowChildren(name).map(::mapOne))
                 }
             }
         }
diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/SplitAction.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/SplitAction.kt
index 51a47c4d..b55a0e08 100644
--- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/SplitAction.kt
+++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/SplitAction.kt
@@ -2,7 +2,10 @@ package space.kscience.dataforge.actions
 
 import kotlinx.coroutines.CoroutineScope
 import kotlinx.coroutines.FlowPreview
-import kotlinx.coroutines.flow.*
+import kotlinx.coroutines.flow.Flow
+import kotlinx.coroutines.flow.asFlow
+import kotlinx.coroutines.flow.flatMapConcat
+import kotlinx.coroutines.flow.map
 import kotlinx.coroutines.launch
 import space.kscience.dataforge.data.*
 import space.kscience.dataforge.meta.Laminate
@@ -72,13 +75,13 @@ internal class SplitAction<T : Any, R : Any>(
         }
 
         return ActiveDataTree<R>(outputType) {
-            populate(dataSet.flowData().flatMapConcat(transform = ::splitOne))
+            populateWith(dataSet.flowData().flatMapConcat(transform = ::splitOne))
             scope?.launch {
                 dataSet.updates.collect { name ->
                     //clear old nodes
                     remove(name)
                     //collect new items
-                    populate(dataSet.flowChildren(name).flatMapConcat(transform = ::splitOne))
+                    populateWith(dataSet.flowChildren(name).flatMapConcat(transform = ::splitOne))
                 }
             }
         }
diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/ActiveDataTree.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/ActiveDataTree.kt
index 8fdc01ab..e403b625 100644
--- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/ActiveDataTree.kt
+++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/ActiveDataTree.kt
@@ -1,9 +1,6 @@
 package space.kscience.dataforge.data
 
-import kotlinx.coroutines.CoroutineScope
-import kotlinx.coroutines.Job
 import kotlinx.coroutines.flow.*
-import kotlinx.coroutines.launch
 import kotlinx.coroutines.sync.Mutex
 import kotlinx.coroutines.sync.withLock
 import space.kscience.dataforge.meta.*
@@ -12,7 +9,7 @@ import kotlin.reflect.KType
 import kotlin.reflect.typeOf
 
 /**
- * A mutable [DataTree.Companion.active]. It
+ * A mutable [DataTree].
  */
 public class ActiveDataTree<T : Any>(
     override val dataType: KType,
@@ -20,20 +17,17 @@ public class ActiveDataTree<T : Any>(
     private val mutex = Mutex()
     private val treeItems = HashMap<NameToken, DataTreeItem<T>>()
 
-    override suspend fun items(): Map<NameToken, DataTreeItem<T>> = mutex.withLock {
-        treeItems.filter { !it.key.body.startsWith("@") }
-    }
+    override val items: Map<NameToken, DataTreeItem<T>>
+        get() = treeItems.filter { !it.key.body.startsWith("@") }
 
     private val _updates = MutableSharedFlow<Name>()
 
     override val updates: Flow<Name>
         get() = _updates
 
-    private suspend fun remove(token: NameToken) {
-        mutex.withLock {
-            if (treeItems.remove(token) != null) {
-                _updates.emit(token.asName())
-            }
+    private suspend fun remove(token: NameToken) = mutex.withLock {
+        if (treeItems.remove(token) != null) {
+            _updates.emit(token.asName())
         }
     }
 
@@ -42,10 +36,8 @@ public class ActiveDataTree<T : Any>(
         (getItem(name.cutLast()).tree as? ActiveDataTree)?.remove(name.lastOrNull()!!)
     }
 
-    private suspend fun set(token: NameToken, data: Data<T>) {
-        mutex.withLock {
-            treeItems[token] = DataTreeItem.Leaf(data)
-        }
+    private suspend fun set(token: NameToken, data: Data<T>) = mutex.withLock {
+        treeItems[token] = DataTreeItem.Leaf(data)
     }
 
     private suspend fun getOrCreateNode(token: NameToken): ActiveDataTree<T> =
@@ -56,15 +48,13 @@ public class ActiveDataTree<T : Any>(
                 }
             }
 
-    private suspend fun getOrCreateNode(name: Name): ActiveDataTree<T> {
-        return when (name.length) {
-            0 -> this
-            1 -> getOrCreateNode(name.firstOrNull()!!)
-            else -> getOrCreateNode(name.firstOrNull()!!).getOrCreateNode(name.cutFirst())
-        }
+    private suspend fun getOrCreateNode(name: Name): ActiveDataTree<T> = when (name.length) {
+        0 -> this
+        1 -> getOrCreateNode(name.firstOrNull()!!)
+        else -> getOrCreateNode(name.firstOrNull()!!).getOrCreateNode(name.cutFirst())
     }
 
-    override suspend fun emit(name: Name, data: Data<T>?) {
+    override suspend fun data(name: Name, data: Data<T>?) {
         if (data == null) {
             remove(name)
         } else {
@@ -77,14 +67,10 @@ public class ActiveDataTree<T : Any>(
         _updates.emit(name)
     }
 
-    /**
-     * Copy given data set and mirror its changes to this [ActiveDataTree] in [this@setAndObserve]. Returns an update [Job]
-     */
-    public fun CoroutineScope.setAndObserve(name: Name, dataSet: DataSet<T>): Job = launch {
-        emit(name, dataSet)
-        dataSet.updates.collect { nameInBranch ->
-            emit(name + nameInBranch, dataSet.getData(nameInBranch))
-        }
+    override suspend fun meta(name: Name, meta: Meta) {
+        val item = getItem(name)
+        if(item is DataTreeItem.Leaf) error("TODO: Can't change meta of existing leaf item.")
+        data(name + DataTree.META_ITEM_NAME_TOKEN, Data.empty(meta))
     }
 }
 
@@ -106,13 +92,12 @@ public suspend inline fun <reified T : Any> ActiveDataTree(
     crossinline block: suspend ActiveDataTree<T>.() -> Unit,
 ): ActiveDataTree<T> = ActiveDataTree<T>(typeOf<T>()).apply { block() }
 
-
 public suspend inline fun <reified T : Any> ActiveDataTree<T>.emit(
     name: Name,
     noinline block: suspend ActiveDataTree<T>.() -> Unit,
-): Unit = emit(name, ActiveDataTree(typeOf<T>(), block))
+): Unit = node(name, ActiveDataTree(typeOf<T>(), block))
 
 public suspend inline fun <reified T : Any> ActiveDataTree<T>.emit(
     name: String,
     noinline block: suspend ActiveDataTree<T>.() -> Unit,
-): Unit = emit(Name.parse(name), ActiveDataTree(typeOf<T>(), block))
+): Unit = node(Name.parse(name), ActiveDataTree(typeOf<T>(), block))
diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/CachingAction.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/CachingAction.kt
index 6685d29c..5e7b62bf 100644
--- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/CachingAction.kt
+++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/CachingAction.kt
@@ -3,7 +3,6 @@ package space.kscience.dataforge.data
 import kotlinx.coroutines.CoroutineScope
 import kotlinx.coroutines.coroutineScope
 import kotlinx.coroutines.flow.Flow
-import kotlinx.coroutines.flow.collect
 import space.kscience.dataforge.actions.Action
 import space.kscience.dataforge.meta.Meta
 import space.kscience.dataforge.names.Name
@@ -37,14 +36,14 @@ public abstract class CachingAction<in T : Any, out R : Any>(
         scope: CoroutineScope?,
     ): DataSet<R> = ActiveDataTree<R>(outputType) {
         coroutineScope {
-            populate(transform(dataSet, meta))
+            populateWith(transform(dataSet, meta))
         }
         scope?.let {
             dataSet.updates.collect {
                 //clear old nodes
                 remove(it)
                 //collect new items
-                populate(scope.transform(dataSet, meta, it))
+                populateWith(scope.transform(dataSet, meta, it))
                 //FIXME if the target is data, updates are fired twice
             }
         }
diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSet.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSet.kt
index 64f1f521..1f319668 100644
--- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSet.kt
+++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSet.kt
@@ -15,6 +15,11 @@ public interface DataSet<out T : Any> {
      */
     public val dataType: KType
 
+    /**
+     * Meta-data associated with this node. If no meta is provided, returns [Meta.EMPTY].
+     */
+    public val meta: Meta
+
     /**
      * Traverse this provider or its child. The order is not guaranteed.
      */
@@ -25,6 +30,7 @@ public interface DataSet<out T : Any> {
      */
     public suspend fun getData(name: Name): Data<T>?
 
+
     /**
      * Get a snapshot of names of top level children of given node. Empty if node does not exist or is a leaf.
      */
@@ -40,6 +46,7 @@ public interface DataSet<out T : Any> {
          */
         public val EMPTY: DataSet<Nothing> = object : DataSet<Nothing> {
             override val dataType: KType = TYPE_OF_NOTHING
+            override val meta: Meta get() = Meta.EMPTY
 
             //private val nothing: Nothing get() = error("this is nothing")
 
@@ -65,9 +72,10 @@ public val <T : Any> DataSet<T>.updates: Flow<Name> get() = if (this is ActiveDa
 /**
  * Flow all data nodes with names starting with [branchName]
  */
-public fun <T : Any> DataSet<T>.flowChildren(branchName: Name): Flow<NamedData<T>> = this@flowChildren.flowData().filter {
-    it.name.startsWith(branchName)
-}
+public fun <T : Any> DataSet<T>.flowChildren(branchName: Name): Flow<NamedData<T>> =
+    this@flowChildren.flowData().filter {
+        it.name.startsWith(branchName)
+    }
 
 /**
  * Start computation for all goals in data node and return a job for the whole node
diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSetBuilder.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSetBuilder.kt
index ea8e3e38..0c13f2c6 100644
--- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSetBuilder.kt
+++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSetBuilder.kt
@@ -2,11 +2,11 @@ package space.kscience.dataforge.data
 
 import kotlinx.coroutines.coroutineScope
 import kotlinx.coroutines.flow.Flow
-import kotlinx.coroutines.flow.collect
 import space.kscience.dataforge.meta.Meta
 import space.kscience.dataforge.meta.MutableMeta
 import space.kscience.dataforge.misc.DFExperimental
 import space.kscience.dataforge.names.Name
+import space.kscience.dataforge.names.isEmpty
 import space.kscience.dataforge.names.plus
 import kotlin.reflect.KType
 
@@ -18,12 +18,12 @@ public interface DataSetBuilder<in T : Any> {
      */
     public suspend fun remove(name: Name)
 
-    public suspend fun emit(name: Name, data: Data<T>?)
+    public suspend fun data(name: Name, data: Data<T>?)
 
     /**
      * Set a current state of given [dataSet] into a branch [name]. Does not propagate updates
      */
-    public suspend fun emit(name: Name, dataSet: DataSet<T>) {
+    public suspend fun node(name: Name, dataSet: DataSet<T>) {
         //remove previous items
         if (name != Name.EMPTY) {
             remove(name)
@@ -31,27 +31,29 @@ public interface DataSetBuilder<in T : Any> {
 
         //Set new items
         dataSet.flowData().collect {
-            emit(name + it.name, it.data)
+            data(name + it.name, it.data)
         }
     }
 
     /**
-     * Append data to node
+     * Set meta for the given node
      */
-    public suspend infix fun String.put(data: Data<T>): Unit = emit(Name.parse(this), data)
+    public suspend fun meta(name: Name, meta: Meta)
 
-    /**
-     * Append node
-     */
-    public suspend infix fun String.put(dataSet: DataSet<T>): Unit = emit(Name.parse(this), dataSet)
-
-    /**
-     * Build and append node
-     */
-    public suspend infix fun String.put(block: suspend DataSetBuilder<T>.() -> Unit): Unit = emit(Name.parse(this), block)
 }
 
-private class SubSetBuilder<in T : Any>(
+/**
+ * Define meta in this [DataSet]
+ */
+public suspend fun <T : Any> DataSetBuilder<T>.meta(value: Meta): Unit = meta(Name.EMPTY, value)
+
+/**
+ * Define meta in this [DataSet]
+ */
+public suspend fun <T : Any> DataSetBuilder<T>.meta(mutableMeta: MutableMeta.() -> Unit): Unit = meta(Meta(mutableMeta))
+
+@PublishedApi
+internal class SubSetBuilder<in T : Any>(
     private val parent: DataSetBuilder<T>,
     private val branch: Name,
 ) : DataSetBuilder<T> {
@@ -61,33 +63,42 @@ private class SubSetBuilder<in T : Any>(
         parent.remove(branch + name)
     }
 
-    override suspend fun emit(name: Name, data: Data<T>?) {
-        parent.emit(branch + name, data)
+    override suspend fun data(name: Name, data: Data<T>?) {
+        parent.data(branch + name, data)
     }
 
-    override suspend fun emit(name: Name, dataSet: DataSet<T>) {
-        parent.emit(branch + name, dataSet)
+    override suspend fun node(name: Name, dataSet: DataSet<T>) {
+        parent.node(branch + name, dataSet)
+    }
+
+    override suspend fun meta(name: Name, meta: Meta) {
+        parent.meta(branch + name, meta)
     }
 }
 
-public suspend fun <T : Any> DataSetBuilder<T>.emit(name: Name, block: suspend DataSetBuilder<T>.() -> Unit) {
-    SubSetBuilder(this, name).apply { block() }
+public suspend inline fun <T : Any> DataSetBuilder<T>.node(
+    name: Name,
+    crossinline block: suspend DataSetBuilder<T>.() -> Unit,
+) {
+    if (name.isEmpty()) block() else SubSetBuilder(this, name).block()
 }
 
 
-public suspend fun <T : Any> DataSetBuilder<T>.emit(name: String, data: Data<T>) {
-    emit(Name.parse(name), data)
+public suspend fun <T : Any> DataSetBuilder<T>.data(name: String, value: Data<T>) {
+    data(Name.parse(name), value)
 }
 
-public suspend fun <T : Any> DataSetBuilder<T>.emit(name: String, set: DataSet<T>) {
-    this.emit(Name.parse(name), set)
+public suspend fun <T : Any> DataSetBuilder<T>.node(name: String, set: DataSet<T>) {
+    node(Name.parse(name), set)
 }
 
-public suspend fun <T : Any> DataSetBuilder<T>.emit(name: String, block: suspend DataSetBuilder<T>.() -> Unit): Unit =
-    this@emit.emit(Name.parse(name), block)
+public suspend inline fun <T : Any> DataSetBuilder<T>.node(
+    name: String,
+    crossinline block: suspend DataSetBuilder<T>.() -> Unit,
+): Unit = node(Name.parse(name), block)
 
-public suspend fun <T : Any> DataSetBuilder<T>.emit(data: NamedData<T>) {
-    emit(data.name, data.data)
+public suspend fun <T : Any> DataSetBuilder<T>.set(value: NamedData<T>) {
+    data(value.name, value.data)
 }
 
 /**
@@ -99,7 +110,7 @@ public suspend inline fun <reified T : Any> DataSetBuilder<T>.produce(
     noinline producer: suspend () -> T,
 ) {
     val data = Data(meta, block = producer)
-    emit(name, data)
+    data(name, data)
 }
 
 public suspend inline fun <reified T : Any> DataSetBuilder<T>.produce(
@@ -108,7 +119,7 @@ public suspend inline fun <reified T : Any> DataSetBuilder<T>.produce(
     noinline producer: suspend () -> T,
 ) {
     val data = Data(meta, block = producer)
-    emit(name, data)
+    data(name, data)
 }
 
 /**
@@ -117,36 +128,34 @@ public suspend inline fun <reified T : Any> DataSetBuilder<T>.produce(
 public suspend inline fun <reified T : Any> DataSetBuilder<T>.static(
     name: String,
     data: T,
-    meta: Meta = Meta.EMPTY
-): Unit =
-    emit(name, Data.static(data, meta))
+    meta: Meta = Meta.EMPTY,
+): Unit = data(name, Data.static(data, meta))
 
 public suspend inline fun <reified T : Any> DataSetBuilder<T>.static(
     name: Name,
     data: T,
-    meta: Meta = Meta.EMPTY
-): Unit =
-    emit(name, Data.static(data, meta))
+    meta: Meta = Meta.EMPTY,
+): Unit = data(name, Data.static(data, meta))
 
 public suspend inline fun <reified T : Any> DataSetBuilder<T>.static(
     name: String,
     data: T,
     mutableMeta: MutableMeta.() -> Unit,
-): Unit = emit(Name.parse(name), Data.static(data, Meta(mutableMeta)))
+): Unit = data(Name.parse(name), Data.static(data, Meta(mutableMeta)))
 
 /**
  * Update data with given node data and meta with node meta.
  */
 @DFExperimental
-public suspend fun <T : Any> DataSetBuilder<T>.populate(tree: DataSet<T>): Unit = coroutineScope {
+public suspend fun <T : Any> DataSetBuilder<T>.populateFrom(tree: DataSet<T>): Unit = coroutineScope {
     tree.flowData().collect {
         //TODO check if the place is occupied
-        emit(it.name, it.data)
+        data(it.name, it.data)
     }
 }
 
-public suspend fun <T : Any> DataSetBuilder<T>.populate(flow: Flow<NamedData<T>>) {
+public suspend fun <T : Any> DataSetBuilder<T>.populateWith(flow: Flow<NamedData<T>>) {
     flow.collect {
-        emit(it.name, it.data)
+        data(it.name, it.data)
     }
 }
diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataTree.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataTree.kt
index ea9d67e2..ca51ce2e 100644
--- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataTree.kt
+++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataTree.kt
@@ -4,6 +4,7 @@ import kotlinx.coroutines.flow.Flow
 import kotlinx.coroutines.flow.emitAll
 import kotlinx.coroutines.flow.flow
 import kotlinx.coroutines.flow.map
+import space.kscience.dataforge.meta.Meta
 import space.kscience.dataforge.misc.Type
 import space.kscience.dataforge.names.*
 import kotlin.collections.component1
@@ -11,8 +12,16 @@ import kotlin.collections.component2
 import kotlin.reflect.KType
 
 public sealed class DataTreeItem<out T : Any> {
-    public class Node<out T : Any>(public val tree: DataTree<T>) : DataTreeItem<T>()
-    public class Leaf<out T : Any>(public val data: Data<T>) : DataTreeItem<T>()
+
+    public abstract val meta: Meta
+
+    public class Node<out T : Any>(public val tree: DataTree<T>) : DataTreeItem<T>() {
+        override val meta: Meta get() = tree.meta
+    }
+
+    public class Leaf<out T : Any>(public val data: Data<T>) : DataTreeItem<T>() {
+        override val meta: Meta get() = data.meta
+    }
 }
 
 public val <T : Any> DataTreeItem<T>.type: KType
@@ -28,13 +37,15 @@ public val <T : Any> DataTreeItem<T>.type: KType
 public interface DataTree<out T : Any> : DataSet<T> {
 
     /**
-     * Children items of this [DataTree] provided asynchronously
+     * Top-level children items of this [DataTree]
      */
-    public suspend fun items(): Map<NameToken, DataTreeItem<T>>
+    public val items: Map<NameToken, DataTreeItem<T>>
+
+    override val meta: Meta get() = items[META_ITEM_NAME_TOKEN]?.meta ?: Meta.EMPTY
 
     override fun flowData(): Flow<NamedData<T>> = flow {
-        items().forEach { (token, childItem: DataTreeItem<T>) ->
-            if(!token.body.startsWith("@")) {
+        items.forEach { (token, childItem: DataTreeItem<T>) ->
+            if (!token.body.startsWith("@")) {
                 when (childItem) {
                     is DataTreeItem.Leaf -> emit(childItem.data.named(token.asName()))
                     is DataTreeItem.Node -> emitAll(childItem.tree.flowData().map { it.named(token + it.name) })
@@ -44,28 +55,33 @@ public interface DataTree<out T : Any> : DataSet<T> {
     }
 
     override suspend fun listTop(prefix: Name): List<Name> =
-        getItem(prefix).tree?.items()?.keys?.map { prefix + it } ?: emptyList()
+        getItem(prefix).tree?.items?.keys?.map { prefix + it } ?: emptyList()
 
     override suspend fun getData(name: Name): Data<T>? = when (name.length) {
         0 -> null
-        1 -> items()[name.firstOrNull()!!].data
-        else -> items()[name.firstOrNull()!!].tree?.getData(name.cutFirst())
+        1 -> items[name.firstOrNull()!!].data
+        else -> items[name.firstOrNull()!!].tree?.getData(name.cutFirst())
     }
 
     public companion object {
         public const val TYPE: String = "dataTree"
+
+        /**
+         * A name token used to designate tree node meta
+         */
+        public val META_ITEM_NAME_TOKEN: NameToken = NameToken("@meta")
     }
 }
 
-public suspend fun <T: Any> DataSet<T>.getData(name: String): Data<T>? = getData(Name.parse(name))
+public suspend fun <T : Any> DataSet<T>.getData(name: String): Data<T>? = getData(Name.parse(name))
 
 /**
  * Get a [DataTreeItem] with given [name] or null if the item does not exist
  */
-public tailrec suspend fun <T : Any> DataTree<T>.getItem(name: Name): DataTreeItem<T>? = when (name.length) {
+public tailrec fun <T : Any> DataTree<T>.getItem(name: Name): DataTreeItem<T>? = when (name.length) {
     0 -> DataTreeItem.Node(this)
-    1 -> items()[name.firstOrNull()]
-    else -> items()[name.firstOrNull()!!].tree?.getItem(name.cutFirst())
+    1 -> items[name.firstOrNull()]
+    else -> items[name.firstOrNull()!!].tree?.getItem(name.cutFirst())
 }
 
 public val <T : Any> DataTreeItem<T>?.tree: DataTree<T>? get() = (this as? DataTreeItem.Node<T>)?.tree
@@ -75,7 +91,7 @@ public val <T : Any> DataTreeItem<T>?.data: Data<T>? get() = (this as? DataTreeI
  * Flow of all children including nodes
  */
 public fun <T : Any> DataTree<T>.itemFlow(): Flow<Pair<Name, DataTreeItem<T>>> = flow {
-    items().forEach { (head, item) ->
+    items.forEach { (head, item) ->
         emit(head.asName() to item)
         if (item is DataTreeItem.Node) {
             val subSequence = item.tree.itemFlow()
@@ -92,5 +108,9 @@ public fun <T : Any> DataTree<T>.itemFlow(): Flow<Pair<Name, DataTreeItem<T>>> =
 public fun <T : Any> DataTree<T>.branch(branchName: Name): DataTree<T> = object : DataTree<T> {
     override val dataType: KType get() = this@branch.dataType
 
-    override suspend fun items(): Map<NameToken, DataTreeItem<T>> = getItem(branchName).tree?.items() ?: emptyMap()
+    override val meta: Meta
+        get() = getItem(branchName)?.meta ?: Meta.EMPTY
+
+    override val items: Map<NameToken, DataTreeItem<T>>
+        get() = getItem(branchName).tree?.items ?: emptyMap()
 }
diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/GroupRule.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/GroupRule.kt
index d26cbfb1..c87f01d2 100644
--- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/GroupRule.kt
+++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/GroupRule.kt
@@ -16,7 +16,6 @@
 package space.kscience.dataforge.data
 
 import kotlinx.coroutines.CoroutineScope
-import kotlinx.coroutines.flow.collect
 import kotlinx.coroutines.launch
 import space.kscience.dataforge.meta.get
 import space.kscience.dataforge.meta.string
@@ -46,7 +45,7 @@ public interface GroupRule {
 
                 set.flowData().collect { data ->
                     val tagValue = data.meta[key]?.string ?: defaultTagValue
-                    map.getOrPut(tagValue) { ActiveDataTree(set.dataType) }.emit(data.name, data.data)
+                    map.getOrPut(tagValue) { ActiveDataTree(set.dataType) }.data(data.name, data.data)
                 }
 
                 scope.launch {
@@ -55,7 +54,7 @@ public interface GroupRule {
 
                         @Suppress("NULLABLE_EXTENSION_OPERATOR_WITH_SAFE_CALL_RECEIVER")
                         val tagValue = data?.meta?.get(key)?.string ?: defaultTagValue
-                        map.getOrPut(tagValue) { ActiveDataTree(set.dataType) }.emit(name, data)
+                        map.getOrPut(tagValue) { ActiveDataTree(set.dataType) }.data(name, data)
                     }
                 }
 
diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/StaticDataTree.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/StaticDataTree.kt
index e68c16e2..f37459dc 100644
--- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/StaticDataTree.kt
+++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/StaticDataTree.kt
@@ -1,7 +1,7 @@
 package space.kscience.dataforge.data
 
 import kotlinx.coroutines.coroutineScope
-import kotlinx.coroutines.flow.collect
+import space.kscience.dataforge.meta.Meta
 import space.kscience.dataforge.misc.DFExperimental
 import space.kscience.dataforge.names.*
 import kotlin.reflect.KType
@@ -12,15 +12,16 @@ internal class StaticDataTree<T : Any>(
     override val dataType: KType,
 ) : DataSetBuilder<T>, DataTree<T> {
 
-    private val items: MutableMap<NameToken, DataTreeItem<T>> = HashMap()
+    private val _items: MutableMap<NameToken, DataTreeItem<T>> = HashMap()
 
-    override suspend fun items(): Map<NameToken, DataTreeItem<T>> = items.filter { !it.key.body.startsWith("@") }
+    override val items: Map<NameToken, DataTreeItem<T>>
+        get() = _items.filter { !it.key.body.startsWith("@") }
 
     override suspend fun remove(name: Name) {
         when (name.length) {
             0 -> error("Can't remove root tree node")
-            1 -> items.remove(name.firstOrNull()!!)
-            else -> (items[name.firstOrNull()!!].tree as? StaticDataTree<T>)?.remove(name.cutFirst())
+            1 -> _items.remove(name.firstOrNull()!!)
+            else -> (_items[name.firstOrNull()!!].tree as? StaticDataTree<T>)?.remove(name.cutFirst())
         }
     }
 
@@ -28,8 +29,8 @@ internal class StaticDataTree<T : Any>(
         0 -> this
         1 -> {
             val itemName = name.firstOrNull()!!
-            (items[itemName].tree as? StaticDataTree<T>) ?: StaticDataTree<T>(dataType).also {
-                items[itemName] = DataTreeItem.Node(it)
+            (_items[itemName].tree as? StaticDataTree<T>) ?: StaticDataTree<T>(dataType).also {
+                _items[itemName] = DataTreeItem.Node(it)
             }
         }
         else -> getOrCreateNode(name.cutLast()).getOrCreateNode(name.lastOrNull()!!.asName())
@@ -40,25 +41,31 @@ internal class StaticDataTree<T : Any>(
         if (item == null) {
             remove(name)
         } else {
-            getOrCreateNode(name.cutLast()).items[name.lastOrNull()!!] = item
+            getOrCreateNode(name.cutLast())._items[name.lastOrNull()!!] = item
         }
     }
 
-    override suspend fun emit(name: Name, data: Data<T>?) {
+    override suspend fun data(name: Name, data: Data<T>?) {
         set(name, data?.let { DataTreeItem.Leaf(it) })
     }
 
-    override suspend fun emit(name: Name, dataSet: DataSet<T>) {
+    override suspend fun node(name: Name, dataSet: DataSet<T>) {
         if (dataSet is StaticDataTree) {
             set(name, DataTreeItem.Node(dataSet))
         } else {
             coroutineScope {
                 dataSet.flowData().collect {
-                    emit(name + it.name, it.data)
+                    data(name + it.name, it.data)
                 }
             }
         }
     }
+
+    override suspend fun meta(name: Name, meta: Meta) {
+        val item = getItem(name)
+        if(item is DataTreeItem.Leaf) TODO("Can't change meta of existing leaf item.")
+        data(name + DataTree.META_ITEM_NAME_TOKEN, Data.empty(meta))
+    }
 }
 
 @Suppress("FunctionName")
@@ -73,6 +80,6 @@ public suspend inline fun <reified T : Any> DataTree(
 ): DataTree<T> = DataTree(typeOf<T>(), block)
 
 @OptIn(DFExperimental::class)
-public suspend fun <T : Any> DataSet<T>.seal(): DataTree<T> = DataTree(dataType){
-    populate(this@seal)
+public suspend fun <T : Any> DataSet<T>.seal(): DataTree<T> = DataTree(dataType) {
+    populateFrom(this@seal)
 }
\ No newline at end of file
diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataFilter.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataFilter.kt
index a526cdd7..f9e4bb47 100644
--- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataFilter.kt
+++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataFilter.kt
@@ -4,6 +4,7 @@ import kotlinx.coroutines.flow.Flow
 import kotlinx.coroutines.flow.filter
 import kotlinx.coroutines.flow.map
 import kotlinx.coroutines.flow.mapNotNull
+import space.kscience.dataforge.meta.Meta
 import space.kscience.dataforge.misc.DFExperimental
 import space.kscience.dataforge.names.Name
 import space.kscience.dataforge.names.isEmpty
@@ -18,8 +19,11 @@ import kotlin.reflect.KType
 public fun <T : Any> DataSet<T>.filter(
     predicate: suspend (Name, Data<T>) -> Boolean,
 ): ActiveDataSet<T> = object : ActiveDataSet<T> {
+
     override val dataType: KType get() = this@filter.dataType
 
+    override val meta: Meta get() = this@filter.meta
+
     override fun flowData(): Flow<NamedData<T>> =
         this@filter.flowData().filter { predicate(it.name, it.data) }
 
@@ -38,8 +42,12 @@ public fun <T : Any> DataSet<T>.filter(
  */
 public fun <T : Any> DataSet<T>.withNamePrefix(prefix: Name): DataSet<T> = if (prefix.isEmpty()) this
 else object : ActiveDataSet<T> {
+
     override val dataType: KType get() = this@withNamePrefix.dataType
 
+    override val meta: Meta get() = this@withNamePrefix.meta
+
+
     override fun flowData(): Flow<NamedData<T>> = this@withNamePrefix.flowData().map { it.data.named(prefix + it.name) }
 
     override suspend fun getData(name: Name): Data<T>? =
@@ -56,6 +64,8 @@ public fun <T : Any> DataSet<T>.branch(branchName: Name): DataSet<T> = if (branc
 } else object : ActiveDataSet<T> {
     override val dataType: KType get() = this@branch.dataType
 
+    override val meta: Meta get() = this@branch.meta
+
     override fun flowData(): Flow<NamedData<T>> = this@branch.flowData().mapNotNull {
         it.name.removeHeadOrNull(branchName)?.let { name ->
             it.data.named(name)
diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataSetMeta.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataSetMeta.kt
deleted file mode 100644
index bdbbf3a8..00000000
--- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataSetMeta.kt
+++ /dev/null
@@ -1,20 +0,0 @@
-package space.kscience.dataforge.data
-
-import space.kscience.dataforge.meta.Meta
-import space.kscience.dataforge.meta.MutableMeta
-
-
-/**
- * Get a metadata node for this set if it is present
- */
-public suspend fun DataSet<*>.getMeta(): Meta? = getData(DataSet.META_KEY)?.meta
-
-/**
- * Add meta-data node to a [DataSet]
- */
-public suspend fun DataSetBuilder<*>.meta(meta: Meta): Unit = emit(DataSet.META_KEY, Data.empty(meta))
-
-/**
- * Add meta-data node to a [DataSet]
- */
-public suspend fun DataSetBuilder<*>.meta(mutableMeta: MutableMeta.() -> Unit): Unit = meta(Meta(mutableMeta))
\ No newline at end of file
diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataTransform.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataTransform.kt
index 14b90729..8c3332df 100644
--- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataTransform.kt
+++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataTransform.kt
@@ -1,6 +1,9 @@
 package space.kscience.dataforge.data
 
-import kotlinx.coroutines.flow.*
+import kotlinx.coroutines.flow.Flow
+import kotlinx.coroutines.flow.fold
+import kotlinx.coroutines.flow.map
+import kotlinx.coroutines.flow.toList
 import space.kscience.dataforge.meta.Meta
 import space.kscience.dataforge.meta.MutableMeta
 import space.kscience.dataforge.meta.seal
@@ -143,7 +146,7 @@ public suspend fun <T : Any, R : Any> DataSet<T>.map(
     metaTransform: MutableMeta.() -> Unit = {},
     block: suspend (T) -> R,
 ): DataTree<R> = DataTree<R>(outputType) {
-    populate(
+    populateWith(
         flowData().map {
             val newMeta = it.meta.toMutableMeta().apply(metaTransform).seal()
             Data(outputType, newMeta, coroutineContext, listOf(it)) {
diff --git a/dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/dataSetBuilderInContext.kt b/dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/dataSetBuilderInContext.kt
new file mode 100644
index 00000000..b105910e
--- /dev/null
+++ b/dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/dataSetBuilderInContext.kt
@@ -0,0 +1,40 @@
+package space.kscience.dataforge.data
+
+import kotlinx.coroutines.CoroutineScope
+import kotlinx.coroutines.Job
+import kotlinx.coroutines.launch
+import space.kscience.dataforge.names.Name
+import space.kscience.dataforge.names.plus
+
+
+/**
+ * Append data to node
+ */
+context(DataSetBuilder<T>) public suspend infix fun <T : Any> String.put(data: Data<T>): Unit =
+    data(Name.parse(this), data)
+
+/**
+ * Append node
+ */
+context(DataSetBuilder<T>) public suspend infix fun <T : Any> String.put(dataSet: DataSet<T>): Unit =
+    node(Name.parse(this), dataSet)
+
+/**
+ * Build and append node
+ */
+context(DataSetBuilder<T>) public suspend infix fun <T : Any> String.put(
+    block: suspend DataSetBuilder<T>.() -> Unit,
+): Unit = node(Name.parse(this), block)
+
+/**
+ * Copy given data set and mirror its changes to this [ActiveDataTree] in [this@setAndObserve]. Returns an update [Job]
+ */
+context(DataSetBuilder<T>) public fun <T : Any> CoroutineScope.setAndWatch(
+    name: Name,
+    dataSet: DataSet<T>,
+): Job = launch {
+    node(name, dataSet)
+    dataSet.updates.collect { nameInBranch ->
+        data(name + nameInBranch, dataSet.getData(nameInBranch))
+    }
+}
\ No newline at end of file
diff --git a/dataforge-data/src/jvmTest/kotlin/space/kscience/dataforge/data/ActionsTest.kt b/dataforge-data/src/jvmTest/kotlin/space/kscience/dataforge/data/ActionsTest.kt
index 4bbd9cd8..da7ec0d9 100644
--- a/dataforge-data/src/jvmTest/kotlin/space/kscience/dataforge/data/ActionsTest.kt
+++ b/dataforge-data/src/jvmTest/kotlin/space/kscience/dataforge/data/ActionsTest.kt
@@ -4,11 +4,12 @@ import kotlinx.coroutines.runBlocking
 import org.junit.jupiter.api.Test
 import space.kscience.dataforge.actions.Action
 import space.kscience.dataforge.actions.map
+import space.kscience.dataforge.misc.DFExperimental
 import kotlin.test.assertEquals
 
-@Suppress("EXPERIMENTAL_API_USAGE")
-class ActionsTest {
-    val data: DataTree<Int> = runBlocking {
+@OptIn(DFExperimental::class)
+internal class ActionsTest {
+    private val data: DataTree<Int> = runBlocking {
         DataTree {
             repeat(10) {
                 static(it.toString(), it)
@@ -32,6 +33,7 @@ class ActionsTest {
         val plusOne = Action.map<Int, Int> {
             result { it + 1 }
         }
+
         val datum = runBlocking {
             val result = plusOne.execute(data, scope = this)
             result.getData("1")?.await()
diff --git a/dataforge-data/src/jvmTest/kotlin/space/kscience/dataforge/data/DataTreeBuilderTest.kt b/dataforge-data/src/jvmTest/kotlin/space/kscience/dataforge/data/DataTreeBuilderTest.kt
index 4bdadece..3e8d716e 100644
--- a/dataforge-data/src/jvmTest/kotlin/space/kscience/dataforge/data/DataTreeBuilderTest.kt
+++ b/dataforge-data/src/jvmTest/kotlin/space/kscience/dataforge/data/DataTreeBuilderTest.kt
@@ -1,7 +1,6 @@
 package space.kscience.dataforge.data
 
 import kotlinx.coroutines.*
-import kotlinx.coroutines.flow.collect
 import space.kscience.dataforge.misc.DFExperimental
 import space.kscience.dataforge.names.asName
 import kotlin.test.Test
@@ -43,7 +42,7 @@ internal class DataTreeBuilderTest {
                 static("b", "b")
             }
             static("root", "root")
-            populate(updateData)
+            populateFrom(updateData)
         }
 
         runBlocking {
@@ -72,7 +71,7 @@ internal class DataTreeBuilderTest {
                     }
                 }
                 val rootNode = ActiveDataTree<Int> {
-                    setAndObserve("sub".asName(), subNode)
+                    setAndWatch("sub".asName(), subNode)
                 }
 
                 launch {
@@ -85,7 +84,7 @@ internal class DataTreeBuilderTest {
                 cancel()
             }
         } catch (t: Throwable) {
-            if (t !is CancellationException) throw  t
+            if (t !is CancellationException) throw t
         }
 
     }
diff --git a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/taskBuilders.kt b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/taskBuilders.kt
index 4744d415..9fd31d1c 100644
--- a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/taskBuilders.kt
+++ b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/taskBuilders.kt
@@ -38,7 +38,7 @@ public suspend inline fun <T : Any, reified R : Any> TaskResultBuilder<R>.pipeFr
             action(it, data.name, meta)
         }
 
-        emit(data.name, res)
+        data(data.name, res)
     }
 }
 
diff --git a/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/fileData.kt b/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/fileData.kt
index 90a69976..d758c45c 100644
--- a/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/fileData.kt
+++ b/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/fileData.kt
@@ -80,15 +80,15 @@ public suspend fun <T : Any> DataSetBuilder<T>.file(
             val data = readDataFile(path, formatResolver)
             val name = data.meta[Envelope.ENVELOPE_NAME_KEY].string
                 ?: path.fileName.toString().replace(".df", "")
-            emit(name, data)
+            data(name, data)
         }
     } else {
         //otherwise, read as directory
         plugin.run {
             val data = readDataDirectory(path, formatResolver)
-            val name = data.getMeta()?.get(Envelope.ENVELOPE_NAME_KEY).string
+            val name = data.meta[Envelope.ENVELOPE_NAME_KEY].string
                 ?: path.fileName.toString().replace(".df", "")
-            emit(name, data)
+            node(name, data)
         }
     }
 }
@@ -143,7 +143,7 @@ public suspend fun <T : Any> IOPlugin.writeDataDirectory(
         } else if (!Files.isDirectory(path)) {
             error("Can't write a node into file")
         }
-        tree.items().forEach { (token, item) ->
+        tree.items.forEach { (token, item) ->
             val childPath = path.resolve(token.toString())
             when (item) {
                 is DataTreeItem.Node -> {
@@ -159,10 +159,8 @@ public suspend fun <T : Any> IOPlugin.writeDataDirectory(
                 }
             }
         }
-        val treeMeta = tree.getMeta()
-        if (treeMeta != null) {
-            writeMetaFile(path, treeMeta, metaFormat ?: JsonMetaFormat)
-        }
+        val treeMeta = tree.meta
+        writeMetaFile(path, treeMeta, metaFormat ?: JsonMetaFormat)
     }
 }
 
@@ -192,7 +190,7 @@ private suspend fun <T : Any> ZipOutputStream.writeNode(
                 val entry = ZipEntry("$name/")
                 putNextEntry(entry)
                 closeEntry()
-                treeItem.tree.items().forEach { (token, item) ->
+                treeItem.tree.items.forEach { (token, item) ->
                     val childName = "$name/$token"
                     writeNode(childName, item, dataFormat, envelopeFormat)
                 }
diff --git a/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/DataPropagationTest.kt b/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/DataPropagationTest.kt
index d3f58a51..91a3ffd2 100644
--- a/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/DataPropagationTest.kt
+++ b/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/DataPropagationTest.kt
@@ -19,13 +19,13 @@ class DataPropagationTestPlugin : WorkspacePlugin() {
         val result: Data<Int> = selectedData.flowData().foldToData(0) { result, data ->
             result + data.await()
         }
-        emit("result", result)
+        data("result", result)
     }
 
 
     val singleData by task<Int> {
         workspace.data.select<Int>().getData("myData[12]")?.let {
-            emit("result", it)
+            data("result", it)
         }
     }
 
diff --git a/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/FileDataTest.kt b/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/FileDataTest.kt
index 2b78d238..4dbf3a85 100644
--- a/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/FileDataTest.kt
+++ b/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/FileDataTest.kt
@@ -22,7 +22,7 @@ import kotlin.test.assertEquals
 class FileDataTest {
     val dataNode = runBlocking {
         DataTree<String> {
-            emit("dir") {
+            node("dir") {
                 static("a", "Some string") {
                     "content" put "Some string"
                 }
diff --git a/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/SimpleWorkspaceTest.kt b/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/SimpleWorkspaceTest.kt
index 39d1d5a1..d0db9823 100644
--- a/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/SimpleWorkspaceTest.kt
+++ b/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/SimpleWorkspaceTest.kt
@@ -39,7 +39,7 @@ class SimpleWorkspaceTest {
         override val tag: PluginTag = PluginTag("test")
 
         val test by task<Any> {
-            populate(
+            populateFrom(
                 workspace.data.map {
                     it.also {
                         logger.info { "Test: $it" }
@@ -66,7 +66,7 @@ class SimpleWorkspaceTest {
 
         val filterOne by task<Int> {
             workspace.data.selectOne<Int>("myData[12]")?.let { source ->
-                emit(source.name, source.map { it })
+                data(source.name, source.map { it })
             }
         }
 
@@ -106,7 +106,7 @@ class SimpleWorkspaceTest {
                 val newData: Data<Int> = data.combine(linearData.getData(data.name)!!) { l, r ->
                     l + r
                 }
-                emit(data.name, newData)
+                data(data.name, newData)
             }
         }
 
@@ -115,7 +115,7 @@ class SimpleWorkspaceTest {
             val res = from(square).foldToData(0) { l, r ->
                 l + r.await()
             }
-            emit("sum", res)
+            data("sum", res)
         }
 
         val averageByGroup by task<Int> {
@@ -125,13 +125,13 @@ class SimpleWorkspaceTest {
                 l + r.await()
             }
 
-            emit("even", evenSum)
+            data("even", evenSum)
             val oddSum = workspace.data.filter { name, _ ->
                 name.toString().toInt() % 2 == 1
             }.select<Int>().foldToData(0) { l, r ->
                 l + r.await()
             }
-            emit("odd", oddSum)
+            data("odd", oddSum)
         }
 
         val delta by task<Int> {
@@ -141,7 +141,7 @@ class SimpleWorkspaceTest {
             val res = even.combine(odd) { l, r ->
                 l - r
             }
-            emit("res", res)
+            data("res", res)
         }
 
         val customPipe by task<Int> {
@@ -149,8 +149,7 @@ class SimpleWorkspaceTest {
                 val meta = data.meta.toMutableMeta().apply {
                     "newValue" put 22
                 }
-                emit(data.name + "new", data.map { (data.meta["value"].int ?: 0) + it })
-
+                data(data.name + "new", data.map { (data.meta["value"].int ?: 0) + it })
             }
         }
 

From 6b41163ed3367c8733b8e2b335b730121c49503c Mon Sep 17 00:00:00 2001
From: Alexander Nozik <altavir@gmail.com>
Date: Sun, 17 Apr 2022 22:21:11 +0300
Subject: [PATCH 03/19] Fix select.kt

---
 .../jvmMain/kotlin/space/kscience/dataforge/data/select.kt  | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/select.kt b/dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/select.kt
index 5e65cee7..df8d3c45 100644
--- a/dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/select.kt
+++ b/dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/select.kt
@@ -35,10 +35,12 @@ private fun <R : Any> Data<*>.castOrNull(type: KType): Data<R>? =
 public fun <R : Any> DataSet<*>.select(
     type: KType,
     namePattern: Name? = null,
-    filter: (name: Name, meta: Meta) -> Boolean = { _, _ -> true }
+    filter: (name: Name, meta: Meta) -> Boolean = { _, _ -> true },
 ): ActiveDataSet<R> = object : ActiveDataSet<R> {
     override val dataType = type
 
+    override val meta: Meta get() = this@select.meta
+
     private fun checkDatum(name: Name, datum: Data<*>): Boolean = datum.type.isSubtypeOf(type)
             && (namePattern == null || name.matches(namePattern))
             && filter(name, datum.meta)
@@ -65,7 +67,7 @@ public fun <R : Any> DataSet<*>.select(
  */
 public inline fun <reified R : Any> DataSet<*>.select(
     namePattern: Name? = null,
-    noinline filter: (name: Name, meta: Meta) -> Boolean = { _, _ -> true }
+    noinline filter: (name: Name, meta: Meta) -> Boolean = { _, _ -> true },
 ): DataSet<R> = select(typeOf<R>(), namePattern, filter)
 
 /**

From eaa9d40d60f9fbb90881f91f74ff6e72bd1c62f3 Mon Sep 17 00:00:00 2001
From: Alexander Nozik <altavir@gmail.com>
Date: Sun, 24 Apr 2022 09:19:14 +0300
Subject: [PATCH 04/19] Change  `suspend DataSet.getData` to `operator
 DataSet.get`

---
 CHANGELOG.md                                       |  1 +
 .../kscience/dataforge/actions/ReduceAction.kt     |  2 +-
 .../space/kscience/dataforge/data/DataSet.kt       |  6 +++---
 .../space/kscience/dataforge/data/DataTree.kt      |  6 +++---
 .../space/kscience/dataforge/data/GroupRule.kt     |  2 +-
 .../space/kscience/dataforge/data/dataFilter.kt    | 14 +++++++-------
 .../dataforge/data/dataSetBuilderInContext.kt      |  2 +-
 .../kotlin/space/kscience/dataforge/data/select.kt | 12 ++++++------
 .../kotlin/space/kscience/dataforge/io/fileIO.kt   |  5 ++---
 .../kscience/dataforge/workspace/TaskResult.kt     |  4 ++--
 .../kscience/dataforge/workspace/Workspace.kt      |  2 +-
 .../dataforge/workspace/SimpleWorkspaceTest.kt     |  2 +-
 12 files changed, 29 insertions(+), 29 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index b677019b..94aafcc1 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -10,6 +10,7 @@
 - `Factory` is now `fun interface` and uses `build` instead of `invoke`. `invoke moved to an extension.
 - KTor 2.0
 - DataTree `items` call is blocking.
+- DataSet `getData` is no longer suspended and renamed to `get`
 
 ### Deprecated
 
diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/ReduceAction.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/ReduceAction.kt
index 39f26dd6..126f1f46 100644
--- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/ReduceAction.kt
+++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/ReduceAction.kt
@@ -48,7 +48,7 @@ public class ReduceGroupBuilder<T : Any, R : Any>(
 
     public fun group(
         groupName: String,
-        filter: suspend (Name, Data<T>) -> Boolean,
+        filter: (Name, Data<T>) -> Boolean,
         action: JoinGroup<T, R>.() -> Unit,
     ) {
         groupRules += { source ->
diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSet.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSet.kt
index 1f319668..f4838c85 100644
--- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSet.kt
+++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSet.kt
@@ -28,7 +28,7 @@ public interface DataSet<out T : Any> {
     /**
      * Get data with given name.
      */
-    public suspend fun getData(name: Name): Data<T>?
+    public operator fun get(name: Name): Data<T>?
 
 
     /**
@@ -52,7 +52,7 @@ public interface DataSet<out T : Any> {
 
             override fun flowData(): Flow<NamedData<Nothing>> = emptyFlow()
 
-            override suspend fun getData(name: Name): Data<Nothing>? = null
+            override fun get(name: Name): Data<Nothing>? = null
         }
     }
 }
@@ -101,4 +101,4 @@ public suspend fun DataSet<*>.toMeta(): Meta = Meta {
     }
 }
 
-public val <T : Any> DataSet<T>.updatesWithData: Flow<NamedData<T>> get() = updates.mapNotNull { getData(it)?.named(it) }
\ No newline at end of file
+public val <T : Any> DataSet<T>.updatesWithData: Flow<NamedData<T>> get() = updates.mapNotNull { get(it)?.named(it) }
\ No newline at end of file
diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataTree.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataTree.kt
index ca51ce2e..8eee0113 100644
--- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataTree.kt
+++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataTree.kt
@@ -57,10 +57,10 @@ public interface DataTree<out T : Any> : DataSet<T> {
     override suspend fun listTop(prefix: Name): List<Name> =
         getItem(prefix).tree?.items?.keys?.map { prefix + it } ?: emptyList()
 
-    override suspend fun getData(name: Name): Data<T>? = when (name.length) {
+    override fun get(name: Name): Data<T>? = when (name.length) {
         0 -> null
         1 -> items[name.firstOrNull()!!].data
-        else -> items[name.firstOrNull()!!].tree?.getData(name.cutFirst())
+        else -> items[name.firstOrNull()!!].tree?.get(name.cutFirst())
     }
 
     public companion object {
@@ -73,7 +73,7 @@ public interface DataTree<out T : Any> : DataSet<T> {
     }
 }
 
-public suspend fun <T : Any> DataSet<T>.getData(name: String): Data<T>? = getData(Name.parse(name))
+public suspend fun <T : Any> DataSet<T>.getData(name: String): Data<T>? = get(Name.parse(name))
 
 /**
  * Get a [DataTreeItem] with given [name] or null if the item does not exist
diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/GroupRule.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/GroupRule.kt
index c87f01d2..beaa3da0 100644
--- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/GroupRule.kt
+++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/GroupRule.kt
@@ -50,7 +50,7 @@ public interface GroupRule {
 
                 scope.launch {
                     set.updates.collect { name ->
-                        val data = set.getData(name)
+                        val data = set.get(name)
 
                         @Suppress("NULLABLE_EXTENSION_OPERATOR_WITH_SAFE_CALL_RECEIVER")
                         val tagValue = data?.meta?.get(key)?.string ?: defaultTagValue
diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataFilter.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataFilter.kt
index f9e4bb47..46f07260 100644
--- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataFilter.kt
+++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataFilter.kt
@@ -17,7 +17,7 @@ import kotlin.reflect.KType
  * A stateless filtered [DataSet]
  */
 public fun <T : Any> DataSet<T>.filter(
-    predicate: suspend (Name, Data<T>) -> Boolean,
+    predicate: (Name, Data<T>) -> Boolean,
 ): ActiveDataSet<T> = object : ActiveDataSet<T> {
 
     override val dataType: KType get() = this@filter.dataType
@@ -27,12 +27,12 @@ public fun <T : Any> DataSet<T>.filter(
     override fun flowData(): Flow<NamedData<T>> =
         this@filter.flowData().filter { predicate(it.name, it.data) }
 
-    override suspend fun getData(name: Name): Data<T>? = this@filter.getData(name)?.takeIf {
+    override fun get(name: Name): Data<T>? = this@filter.get(name)?.takeIf {
         predicate(name, it)
     }
 
     override val updates: Flow<Name> = this@filter.updates.filter flowFilter@{ name ->
-        val theData = this@filter.getData(name) ?: return@flowFilter false
+        val theData = this@filter.get(name) ?: return@flowFilter false
         predicate(name, theData)
     }
 }
@@ -50,8 +50,8 @@ else object : ActiveDataSet<T> {
 
     override fun flowData(): Flow<NamedData<T>> = this@withNamePrefix.flowData().map { it.data.named(prefix + it.name) }
 
-    override suspend fun getData(name: Name): Data<T>? =
-        name.removeHeadOrNull(name)?.let { this@withNamePrefix.getData(it) }
+    override fun get(name: Name): Data<T>? =
+        name.removeHeadOrNull(name)?.let { this@withNamePrefix.get(it) }
 
     override val updates: Flow<Name> get() = this@withNamePrefix.updates.map { prefix + it }
 }
@@ -72,7 +72,7 @@ public fun <T : Any> DataSet<T>.branch(branchName: Name): DataSet<T> = if (branc
         }
     }
 
-    override suspend fun getData(name: Name): Data<T>? = this@branch.getData(branchName + name)
+    override fun get(name: Name): Data<T>? = this@branch.get(branchName + name)
 
     override val updates: Flow<Name> get() = this@branch.updates.mapNotNull { it.removeHeadOrNull(branchName) }
 }
@@ -80,5 +80,5 @@ public fun <T : Any> DataSet<T>.branch(branchName: Name): DataSet<T> = if (branc
 public fun <T : Any> DataSet<T>.branch(branchName: String): DataSet<T> = this@branch.branch(Name.parse(branchName))
 
 @DFExperimental
-public suspend fun <T : Any> DataSet<T>.rootData(): Data<T>? = getData(Name.EMPTY)
+public suspend fun <T : Any> DataSet<T>.rootData(): Data<T>? = get(Name.EMPTY)
 
diff --git a/dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/dataSetBuilderInContext.kt b/dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/dataSetBuilderInContext.kt
index b105910e..51bfa187 100644
--- a/dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/dataSetBuilderInContext.kt
+++ b/dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/dataSetBuilderInContext.kt
@@ -35,6 +35,6 @@ context(DataSetBuilder<T>) public fun <T : Any> CoroutineScope.setAndWatch(
 ): Job = launch {
     node(name, dataSet)
     dataSet.updates.collect { nameInBranch ->
-        data(name + nameInBranch, dataSet.getData(nameInBranch))
+        data(name + nameInBranch, dataSet.get(nameInBranch))
     }
 }
\ No newline at end of file
diff --git a/dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/select.kt b/dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/select.kt
index df8d3c45..6c53c147 100644
--- a/dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/select.kt
+++ b/dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/select.kt
@@ -52,12 +52,12 @@ public fun <R : Any> DataSet<*>.select(
         it as NamedData<R>
     }
 
-    override suspend fun getData(name: Name): Data<R>? = this@select.getData(name)?.let { datum ->
+    override fun get(name: Name): Data<R>? = this@select.get(name)?.let { datum ->
         if (checkDatum(name, datum)) datum.castOrNull(type) else null
     }
 
     override val updates: Flow<Name> = this@select.updates.filter {
-        val datum = this@select.getData(it) ?: return@filter false
+        val datum = this@select.get(it) ?: return@filter false
         checkDatum(it, datum)
     }
 }
@@ -73,11 +73,11 @@ public inline fun <reified R : Any> DataSet<*>.select(
 /**
  * Select a single datum if it is present and of given [type]
  */
-public suspend fun <R : Any> DataSet<*>.selectOne(type: KType, name: Name): NamedData<R>? =
-    getData(name)?.castOrNull<R>(type)?.named(name)
+public fun <R : Any> DataSet<*>.selectOne(type: KType, name: Name): NamedData<R>? =
+    get(name)?.castOrNull<R>(type)?.named(name)
 
-public suspend inline fun <reified R : Any> DataSet<*>.selectOne(name: Name): NamedData<R>? =
+public inline fun <reified R : Any> DataSet<*>.selectOne(name: Name): NamedData<R>? =
     selectOne(typeOf<R>(), name)
 
-public suspend inline fun <reified R : Any> DataSet<*>.selectOne(name: String): NamedData<R>? =
+public inline fun <reified R : Any> DataSet<*>.selectOne(name: String): NamedData<R>? =
     selectOne(typeOf<R>(), Name.parse(name))
\ No newline at end of file
diff --git a/dataforge-io/src/jvmMain/kotlin/space/kscience/dataforge/io/fileIO.kt b/dataforge-io/src/jvmMain/kotlin/space/kscience/dataforge/io/fileIO.kt
index 1b600200..f2c664d4 100644
--- a/dataforge-io/src/jvmMain/kotlin/space/kscience/dataforge/io/fileIO.kt
+++ b/dataforge-io/src/jvmMain/kotlin/space/kscience/dataforge/io/fileIO.kt
@@ -88,9 +88,8 @@ public fun EnvelopeFormat.readFile(path: Path): Envelope {
  */
 @Suppress("UNCHECKED_CAST")
 @DFExperimental
-public inline fun <reified T : Any> IOPlugin.resolveIOFormat(): IOFormat<T>? {
-    return ioFormatFactories.find { it.type.isSupertypeOf(typeOf<T>()) } as IOFormat<T>?
-}
+public inline fun <reified T : Any> IOPlugin.resolveIOFormat(): IOFormat<T>? =
+    ioFormatFactories.find { it.type.isSupertypeOf(typeOf<T>()) } as IOFormat<T>?
 
 /**
  * Read file containing meta using given [formatOverride] or file extension to infer meta type.
diff --git a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/TaskResult.kt b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/TaskResult.kt
index 6a12e3bc..fc8e1e06 100644
--- a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/TaskResult.kt
+++ b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/TaskResult.kt
@@ -26,7 +26,7 @@ public interface TaskResult<out T : Any> : DataSet<T> {
     public val taskMeta: Meta
 
     override fun flowData(): Flow<TaskData<T>>
-    override suspend fun getData(name: Name): TaskData<T>?
+    override fun get(name: Name): TaskData<T>?
 }
 
 private class TaskResultImpl<out T : Any>(
@@ -40,7 +40,7 @@ private class TaskResultImpl<out T : Any>(
         workspace.wrapData(it, it.name, taskName, taskMeta)
     }
 
-    override suspend fun getData(name: Name): TaskData<T>? = dataSet.getData(name)?.let {
+    override fun get(name: Name): TaskData<T>? = dataSet.get(name)?.let {
         workspace.wrapData(it, name, taskName, taskMeta)
     }
 }
diff --git a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/Workspace.kt b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/Workspace.kt
index bc2eb7dd..3aa2c0f4 100644
--- a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/Workspace.kt
+++ b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/Workspace.kt
@@ -46,7 +46,7 @@ public interface Workspace : ContextAware, Provider {
     }
 
     public suspend fun produceData(taskName: Name, taskMeta: Meta, name: Name): TaskData<*>? =
-        produce(taskName, taskMeta).getData(name)
+        produce(taskName, taskMeta).get(name)
 
     public companion object {
         public const val TYPE: String = "workspace"
diff --git a/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/SimpleWorkspaceTest.kt b/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/SimpleWorkspaceTest.kt
index d0db9823..e1a5a466 100644
--- a/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/SimpleWorkspaceTest.kt
+++ b/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/SimpleWorkspaceTest.kt
@@ -103,7 +103,7 @@ class SimpleWorkspaceTest {
             val squareData = from(square)
             val linearData = from(linear)
             squareData.forEach { data ->
-                val newData: Data<Int> = data.combine(linearData.getData(data.name)!!) { l, r ->
+                val newData: Data<Int> = data.combine(linearData.get(data.name)!!) { l, r ->
                     l + r
                 }
                 data(data.name, newData)

From 77857289f03b72e6378073c81c86e829dfd72aec Mon Sep 17 00:00:00 2001
From: Alexander Nozik <altavir@gmail.com>
Date: Sun, 24 Apr 2022 09:57:33 +0300
Subject: [PATCH 05/19] DataSet flow to sequence

---
 CHANGELOG.md                                  |  1 +
 .../kscience/dataforge/actions/MapAction.kt   |  9 +++---
 .../dataforge/actions/ReduceAction.kt         |  3 +-
 .../kscience/dataforge/actions/SplitAction.kt | 12 +++----
 .../space/kscience/dataforge/data/DataSet.kt  | 22 +++++++------
 .../kscience/dataforge/data/DataSetBuilder.kt | 10 ++++--
 .../space/kscience/dataforge/data/DataTree.kt |  8 ++---
 .../kscience/dataforge/data/GroupRule.kt      |  2 +-
 .../kscience/dataforge/data/StaticDataTree.kt |  2 +-
 .../kscience/dataforge/data/dataFilter.kt     |  9 +++---
 .../kscience/dataforge/data/dataTransform.kt  | 32 +++++++++----------
 .../space/kscience/dataforge/data/select.kt   |  7 ++--
 .../dataforge/workspace/TaskResult.kt         |  6 ++--
 .../workspace/DataPropagationTest.kt          |  7 ++--
 .../workspace/SimpleWorkspaceTest.kt          |  8 ++---
 15 files changed, 68 insertions(+), 70 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 94aafcc1..1d27c79b 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -11,6 +11,7 @@
 - KTor 2.0
 - DataTree `items` call is blocking.
 - DataSet `getData` is no longer suspended and renamed to `get`
+- DataSet operates with sequences of data instead of flows
 
 ### Deprecated
 
diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/MapAction.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/MapAction.kt
index e7001d07..157564b1 100644
--- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/MapAction.kt
+++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/MapAction.kt
@@ -1,7 +1,6 @@
 package space.kscience.dataforge.actions
 
 import kotlinx.coroutines.CoroutineScope
-import kotlinx.coroutines.flow.map
 import kotlinx.coroutines.launch
 import space.kscience.dataforge.data.*
 import space.kscience.dataforge.meta.Meta
@@ -50,7 +49,7 @@ internal class MapAction<in T : Any, out R : Any>(
         meta: Meta,
         scope: CoroutineScope?,
     ): DataSet<R> {
-        suspend fun mapOne(data: NamedData<T>): NamedData<R> {
+        fun mapOne(data: NamedData<T>): NamedData<R> {
             // Creating a new environment for action using **old** name, old meta and task meta
             val env = ActionEnv(data.name, data.meta, meta)
 
@@ -75,16 +74,16 @@ internal class MapAction<in T : Any, out R : Any>(
             return newData.named(newName)
         }
 
-        val flow = dataSet.flowData().map(::mapOne)
+        val sequence = dataSet.dataSequence().map(::mapOne)
 
         return ActiveDataTree(outputType) {
-            populateWith(flow)
+            populateWith(sequence)
             scope?.launch {
                 dataSet.updates.collect { name ->
                     //clear old nodes
                     remove(name)
                     //collect new items
-                    populateWith(dataSet.flowChildren(name).map(::mapOne))
+                    populateWith(dataSet.children(name).map(::mapOne))
                 }
             }
         }
diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/ReduceAction.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/ReduceAction.kt
index 126f1f46..5bb22bfc 100644
--- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/ReduceAction.kt
+++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/ReduceAction.kt
@@ -3,7 +3,6 @@ package space.kscience.dataforge.actions
 import kotlinx.coroutines.CoroutineScope
 import kotlinx.coroutines.flow.Flow
 import kotlinx.coroutines.flow.flow
-import kotlinx.coroutines.flow.fold
 import space.kscience.dataforge.data.*
 import space.kscience.dataforge.meta.Meta
 import space.kscience.dataforge.meta.MutableMeta
@@ -84,7 +83,7 @@ internal class ReduceAction<T : Any, R : Any>(
 
     override fun CoroutineScope.transform(set: DataSet<T>, meta: Meta, key: Name): Flow<NamedData<R>> = flow {
         ReduceGroupBuilder<T, R>(inputType, this@transform, meta).apply(action).buildGroups(set).forEach { group ->
-            val dataFlow: Map<Name, Data<T>> = group.set.flowData().fold(HashMap()) { acc, value ->
+            val dataFlow: Map<Name, Data<T>> = group.set.dataSequence().fold(HashMap()) { acc, value ->
                 acc.apply {
                     acc[value.name] = value.data
                 }
diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/SplitAction.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/SplitAction.kt
index b55a0e08..88d047cd 100644
--- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/SplitAction.kt
+++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/SplitAction.kt
@@ -2,10 +2,6 @@ package space.kscience.dataforge.actions
 
 import kotlinx.coroutines.CoroutineScope
 import kotlinx.coroutines.FlowPreview
-import kotlinx.coroutines.flow.Flow
-import kotlinx.coroutines.flow.asFlow
-import kotlinx.coroutines.flow.flatMapConcat
-import kotlinx.coroutines.flow.map
 import kotlinx.coroutines.launch
 import space.kscience.dataforge.data.*
 import space.kscience.dataforge.meta.Laminate
@@ -58,14 +54,14 @@ internal class SplitAction<T : Any, R : Any>(
         scope: CoroutineScope?,
     ): DataSet<R> {
 
-        suspend fun splitOne(data: NamedData<T>): Flow<NamedData<R>> {
+        fun splitOne(data: NamedData<T>): Sequence<NamedData<R>> {
             val laminate = Laminate(data.meta, meta)
 
             val split = SplitBuilder<T, R>(data.name, data.meta).apply(action)
 
 
             // apply individual fragment rules to result
-            return split.fragments.entries.asFlow().map { (fragmentName, rule) ->
+            return split.fragments.entries.asSequence().map { (fragmentName, rule) ->
                 val env = SplitBuilder.FragmentRule<T, R>(fragmentName, laminate.toMutableMeta()).apply(rule)
                 //data.map<R>(outputType, meta = env.meta) { env.result(it) }.named(fragmentName)
                 @OptIn(DFInternal::class) Data(outputType, meta = env.meta, dependencies = listOf(data)) {
@@ -75,13 +71,13 @@ internal class SplitAction<T : Any, R : Any>(
         }
 
         return ActiveDataTree<R>(outputType) {
-            populateWith(dataSet.flowData().flatMapConcat(transform = ::splitOne))
+            populateWith(dataSet.dataSequence().flatMap (transform = ::splitOne))
             scope?.launch {
                 dataSet.updates.collect { name ->
                     //clear old nodes
                     remove(name)
                     //collect new items
-                    populateWith(dataSet.flowChildren(name).flatMapConcat(transform = ::splitOne))
+                    populateWith(dataSet.children(name).flatMap(transform = ::splitOne))
                 }
             }
         }
diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSet.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSet.kt
index f4838c85..d4ca9296 100644
--- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSet.kt
+++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSet.kt
@@ -1,7 +1,9 @@
 package space.kscience.dataforge.data
 
 import kotlinx.coroutines.*
-import kotlinx.coroutines.flow.*
+import kotlinx.coroutines.flow.Flow
+import kotlinx.coroutines.flow.emptyFlow
+import kotlinx.coroutines.flow.mapNotNull
 import space.kscience.dataforge.data.Data.Companion.TYPE_OF_NOTHING
 import space.kscience.dataforge.meta.Meta
 import space.kscience.dataforge.meta.set
@@ -23,7 +25,7 @@ public interface DataSet<out T : Any> {
     /**
      * Traverse this provider or its child. The order is not guaranteed.
      */
-    public fun flowData(): Flow<NamedData<T>>
+    public fun dataSequence(): Sequence<NamedData<T>>
 
     /**
      * Get data with given name.
@@ -34,8 +36,8 @@ public interface DataSet<out T : Any> {
     /**
      * Get a snapshot of names of top level children of given node. Empty if node does not exist or is a leaf.
      */
-    public suspend fun listTop(prefix: Name = Name.EMPTY): List<Name> =
-        flowData().map { it.name }.filter { it.startsWith(prefix) && (it.length == prefix.length + 1) }.toList()
+    public fun listTop(prefix: Name = Name.EMPTY): List<Name> =
+        dataSequence().map { it.name }.filter { it.startsWith(prefix) && (it.length == prefix.length + 1) }.toList()
     // By default, traverses the whole tree. Could be optimized in descendants
 
     public companion object {
@@ -50,13 +52,15 @@ public interface DataSet<out T : Any> {
 
             //private val nothing: Nothing get() = error("this is nothing")
 
-            override fun flowData(): Flow<NamedData<Nothing>> = emptyFlow()
+            override fun dataSequence(): Sequence<NamedData<Nothing>> = emptySequence()
 
             override fun get(name: Name): Data<Nothing>? = null
         }
     }
 }
 
+public operator fun <T: Any> DataSet<T>.get(name:String): Data<T>? = get(name.parseAsName())
+
 public interface ActiveDataSet<T : Any> : DataSet<T> {
     /**
      * A flow of updated item names. Updates are propagated in a form of [Flow] of names of updated nodes.
@@ -72,8 +76,8 @@ public val <T : Any> DataSet<T>.updates: Flow<Name> get() = if (this is ActiveDa
 /**
  * Flow all data nodes with names starting with [branchName]
  */
-public fun <T : Any> DataSet<T>.flowChildren(branchName: Name): Flow<NamedData<T>> =
-    this@flowChildren.flowData().filter {
+public fun <T : Any> DataSet<T>.children(branchName: Name): Sequence<NamedData<T>> =
+    this@children.dataSequence().filter {
         it.name.startsWith(branchName)
     }
 
@@ -81,7 +85,7 @@ public fun <T : Any> DataSet<T>.flowChildren(branchName: Name): Flow<NamedData<T
  * Start computation for all goals in data node and return a job for the whole node
  */
 public fun <T : Any> DataSet<T>.startAll(coroutineScope: CoroutineScope): Job = coroutineScope.launch {
-    flowData().map {
+    dataSequence().map {
         it.launch(this@launch)
     }.toList().joinAll()
 }
@@ -89,7 +93,7 @@ public fun <T : Any> DataSet<T>.startAll(coroutineScope: CoroutineScope): Job =
 public suspend fun <T : Any> DataSet<T>.join(): Unit = coroutineScope { startAll(this).join() }
 
 public suspend fun DataSet<*>.toMeta(): Meta = Meta {
-    flowData().collect {
+    dataSequence().forEach {
         if (it.name.endsWith(DataSet.META_KEY)) {
             set(it.name, it.meta)
         } else {
diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSetBuilder.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSetBuilder.kt
index 0c13f2c6..d279e34c 100644
--- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSetBuilder.kt
+++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSetBuilder.kt
@@ -30,7 +30,7 @@ public interface DataSetBuilder<in T : Any> {
         }
 
         //Set new items
-        dataSet.flowData().collect {
+        dataSet.dataSequence().forEach {
             data(name + it.name, it.data)
         }
     }
@@ -148,7 +148,7 @@ public suspend inline fun <reified T : Any> DataSetBuilder<T>.static(
  */
 @DFExperimental
 public suspend fun <T : Any> DataSetBuilder<T>.populateFrom(tree: DataSet<T>): Unit = coroutineScope {
-    tree.flowData().collect {
+    tree.dataSequence().forEach {
         //TODO check if the place is occupied
         data(it.name, it.data)
     }
@@ -159,3 +159,9 @@ public suspend fun <T : Any> DataSetBuilder<T>.populateWith(flow: Flow<NamedData
         data(it.name, it.data)
     }
 }
+
+public suspend fun <T : Any> DataSetBuilder<T>.populateWith(sequence: Sequence<NamedData<T>>) {
+    sequence.forEach {
+        data(it.name, it.data)
+    }
+}
diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataTree.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataTree.kt
index 8eee0113..aab14e25 100644
--- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataTree.kt
+++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataTree.kt
@@ -43,18 +43,18 @@ public interface DataTree<out T : Any> : DataSet<T> {
 
     override val meta: Meta get() = items[META_ITEM_NAME_TOKEN]?.meta ?: Meta.EMPTY
 
-    override fun flowData(): Flow<NamedData<T>> = flow {
+    override fun dataSequence(): Sequence<NamedData<T>> = sequence {
         items.forEach { (token, childItem: DataTreeItem<T>) ->
             if (!token.body.startsWith("@")) {
                 when (childItem) {
-                    is DataTreeItem.Leaf -> emit(childItem.data.named(token.asName()))
-                    is DataTreeItem.Node -> emitAll(childItem.tree.flowData().map { it.named(token + it.name) })
+                    is DataTreeItem.Leaf -> yield(childItem.data.named(token.asName()))
+                    is DataTreeItem.Node -> yieldAll(childItem.tree.dataSequence().map { it.named(token + it.name) })
                 }
             }
         }
     }
 
-    override suspend fun listTop(prefix: Name): List<Name> =
+    override fun listTop(prefix: Name): List<Name> =
         getItem(prefix).tree?.items?.keys?.map { prefix + it } ?: emptyList()
 
     override fun get(name: Name): Data<T>? = when (name.length) {
diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/GroupRule.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/GroupRule.kt
index beaa3da0..5ef8a6d5 100644
--- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/GroupRule.kt
+++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/GroupRule.kt
@@ -43,7 +43,7 @@ public interface GroupRule {
             ): Map<String, DataSet<T>> {
                 val map = HashMap<String, ActiveDataTree<T>>()
 
-                set.flowData().collect { data ->
+                set.dataSequence().forEach { data ->
                     val tagValue = data.meta[key]?.string ?: defaultTagValue
                     map.getOrPut(tagValue) { ActiveDataTree(set.dataType) }.data(data.name, data.data)
                 }
diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/StaticDataTree.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/StaticDataTree.kt
index f37459dc..06b1ff6f 100644
--- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/StaticDataTree.kt
+++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/StaticDataTree.kt
@@ -54,7 +54,7 @@ internal class StaticDataTree<T : Any>(
             set(name, DataTreeItem.Node(dataSet))
         } else {
             coroutineScope {
-                dataSet.flowData().collect {
+                dataSet.dataSequence().forEach {
                     data(name + it.name, it.data)
                 }
             }
diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataFilter.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataFilter.kt
index 46f07260..5ee7027d 100644
--- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataFilter.kt
+++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataFilter.kt
@@ -24,8 +24,8 @@ public fun <T : Any> DataSet<T>.filter(
 
     override val meta: Meta get() = this@filter.meta
 
-    override fun flowData(): Flow<NamedData<T>> =
-        this@filter.flowData().filter { predicate(it.name, it.data) }
+    override fun dataSequence(): Sequence<NamedData<T>> =
+        this@filter.dataSequence().filter { predicate(it.name, it.data) }
 
     override fun get(name: Name): Data<T>? = this@filter.get(name)?.takeIf {
         predicate(name, it)
@@ -48,7 +48,8 @@ else object : ActiveDataSet<T> {
     override val meta: Meta get() = this@withNamePrefix.meta
 
 
-    override fun flowData(): Flow<NamedData<T>> = this@withNamePrefix.flowData().map { it.data.named(prefix + it.name) }
+    override fun dataSequence(): Sequence<NamedData<T>> =
+        this@withNamePrefix.dataSequence().map { it.data.named(prefix + it.name) }
 
     override fun get(name: Name): Data<T>? =
         name.removeHeadOrNull(name)?.let { this@withNamePrefix.get(it) }
@@ -66,7 +67,7 @@ public fun <T : Any> DataSet<T>.branch(branchName: Name): DataSet<T> = if (branc
 
     override val meta: Meta get() = this@branch.meta
 
-    override fun flowData(): Flow<NamedData<T>> = this@branch.flowData().mapNotNull {
+    override fun dataSequence(): Sequence<NamedData<T>> = this@branch.dataSequence().mapNotNull {
         it.name.removeHeadOrNull(branchName)?.let { name ->
             it.data.named(name)
         }
diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataTransform.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataTransform.kt
index 8c3332df..c8180eb5 100644
--- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataTransform.kt
+++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataTransform.kt
@@ -1,9 +1,7 @@
 package space.kscience.dataforge.data
 
 import kotlinx.coroutines.flow.Flow
-import kotlinx.coroutines.flow.fold
 import kotlinx.coroutines.flow.map
-import kotlinx.coroutines.flow.toList
 import space.kscience.dataforge.meta.Meta
 import space.kscience.dataforge.meta.MutableMeta
 import space.kscience.dataforge.meta.seal
@@ -100,11 +98,11 @@ public inline fun <K, T : Any, reified R : Any> Map<K, Data<T>>.reduceToData(
  * Transform a [Flow] of [NamedData] to a single [Data].
  */
 @DFInternal
-public suspend fun <T : Any, R : Any> Flow<NamedData<T>>.reduceToData(
+public inline fun <T : Any, R : Any> Sequence<NamedData<T>>.reduceToData(
     outputType: KType,
     coroutineContext: CoroutineContext = EmptyCoroutineContext,
     meta: Meta = Meta.EMPTY,
-    transformation: suspend (Flow<NamedData<T>>) -> R,
+    crossinline transformation: suspend (Sequence<NamedData<T>>) -> R,
 ): Data<R> = Data(
     outputType,
     meta,
@@ -115,10 +113,10 @@ public suspend fun <T : Any, R : Any> Flow<NamedData<T>>.reduceToData(
 }
 
 @OptIn(DFInternal::class)
-public suspend inline fun <T : Any, reified R : Any> Flow<NamedData<T>>.reduceToData(
+public inline fun <T : Any, reified R : Any> Sequence<NamedData<T>>.reduceToData(
     coroutineContext: CoroutineContext = EmptyCoroutineContext,
     meta: Meta = Meta.EMPTY,
-    noinline transformation: suspend (Flow<NamedData<T>>) -> R,
+    crossinline transformation: suspend (Sequence<NamedData<T>>) -> R,
 ): Data<R> = reduceToData(typeOf<R>(), coroutineContext, meta) {
     transformation(it)
 }
@@ -126,15 +124,15 @@ public suspend inline fun <T : Any, reified R : Any> Flow<NamedData<T>>.reduceTo
 /**
  * Fold a flow of named data into a single [Data]
  */
-public suspend inline fun <T : Any, reified R : Any> Flow<NamedData<T>>.foldToData(
+public inline fun <T : Any, reified R : Any> Sequence<NamedData<T>>.foldToData(
     initial: R,
     coroutineContext: CoroutineContext = EmptyCoroutineContext,
     meta: Meta = Meta.EMPTY,
-    noinline block: suspend (result: R, data: NamedData<T>) -> R,
+    crossinline block: suspend (result: R, data: NamedData<T>) -> R,
 ): Data<R> = reduceToData(
     coroutineContext, meta
 ) {
-    it.fold(initial, block)
+    it.fold(initial) { acc, t -> block(acc, t) }
 }
 
 //DataSet operations
@@ -147,7 +145,7 @@ public suspend fun <T : Any, R : Any> DataSet<T>.map(
     block: suspend (T) -> R,
 ): DataTree<R> = DataTree<R>(outputType) {
     populateWith(
-        flowData().map {
+        dataSequence().map {
             val newMeta = it.meta.toMutableMeta().apply(metaTransform).seal()
             Data(outputType, newMeta, coroutineContext, listOf(it)) {
                 block(it.await())
@@ -165,20 +163,20 @@ public suspend inline fun <T : Any, reified R : Any> DataSet<T>.map(
 
 public suspend fun <T : Any> DataSet<T>.forEach(block: suspend (NamedData<T>) -> Unit) {
     contract { callsInPlace(block, InvocationKind.EXACTLY_ONCE) }
-    flowData().collect {
+    dataSequence().forEach {
         block(it)
     }
 }
 
-public suspend inline fun <T : Any, reified R : Any> DataSet<T>.reduceToData(
+public inline fun <T : Any, reified R : Any> DataSet<T>.reduceToData(
     coroutineContext: CoroutineContext = EmptyCoroutineContext,
     meta: Meta = Meta.EMPTY,
-    noinline transformation: suspend (Flow<NamedData<T>>) -> R,
-): Data<R> = flowData().reduceToData(coroutineContext, meta, transformation)
+    crossinline transformation: suspend (Sequence<NamedData<T>>) -> R,
+): Data<R> = dataSequence().reduceToData(coroutineContext, meta, transformation)
 
-public suspend inline fun <T : Any, reified R : Any> DataSet<T>.foldToData(
+public inline fun <T : Any, reified R : Any> DataSet<T>.foldToData(
     initial: R,
     coroutineContext: CoroutineContext = EmptyCoroutineContext,
     meta: Meta = Meta.EMPTY,
-    noinline block: suspend (result: R, data: NamedData<T>) -> R,
-): Data<R> = flowData().foldToData(initial, coroutineContext, meta, block)
\ No newline at end of file
+    crossinline block: suspend (result: R, data: NamedData<T>) -> R,
+): Data<R> = dataSequence().foldToData(initial, coroutineContext, meta, block)
\ No newline at end of file
diff --git a/dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/select.kt b/dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/select.kt
index 6c53c147..f32ff3f3 100644
--- a/dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/select.kt
+++ b/dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/select.kt
@@ -2,7 +2,6 @@ package space.kscience.dataforge.data
 
 import kotlinx.coroutines.flow.Flow
 import kotlinx.coroutines.flow.filter
-import kotlinx.coroutines.flow.map
 import space.kscience.dataforge.meta.Meta
 import space.kscience.dataforge.misc.DFExperimental
 import space.kscience.dataforge.names.Name
@@ -45,19 +44,19 @@ public fun <R : Any> DataSet<*>.select(
             && (namePattern == null || name.matches(namePattern))
             && filter(name, datum.meta)
 
-    override fun flowData(): Flow<NamedData<R>> = this@select.flowData().filter {
+    override fun dataSequence(): Sequence<NamedData<R>> = this@select.dataSequence().filter {
         checkDatum(it.name, it.data)
     }.map {
         @Suppress("UNCHECKED_CAST")
         it as NamedData<R>
     }
 
-    override fun get(name: Name): Data<R>? = this@select.get(name)?.let { datum ->
+    override fun get(name: Name): Data<R>? = this@select[name]?.let { datum ->
         if (checkDatum(name, datum)) datum.castOrNull(type) else null
     }
 
     override val updates: Flow<Name> = this@select.updates.filter {
-        val datum = this@select.get(it) ?: return@filter false
+        val datum = this@select[it] ?: return@filter false
         checkDatum(it, datum)
     }
 }
diff --git a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/TaskResult.kt b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/TaskResult.kt
index fc8e1e06..a1fb4e84 100644
--- a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/TaskResult.kt
+++ b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/TaskResult.kt
@@ -1,7 +1,5 @@
 package space.kscience.dataforge.workspace
 
-import kotlinx.coroutines.flow.Flow
-import kotlinx.coroutines.flow.map
 import space.kscience.dataforge.data.DataSet
 import space.kscience.dataforge.meta.Meta
 import space.kscience.dataforge.names.Name
@@ -25,7 +23,7 @@ public interface TaskResult<out T : Any> : DataSet<T> {
      */
     public val taskMeta: Meta
 
-    override fun flowData(): Flow<TaskData<T>>
+    override fun dataSequence(): Sequence<TaskData<T>>
     override fun get(name: Name): TaskData<T>?
 }
 
@@ -36,7 +34,7 @@ private class TaskResultImpl<out T : Any>(
     override val taskMeta: Meta,
 ) : TaskResult<T>, DataSet<T> by dataSet {
 
-    override fun flowData(): Flow<TaskData<T>> = dataSet.flowData().map {
+    override fun dataSequence(): Sequence<TaskData<T>> = dataSet.dataSequence().map {
         workspace.wrapData(it, it.name, taskName, taskMeta)
     }
 
diff --git a/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/DataPropagationTest.kt b/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/DataPropagationTest.kt
index 91a3ffd2..07b8fe36 100644
--- a/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/DataPropagationTest.kt
+++ b/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/DataPropagationTest.kt
@@ -1,6 +1,5 @@
 package space.kscience.dataforge.workspace
 
-import kotlinx.coroutines.flow.single
 import kotlinx.coroutines.runBlocking
 import space.kscience.dataforge.context.Context
 import space.kscience.dataforge.context.PluginFactory
@@ -16,7 +15,7 @@ class DataPropagationTestPlugin : WorkspacePlugin() {
 
     val allData by task<Int> {
         val selectedData = workspace.data.select<Int>()
-        val result: Data<Int> = selectedData.flowData().foldToData(0) { result, data ->
+        val result: Data<Int> = selectedData.dataSequence().foldToData(0) { result, data ->
             result + data.await()
         }
         data("result", result)
@@ -58,7 +57,7 @@ class DataPropagationTest {
     fun testAllData() {
         runBlocking {
             val node = testWorkspace.produce("Test.allData")
-            assertEquals(4950, node.flowData().single().await())
+            assertEquals(4950, node.dataSequence().single().await())
         }
     }
 
@@ -66,7 +65,7 @@ class DataPropagationTest {
     fun testSingleData() {
         runBlocking {
             val node = testWorkspace.produce("Test.singleData")
-            assertEquals(12, node.flowData().single().await())
+            assertEquals(12, node.dataSequence().single().await())
         }
     }
 }
\ No newline at end of file
diff --git a/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/SimpleWorkspaceTest.kt b/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/SimpleWorkspaceTest.kt
index e1a5a466..3bd09251 100644
--- a/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/SimpleWorkspaceTest.kt
+++ b/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/SimpleWorkspaceTest.kt
@@ -2,8 +2,6 @@
 
 package space.kscience.dataforge.workspace
 
-import kotlinx.coroutines.flow.first
-import kotlinx.coroutines.flow.single
 import kotlinx.coroutines.runBlocking
 import org.junit.jupiter.api.Timeout
 import space.kscience.dataforge.context.*
@@ -161,7 +159,7 @@ class SimpleWorkspaceTest {
     fun testWorkspace() {
         runBlocking {
             val node = workspace.runBlocking("sum")
-            val res = node.flowData().single()
+            val res = node.dataSequence().single()
             assertEquals(328350, res.await())
         }
     }
@@ -171,7 +169,7 @@ class SimpleWorkspaceTest {
     fun testMetaPropagation() {
         runBlocking {
             val node = workspace.produce("sum") { "testFlag" put true }
-            val res = node.flowData().single().await()
+            val res = node.dataSequence().single().await()
         }
     }
 
@@ -194,7 +192,7 @@ class SimpleWorkspaceTest {
     fun testFilter() {
         runBlocking {
             val node = workspace.produce("filterOne")
-            assertEquals(12, node.flowData().first().await())
+            assertEquals(12, node.dataSequence().first().await())
         }
     }
 }
\ No newline at end of file

From 6d396368b7bfb862672ce849b9e283e6af145ed3 Mon Sep 17 00:00:00 2001
From: Alexander Nozik <altavir@gmail.com>
Date: Sun, 24 Apr 2022 14:44:31 +0300
Subject: [PATCH 06/19] Fixe meta file name

---
 CHANGELOG.md                                              | 1 +
 build.gradle.kts                                          | 2 +-
 .../kotlin/space/kscience/dataforge/data/DataSet.kt       | 3 ++-
 .../jvmMain/kotlin/space/kscience/dataforge/io/fileIO.kt  | 8 +++++---
 4 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 1d27c79b..52af8804 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -18,6 +18,7 @@
 ### Removed
 
 ### Fixed
+- Meta file name in readMeta from directory
 
 ### Security
 
diff --git a/build.gradle.kts b/build.gradle.kts
index 78ca749b..977b44f8 100644
--- a/build.gradle.kts
+++ b/build.gradle.kts
@@ -4,7 +4,7 @@ plugins {
 
 allprojects {
     group = "space.kscience"
-    version = "0.6.0-dev-1"
+    version = "0.6.0-dev-2"
     repositories{
         mavenCentral()
     }
diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSet.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSet.kt
index d4ca9296..45e60bb4 100644
--- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSet.kt
+++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSet.kt
@@ -10,7 +10,8 @@ import space.kscience.dataforge.meta.set
 import space.kscience.dataforge.names.*
 import kotlin.reflect.KType
 
-public interface DataSet<out T : Any> {
+public interface
+DataSet<out T : Any> {
 
     /**
      * The minimal common ancestor to all data in the node
diff --git a/dataforge-io/src/jvmMain/kotlin/space/kscience/dataforge/io/fileIO.kt b/dataforge-io/src/jvmMain/kotlin/space/kscience/dataforge/io/fileIO.kt
index f2c664d4..1f2d3041 100644
--- a/dataforge-io/src/jvmMain/kotlin/space/kscience/dataforge/io/fileIO.kt
+++ b/dataforge-io/src/jvmMain/kotlin/space/kscience/dataforge/io/fileIO.kt
@@ -91,6 +91,10 @@ public fun EnvelopeFormat.readFile(path: Path): Envelope {
 public inline fun <reified T : Any> IOPlugin.resolveIOFormat(): IOFormat<T>? =
     ioFormatFactories.find { it.type.isSupertypeOf(typeOf<T>()) } as IOFormat<T>?
 
+
+public val IOPlugin.Companion.META_FILE_NAME: String get() = "@meta"
+public val IOPlugin.Companion.DATA_FILE_NAME: String get() = "@data"
+
 /**
  * Read file containing meta using given [formatOverride] or file extension to infer meta type.
  * If [path] is a directory search for file starting with `meta` in it
@@ -103,7 +107,7 @@ public fun IOPlugin.readMetaFile(
     if (!Files.exists(path)) error("Meta file $path does not exist")
 
     val actualPath: Path = if (Files.isDirectory(path)) {
-        Files.list(path).asSequence().find { it.fileName.startsWith("meta") }
+        Files.list(path).asSequence().find { it.fileName.startsWith(IOPlugin.META_FILE_NAME) }
             ?: error("The directory $path does not contain meta file")
     } else {
         path
@@ -147,8 +151,6 @@ public fun IOPlugin.peekFileEnvelopeFormat(path: Path): EnvelopeFormat? {
     return peekBinaryEnvelopeFormat(binary)
 }
 
-public val IOPlugin.Companion.META_FILE_NAME: String get() = "@meta"
-public val IOPlugin.Companion.DATA_FILE_NAME: String get() = "@data"
 
 /**
  * Read and envelope from file if the file exists, return null if file does not exist.

From 82d37f4b5582a4f8c955c0be7aa73727ae96135b Mon Sep 17 00:00:00 2001
From: Alexander Nozik <altavir@gmail.com>
Date: Fri, 29 Apr 2022 18:35:44 +0300
Subject: [PATCH 07/19] Fix text envelope formats partial reads

---
 CHANGELOG.md                                  |   2 +
 build.gradle.kts                              |   2 +-
 .../space/kscience/dataforge/data/Data.kt     |   4 +
 .../io/yaml/FrontMatterEnvelopeFormat.kt      |  62 ++++-------
 .../kscience/dataforge/io/yaml/YamlPlugin.kt  |   2 +
 .../io/yaml/FrontMatterEnvelopeFormatTest.kt  |  34 ++++++
 .../kscience/dataforge/io/EnvelopeFormat.kt   |   4 +-
 .../dataforge/io/TaggedEnvelopeFormat.kt      |   4 +-
 .../dataforge/io/TaglessEnvelopeFormat.kt     |  97 +++++++++-------
 .../space/kscience/dataforge/io/ioMisc.kt     | 105 +++++++++++++++++-
 .../dataforge/io/EnvelopeFormatTest.kt        |  15 ++-
 .../space/kscience/dataforge/io/IOTest.kt     |  40 +++++++
 .../kscience/dataforge/workspace/Workspace.kt |   5 +-
 13 files changed, 284 insertions(+), 92 deletions(-)
 create mode 100644 dataforge-io/dataforge-io-yaml/src/commonTest/kotlin/space/kscience/dataforge/io/yaml/FrontMatterEnvelopeFormatTest.kt

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 52af8804..26208ef7 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -12,6 +12,7 @@
 - DataTree `items` call is blocking.
 - DataSet `getData` is no longer suspended and renamed to `get`
 - DataSet operates with sequences of data instead of flows
+- PartialEnvelope uses `Int` instead `UInt`.
 
 ### Deprecated
 
@@ -19,6 +20,7 @@
 
 ### Fixed
 - Meta file name in readMeta from directory
+- Tagless and FrontMatter envelope partial readers fix.
 
 ### Security
 
diff --git a/build.gradle.kts b/build.gradle.kts
index 977b44f8..9db78013 100644
--- a/build.gradle.kts
+++ b/build.gradle.kts
@@ -4,7 +4,7 @@ plugins {
 
 allprojects {
     group = "space.kscience"
-    version = "0.6.0-dev-2"
+    version = "0.6.0-dev-3"
     repositories{
         mavenCentral()
     }
diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/Data.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/Data.kt
index 8e681678..41182882 100644
--- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/Data.kt
+++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/Data.kt
@@ -83,6 +83,10 @@ public class StaticData<T : Any>(
     override val meta: Meta = Meta.EMPTY,
 ) : Data<T>, StaticGoal<T>(value)
 
+@Suppress("FunctionName")
+public inline fun <reified T : Any> Data(value: T, meta: Meta = Meta.EMPTY): StaticData<T> =
+    StaticData(typeOf<T>(), value, meta)
+
 @Suppress("FunctionName")
 @DFInternal
 public fun <T : Any> Data(
diff --git a/dataforge-io/dataforge-io-yaml/src/commonMain/kotlin/space/kscience/dataforge/io/yaml/FrontMatterEnvelopeFormat.kt b/dataforge-io/dataforge-io-yaml/src/commonMain/kotlin/space/kscience/dataforge/io/yaml/FrontMatterEnvelopeFormat.kt
index 001c8ef0..2734ebd9 100644
--- a/dataforge-io/dataforge-io-yaml/src/commonMain/kotlin/space/kscience/dataforge/io/yaml/FrontMatterEnvelopeFormat.kt
+++ b/dataforge-io/dataforge-io-yaml/src/commonMain/kotlin/space/kscience/dataforge/io/yaml/FrontMatterEnvelopeFormat.kt
@@ -2,8 +2,8 @@ package space.kscience.dataforge.io.yaml
 
 import io.ktor.utils.io.core.Input
 import io.ktor.utils.io.core.Output
+import io.ktor.utils.io.core.buildPacket
 import io.ktor.utils.io.core.readBytes
-import io.ktor.utils.io.core.readUTF8Line
 import space.kscience.dataforge.context.Context
 import space.kscience.dataforge.context.Global
 import space.kscience.dataforge.io.*
@@ -11,6 +11,8 @@ import space.kscience.dataforge.io.IOFormat.Companion.META_KEY
 import space.kscience.dataforge.io.IOFormat.Companion.NAME_KEY
 import space.kscience.dataforge.meta.Meta
 import space.kscience.dataforge.misc.DFExperimental
+import space.kscience.dataforge.names.Name
+import space.kscience.dataforge.names.plus
 
 @DFExperimental
 public class FrontMatterEnvelopeFormat(
@@ -19,51 +21,33 @@ public class FrontMatterEnvelopeFormat(
 ) : EnvelopeFormat {
 
     override fun readPartial(input: Input): PartialEnvelope {
-        var line: String
-        var offset = 0u
-        do {
-            line = input.readUTF8Line() ?: error("Input does not contain front matter separator")
-            offset += line.encodeToByteArray().size.toUInt()
-        } while (!line.startsWith(SEPARATOR))
+        var offset = 0
 
-        val readMetaFormat =
-            metaTypeRegex.matchEntire(line)?.groupValues?.first()
-                ?.let { io.resolveMetaFormat(it) } ?: YamlMetaFormat
+        offset += input.discardWithSeparator(
+            SEPARATOR.encodeToByteArray(),
+            atMost = 1024,
+            skipUntilEndOfLine = false
+        )
+
+        val line = input.readSafeUtf8Line()
+        val readMetaFormat = line.trim().takeIf { it.isNotBlank() }?.let { io.resolveMetaFormat(it) } ?: YamlMetaFormat
 
         //TODO replace by preview
-        val meta = Binary {
-            do {
-                line = input.readSafeUtf8Line()
-                writeUtf8String(line + "\r\n")
-                offset += line.encodeToByteArray().size.toUInt()
-            } while (!line.startsWith(SEPARATOR))
-        }.read {
-            readMetaFormat.readMeta(input)
-
+        val packet =  buildPacket {
+            offset += input.readBytesWithSeparatorTo(
+                this,
+                SEPARATOR.encodeToByteArray(),
+                skipUntilEndOfLine = true
+            )
         }
+        val meta = readMetaFormat.readMeta(packet)
         return PartialEnvelope(meta, offset, null)
     }
 
     override fun readObject(input: Input): Envelope {
-        var line: String
-        do {
-            line = input.readSafeUtf8Line() //?: error("Input does not contain front matter separator")
-        } while (!line.startsWith(SEPARATOR))
-
-        val readMetaFormat =
-            metaTypeRegex.matchEntire(line)?.groupValues?.first()
-                ?.let { io.resolveMetaFormat(it) } ?: YamlMetaFormat
-
-        val meta = Binary {
-            do {
-                writeUtf8String(input.readSafeUtf8Line() + "\r\n")
-            } while (!line.startsWith(SEPARATOR))
-        }.read {
-            readMetaFormat.readMeta(input)
-        }
-        val bytes = input.readBytes()
-        val data = bytes.asBinary()
-        return SimpleEnvelope(meta, data)
+        val partial = readPartial(input)
+        val data = input.readBytes().asBinary()
+        return SimpleEnvelope(partial.meta, data)
     }
 
     override fun writeEnvelope(
@@ -92,6 +76,8 @@ public class FrontMatterEnvelopeFormat(
 
         private val metaTypeRegex = "---(\\w*)\\s*".toRegex()
 
+        override val name: Name = EnvelopeFormatFactory.ENVELOPE_FACTORY_NAME + "frontMatter"
+
         override fun build(context: Context, meta: Meta): EnvelopeFormat {
             return FrontMatterEnvelopeFormat(context.io, meta)
         }
diff --git a/dataforge-io/dataforge-io-yaml/src/commonMain/kotlin/space/kscience/dataforge/io/yaml/YamlPlugin.kt b/dataforge-io/dataforge-io-yaml/src/commonMain/kotlin/space/kscience/dataforge/io/yaml/YamlPlugin.kt
index 1e7530f9..dea3a38c 100644
--- a/dataforge-io/dataforge-io-yaml/src/commonMain/kotlin/space/kscience/dataforge/io/yaml/YamlPlugin.kt
+++ b/dataforge-io/dataforge-io-yaml/src/commonMain/kotlin/space/kscience/dataforge/io/yaml/YamlPlugin.kt
@@ -4,6 +4,7 @@ import space.kscience.dataforge.context.AbstractPlugin
 import space.kscience.dataforge.context.Context
 import space.kscience.dataforge.context.PluginFactory
 import space.kscience.dataforge.context.PluginTag
+import space.kscience.dataforge.io.EnvelopeFormatFactory
 import space.kscience.dataforge.io.IOPlugin
 import space.kscience.dataforge.io.MetaFormatFactory
 import space.kscience.dataforge.meta.Meta
@@ -20,6 +21,7 @@ public class YamlPlugin(meta: Meta) : AbstractPlugin(meta) {
 
     override fun content(target: String): Map<Name, Any> = when (target) {
         MetaFormatFactory.META_FORMAT_TYPE -> mapOf("yaml".asName() to YamlMetaFormat)
+        EnvelopeFormatFactory.ENVELOPE_FORMAT_TYPE -> mapOf(FrontMatterEnvelopeFormat.name to FrontMatterEnvelopeFormat)
         else -> super.content(target)
     }
 
diff --git a/dataforge-io/dataforge-io-yaml/src/commonTest/kotlin/space/kscience/dataforge/io/yaml/FrontMatterEnvelopeFormatTest.kt b/dataforge-io/dataforge-io-yaml/src/commonTest/kotlin/space/kscience/dataforge/io/yaml/FrontMatterEnvelopeFormatTest.kt
new file mode 100644
index 00000000..0c242597
--- /dev/null
+++ b/dataforge-io/dataforge-io-yaml/src/commonTest/kotlin/space/kscience/dataforge/io/yaml/FrontMatterEnvelopeFormatTest.kt
@@ -0,0 +1,34 @@
+package space.kscience.dataforge.io.yaml
+
+import space.kscience.dataforge.context.Context
+import space.kscience.dataforge.io.io
+import space.kscience.dataforge.io.readEnvelope
+import space.kscience.dataforge.io.toByteArray
+import space.kscience.dataforge.meta.get
+import space.kscience.dataforge.meta.string
+import kotlin.test.Test
+import kotlin.test.assertEquals
+
+internal class FrontMatterEnvelopeFormatTest {
+
+    val context = Context {
+        plugin(YamlPlugin)
+    }
+
+    @Test
+    fun frontMatter(){
+        val text = """
+            ---
+            content_type: magprog
+            magprog_section: contacts
+            section_title: Контакты
+            language: ru
+            ---
+            Some text here
+        """.trimIndent()
+
+        val envelope = context.io.readEnvelope(text)
+        assertEquals("Some text here", envelope.data!!.toByteArray().decodeToString().trim())
+        assertEquals("magprog", envelope.meta["content_type"].string)
+    }
+}
\ No newline at end of file
diff --git a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/EnvelopeFormat.kt b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/EnvelopeFormat.kt
index 9cb2cef0..4dea572b 100644
--- a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/EnvelopeFormat.kt
+++ b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/EnvelopeFormat.kt
@@ -14,7 +14,7 @@ import kotlin.reflect.typeOf
 /**
  * A partially read envelope with meta, but without data
  */
-public data class PartialEnvelope(val meta: Meta, val dataOffset: UInt, val dataSize: ULong?)
+public data class PartialEnvelope(val meta: Meta, val dataOffset: Int, val dataSize: ULong?)
 
 public interface EnvelopeFormat : IOFormat<Envelope> {
     override val type: KType get() = typeOf<Envelope>()
@@ -39,7 +39,6 @@ public fun EnvelopeFormat.read(input: Input): Envelope = readObject(input)
 
 @Type(ENVELOPE_FORMAT_TYPE)
 public interface EnvelopeFormatFactory : IOFormatFactory<Envelope>, EnvelopeFormat {
-    override val name: Name get() = "envelope".asName()
     override val type: KType get() = typeOf<Envelope>()
 
     override fun build(context: Context, meta: Meta): EnvelopeFormat
@@ -51,6 +50,7 @@ public interface EnvelopeFormatFactory : IOFormatFactory<Envelope>, EnvelopeForm
     public fun peekFormat(io: IOPlugin, binary: Binary): EnvelopeFormat?
 
     public companion object {
+        public val ENVELOPE_FACTORY_NAME: Name = "envelope".asName()
         public const val ENVELOPE_FORMAT_TYPE: String = "io.format.envelope"
     }
 }
\ No newline at end of file
diff --git a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/TaggedEnvelopeFormat.kt b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/TaggedEnvelopeFormat.kt
index 96eaddb6..933dc343 100644
--- a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/TaggedEnvelopeFormat.kt
+++ b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/TaggedEnvelopeFormat.kt
@@ -92,7 +92,7 @@ public class TaggedEnvelopeFormat(
         val meta: Meta = metaFormat.readObject(metaBinary)
 
 
-        return PartialEnvelope(meta, version.tagSize + tag.metaSize, tag.dataSize)
+        return PartialEnvelope(meta, (version.tagSize + tag.metaSize).toInt(), tag.dataSize)
     }
 
     private data class Tag(
@@ -117,7 +117,7 @@ public class TaggedEnvelopeFormat(
         private const val START_SEQUENCE = "#~"
         private const val END_SEQUENCE = "~#\r\n"
 
-        override val name: Name = super.name + "tagged"
+        override val name: Name = EnvelopeFormatFactory.ENVELOPE_FACTORY_NAME + "tagged"
 
         override fun build(context: Context, meta: Meta): EnvelopeFormat {
             val io = context.io
diff --git a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/TaglessEnvelopeFormat.kt b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/TaglessEnvelopeFormat.kt
index 56e3f582..9a0f4a98 100644
--- a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/TaglessEnvelopeFormat.kt
+++ b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/TaglessEnvelopeFormat.kt
@@ -10,7 +10,7 @@ import space.kscience.dataforge.meta.get
 import space.kscience.dataforge.meta.isEmpty
 import space.kscience.dataforge.meta.string
 import space.kscience.dataforge.names.Name
-import space.kscience.dataforge.names.asName
+import space.kscience.dataforge.names.plus
 import kotlin.collections.set
 
 /**
@@ -33,7 +33,7 @@ public class TaglessEnvelopeFormat(
         output: Output,
         envelope: Envelope,
         metaFormatFactory: MetaFormatFactory,
-        formatMeta: Meta
+        formatMeta: Meta,
     ) {
         val metaFormat = metaFormatFactory.build(this.io.context, formatMeta)
 
@@ -66,13 +66,16 @@ public class TaglessEnvelopeFormat(
     }
 
     override fun readObject(input: Input): Envelope {
-        var line: String
-        do {
-            line = input.readSafeUtf8Line() // ?: error("Input does not contain tagless envelope header")
-        } while (!line.startsWith(TAGLESS_ENVELOPE_HEADER))
+        //read preamble
+        input.discardWithSeparator(
+            TAGLESS_ENVELOPE_HEADER.encodeToByteArray(),
+            atMost = 1024,
+            skipUntilEndOfLine = true
+        )
+
         val properties = HashMap<String, String>()
 
-        line = ""
+        var line = ""
         while (line.isBlank() || line.startsWith("#?")) {
             if (line.startsWith("#?")) {
                 val match = propertyPattern.find(line)
@@ -80,9 +83,17 @@ public class TaglessEnvelopeFormat(
                 val (key, value) = match.destructured
                 properties[key] = value
             }
-            //If can't read line, return envelope without data
-            if (input.endOfInput) return SimpleEnvelope(Meta.EMPTY, null)
-            line = input.readSafeUtf8Line()
+            try {
+                line = ByteArray {
+                    try {
+                        input.readBytesWithSeparatorTo(this, byteArrayOf('\n'.code.toByte()), 1024)
+                    } catch (ex: BufferLimitExceededException) {
+                        throw IllegalStateException("Property line exceeds maximum line length (1024)", ex)
+                    }
+                }.decodeToString().trim()
+            } catch (ex: EOFException) {
+                return SimpleEnvelope(Meta.EMPTY, Binary.EMPTY)
+            }
         }
 
         var meta: Meta = Meta.EMPTY
@@ -93,18 +104,16 @@ public class TaglessEnvelopeFormat(
             meta = if (metaSize != null) {
                 metaFormat.readObject(input.readBinary(metaSize))
             } else {
-                metaFormat.readObject(input)
+                error("Can't partially read an envelope with undefined meta size")
             }
         }
 
-        do {
-            try {
-                line = input.readSafeUtf8Line()
-            } catch (ex: EOFException) {
-                //returning an Envelope without data if end of input is reached
-                return SimpleEnvelope(meta, null)
-            }
-        } while (!line.startsWith(dataStart))
+        //skip until data start
+        input.discardWithSeparator(
+            dataStart.encodeToByteArray(),
+            atMost = 1024,
+            skipUntilEndOfLine = true
+        )
 
         val data: Binary = if (properties.containsKey(DATA_LENGTH_PROPERTY)) {
             input.readBinary(properties[DATA_LENGTH_PROPERTY]!!.toInt())
@@ -112,24 +121,27 @@ public class TaglessEnvelopeFormat(
 //            readByteArray(bytes)
 //            bytes.asBinary()
         } else {
-            Binary {
-                input.copyTo(this)
-            }
+            input.readBytes().asBinary()
         }
 
         return SimpleEnvelope(meta, data)
     }
 
+
     override fun readPartial(input: Input): PartialEnvelope {
-        var offset = 0u
-        var line: String
-        do {
-            line = input.readSafeUtf8Line()// ?: error("Input does not contain tagless envelope header")
-            offset += line.encodeToByteArray().size.toUInt()
-        } while (!line.startsWith(TAGLESS_ENVELOPE_HEADER))
+        var offset = 0
+
+        //read preamble
+
+        offset += input.discardWithSeparator(
+            TAGLESS_ENVELOPE_HEADER.encodeToByteArray(),
+            atMost = 1024,
+            skipUntilEndOfLine = true
+        )
+
         val properties = HashMap<String, String>()
 
-        line = ""
+        var line = ""
         while (line.isBlank() || line.startsWith("#?")) {
             if (line.startsWith("#?")) {
                 val match = propertyPattern.find(line)
@@ -138,10 +150,16 @@ public class TaglessEnvelopeFormat(
                 properties[key] = value
             }
             try {
-                line = input.readSafeUtf8Line()
-                offset += line.encodeToByteArray().size.toUInt()
+                line = ByteArray {
+                    val read = try {
+                        input.readBytesWithSeparatorTo(this, byteArrayOf('\n'.code.toByte()), 1024)
+                    } catch (ex: BufferLimitExceededException) {
+                        throw IllegalStateException("Property line exceeds maximum line length (1024)", ex)
+                    }
+                    offset += read
+                }.decodeToString().trim()
             } catch (ex: EOFException) {
-                return PartialEnvelope(Meta.EMPTY, offset.toUInt(), 0.toULong())
+                return PartialEnvelope(Meta.EMPTY, offset, 0.toULong())
             }
         }
 
@@ -151,18 +169,19 @@ public class TaglessEnvelopeFormat(
             val metaFormat = properties[META_TYPE_PROPERTY]?.let { io.resolveMetaFormat(it) } ?: JsonMetaFormat
             val metaSize = properties[META_LENGTH_PROPERTY]?.toInt()
             meta = if (metaSize != null) {
-                offset += metaSize.toUInt()
+                offset += metaSize
                 metaFormat.readObject(input.readBinary(metaSize))
             } else {
                 error("Can't partially read an envelope with undefined meta size")
             }
         }
 
-        do {
-            line = input.readSafeUtf8Line() //?: return PartialEnvelope(Meta.EMPTY, offset.toUInt(), 0.toULong())
-            offset += line.encodeToByteArray().size.toUInt()
-            //returning an Envelope without data if end of input is reached
-        } while (!line.startsWith(dataStart))
+        //skip until data start
+        offset += input.discardWithSeparator(
+            dataStart.encodeToByteArray(),
+            atMost = 1024,
+            skipUntilEndOfLine = true
+        )
 
         val dataSize = properties[DATA_LENGTH_PROPERTY]?.toULong()
         return PartialEnvelope(meta, offset, dataSize)
@@ -192,7 +211,7 @@ public class TaglessEnvelopeFormat(
 
         public const val code: Int = 0x4446544c //DFTL
 
-        override val name: Name = TAGLESS_ENVELOPE_TYPE.asName()
+        override val name: Name = EnvelopeFormatFactory.ENVELOPE_FACTORY_NAME + TAGLESS_ENVELOPE_TYPE
 
         override fun build(context: Context, meta: Meta): EnvelopeFormat = TaglessEnvelopeFormat(context.io, meta)
 
diff --git a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/ioMisc.kt b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/ioMisc.kt
index acb16916..e72c0eed 100644
--- a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/ioMisc.kt
+++ b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/ioMisc.kt
@@ -1,8 +1,10 @@
 package space.kscience.dataforge.io
 
+import io.ktor.utils.io.bits.Memory
 import io.ktor.utils.io.charsets.Charsets
 import io.ktor.utils.io.charsets.decodeExactBytes
 import io.ktor.utils.io.core.*
+import io.ktor.utils.io.core.internal.ChunkBuffer
 import space.kscience.dataforge.meta.Meta
 import space.kscience.dataforge.misc.DFExperimental
 import kotlin.math.min
@@ -86,11 +88,110 @@ public fun EnvelopeFormat.readBinary(binary: Binary): Envelope {
  * A zero-copy read from
  */
 @DFExperimental
-public fun IOPlugin.readEnvelopeBinary(
+public fun IOPlugin.readEnvelope(
     binary: Binary,
     readNonEnvelopes: Boolean = false,
     formatPicker: IOPlugin.(Binary) -> EnvelopeFormat? = IOPlugin::peekBinaryEnvelopeFormat,
 ): Envelope = formatPicker(binary)?.readBinary(binary) ?: if (readNonEnvelopes) {
     // if no format accepts file, read it as binary
     SimpleEnvelope(Meta.EMPTY, binary)
-} else error("Can't infer format for $binary")
\ No newline at end of file
+} else error("Can't infer format for $binary")
+
+@DFExperimental
+public fun IOPlugin.readEnvelope(
+    string: String,
+    readNonEnvelopes: Boolean = false,
+    formatPicker: IOPlugin.(Binary) -> EnvelopeFormat? = IOPlugin::peekBinaryEnvelopeFormat,
+): Envelope = readEnvelope(string.encodeToByteArray().asBinary(), readNonEnvelopes, formatPicker)
+
+
+private class RingByteArray(
+    private val buffer: ByteArray,
+    private var startIndex: Int = 0,
+    var size: Int = 0,
+) {
+    operator fun get(index: Int): Byte {
+        require(index >= 0) { "Index must be positive" }
+        require(index < size) { "Index $index is out of circular buffer size $size" }
+        return buffer[startIndex.forward(index)]
+    }
+
+    fun isFull(): Boolean = size == buffer.size
+
+    fun push(element: Byte) {
+        buffer[startIndex.forward(size)] = element
+        if (isFull()) startIndex++ else size++
+
+    }
+
+    private fun Int.forward(n: Int): Int = (this + n) % (buffer.size)
+
+    fun compare(inputArray: ByteArray): Boolean = when {
+        inputArray.size != buffer.size -> false
+        size < buffer.size -> false
+        else -> inputArray.indices.all { inputArray[it] == get(it) }
+    }
+}
+
+/**
+ * Read [Input] into [output] until designated multy-byte [separator] and optionally continues until
+ * the end of the line after it. Throw error if [separator] not found and [atMost] bytes are read.
+ * Also fails if [separator] not found until the end of input.
+ *
+ * Separator itself is not read into Output.
+ *
+ * @return bytes actually being read, including separator
+ */
+public fun Input.readBytesWithSeparatorTo(
+    output: Output,
+    separator: ByteArray,
+    atMost: Int = Int.MAX_VALUE,
+    skipUntilEndOfLine: Boolean = false,
+): Int {
+    var counter = 0
+    val rb = RingByteArray(ByteArray(separator.size))
+    var separatorFound = false
+    takeWhile { buffer ->
+        while (buffer.canRead()) {
+            val byte = buffer.readByte()
+            counter++
+            if (counter >= atMost) error("Maximum number of bytes to be read $atMost reached.")
+            //If end-of-line-search is on, terminate
+            if (separatorFound) {
+                if (endOfInput || byte == '\n'.code.toByte()) {
+                    return counter
+                }
+            } else {
+                rb.push(byte)
+                if (rb.compare(separator)) {
+                    separatorFound = true
+                    if (!skipUntilEndOfLine) {
+                        return counter
+                    }
+                } else if (rb.isFull()) {
+                    output.writeByte(rb[0])
+                }
+            }
+        }
+        !endOfInput
+    }
+    error("Read to the end of input without encountering ${separator.decodeToString()}")
+}
+
+public fun Input.discardWithSeparator(
+    separator: ByteArray,
+    atMost: Int = Int.MAX_VALUE,
+    skipUntilEndOfLine: Boolean = false,
+): Int {
+    val dummy: Output = object :Output(ChunkBuffer.Pool){
+        override fun closeDestination() {
+            // Do nothing
+        }
+
+        override fun flush(source: Memory, offset: Int, length: Int) {
+            // Do nothing
+        }
+    }
+
+    return readBytesWithSeparatorTo(dummy, separator, atMost, skipUntilEndOfLine)
+}
diff --git a/dataforge-io/src/commonTest/kotlin/space/kscience/dataforge/io/EnvelopeFormatTest.kt b/dataforge-io/src/commonTest/kotlin/space/kscience/dataforge/io/EnvelopeFormatTest.kt
index edadc7f8..f6106f4b 100644
--- a/dataforge-io/src/commonTest/kotlin/space/kscience/dataforge/io/EnvelopeFormatTest.kt
+++ b/dataforge-io/src/commonTest/kotlin/space/kscience/dataforge/io/EnvelopeFormatTest.kt
@@ -1,5 +1,6 @@
 package space.kscience.dataforge.io
 
+import io.ktor.utils.io.core.ByteReadPacket
 import io.ktor.utils.io.core.readDouble
 import io.ktor.utils.io.core.writeDouble
 import kotlin.test.Test
@@ -9,10 +10,10 @@ import kotlin.test.assertEquals
 class EnvelopeFormatTest {
     val envelope = Envelope {
         type = "test.format"
-        meta{
+        meta {
             "d" put 22.2
         }
-        data{
+        data {
             writeDouble(22.2)
 //            repeat(2000){
 //                writeInt(it)
@@ -21,12 +22,12 @@ class EnvelopeFormatTest {
     }
 
     @Test
-    fun testTaggedFormat(){
+    fun testTaggedFormat() {
         TaggedEnvelopeFormat.run {
             val byteArray = writeToByteArray(envelope)
             //println(byteArray.decodeToString())
             val res = readFromByteArray(byteArray)
-            assertEquals(envelope.meta,res.meta)
+            assertEquals(envelope.meta, res.meta)
             val double = res.data?.read {
                 readDouble()
             }
@@ -35,12 +36,14 @@ class EnvelopeFormatTest {
     }
 
     @Test
-    fun testTaglessFormat(){
+    fun testTaglessFormat() {
         TaglessEnvelopeFormat.run {
             val byteArray = writeToByteArray(envelope)
             //println(byteArray.decodeToString())
+            val partial = readPartial(ByteReadPacket(byteArray))
+            assertEquals(8, partial.dataSize?.toInt())
             val res = readFromByteArray(byteArray)
-            assertEquals(envelope.meta,res.meta)
+            assertEquals(envelope.meta, res.meta)
             val double = res.data?.read {
                 readDouble()
             }
diff --git a/dataforge-io/src/commonTest/kotlin/space/kscience/dataforge/io/IOTest.kt b/dataforge-io/src/commonTest/kotlin/space/kscience/dataforge/io/IOTest.kt
index 197d1c30..a2583bb1 100644
--- a/dataforge-io/src/commonTest/kotlin/space/kscience/dataforge/io/IOTest.kt
+++ b/dataforge-io/src/commonTest/kotlin/space/kscience/dataforge/io/IOTest.kt
@@ -2,8 +2,10 @@ package space.kscience.dataforge.io
 
 import io.ktor.utils.io.core.ByteReadPacket
 import io.ktor.utils.io.core.readBytes
+import io.ktor.utils.io.core.readUTF8Line
 import kotlin.test.Test
 import kotlin.test.assertEquals
+import kotlin.test.assertFails
 
 class IOTest {
     @Test
@@ -14,4 +16,42 @@ class IOTest {
         val second = input.readBytes(4)
         assertEquals(4.toByte(), second[0])
     }
+
+    @Test
+    fun readUntilSeparator() {
+        val source = """
+            aaa
+            bbb
+            ---
+            ccc
+            ddd
+        """.trimIndent()
+
+        val binary = source.encodeToByteArray().asBinary()
+
+        binary.read {
+            val array = ByteArray {
+                val read = readBytesWithSeparatorTo(this, "---".encodeToByteArray(),  skipUntilEndOfLine = true)
+                assertEquals(12, read)
+            }
+            assertEquals("""
+                aaa
+                bbb
+            """.trimIndent(),array.decodeToString().trim())
+            assertEquals("ccc", readUTF8Line()?.trim())
+        }
+
+        assertFails {
+            binary.read {
+                discardWithSeparator("---".encodeToByteArray(), atMost = 3)
+            }
+        }
+
+        assertFails {
+            binary.read{
+                discardWithSeparator("-+-".encodeToByteArray())
+            }
+        }
+
+    }
 }
\ No newline at end of file
diff --git a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/Workspace.kt b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/Workspace.kt
index 3aa2c0f4..6b120c76 100644
--- a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/Workspace.kt
+++ b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/Workspace.kt
@@ -1,6 +1,7 @@
 package space.kscience.dataforge.workspace
 
 import space.kscience.dataforge.context.ContextAware
+import space.kscience.dataforge.data.Data
 import space.kscience.dataforge.data.DataSet
 import space.kscience.dataforge.meta.Meta
 import space.kscience.dataforge.meta.MutableMeta
@@ -34,7 +35,7 @@ public interface Workspace : ContextAware, Provider {
         return when (target) {
             "target", Meta.TYPE -> targets.mapKeys { Name.parse(it.key)}
             Task.TYPE -> tasks
-            //Data.TYPE -> data.flow().toMap()
+            Data.TYPE -> data.dataSequence().associateBy { it.name }
             else -> emptyMap()
         }
     }
@@ -46,7 +47,7 @@ public interface Workspace : ContextAware, Provider {
     }
 
     public suspend fun produceData(taskName: Name, taskMeta: Meta, name: Name): TaskData<*>? =
-        produce(taskName, taskMeta).get(name)
+        produce(taskName, taskMeta)[name]
 
     public companion object {
         public const val TYPE: String = "workspace"

From 665f317e4ec281edf38438f47892daddb2a3211b Mon Sep 17 00:00:00 2001
From: Alexander Nozik <altavir@gmail.com>
Date: Sun, 1 May 2022 19:29:13 +0300
Subject: [PATCH 08/19] Remove obsolete `getData` for DataSet

---
 .../kscience/dataforge/actions/MapAction.kt      | 16 ++++++++++++----
 .../space/kscience/dataforge/data/DataTree.kt    |  2 --
 .../space/kscience/dataforge/data/NamedData.kt   |  3 +++
 .../space/kscience/dataforge/data/ActionsTest.kt |  4 ++--
 .../dataforge/data/DataTreeBuilderTest.kt        | 14 +++++++-------
 .../dataforge/workspace/DataPropagationTest.kt   |  2 +-
 .../kscience/dataforge/workspace/FileDataTest.kt |  8 ++++----
 .../dataforge/workspace/SimpleWorkspaceTest.kt   |  4 ++--
 8 files changed, 31 insertions(+), 22 deletions(-)

diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/MapAction.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/MapAction.kt
index 157564b1..08fa97b4 100644
--- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/MapAction.kt
+++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/MapAction.kt
@@ -27,13 +27,20 @@ public data class ActionEnv(
  * Action environment
  */
 @DFBuilder
-public class MapActionBuilder<T, R>(public var name: Name, public var meta: MutableMeta, public val actionMeta: Meta) {
+public class MapActionBuilder<T, R>(
+    public var name: Name,
+    public var meta: MutableMeta,
+    public val actionMeta: Meta,
+    public var outputType: KType
+) {
+
     public lateinit var result: suspend ActionEnv.(T) -> R
 
     /**
      * Calculate the result of goal
      */
-    public fun result(f: suspend ActionEnv.(T) -> R) {
+    public inline fun <reified R1: R> result(noinline f: suspend ActionEnv.(T) -> R1) {
+        outputType = typeOf<R1>()
         result = f;
     }
 }
@@ -57,7 +64,8 @@ internal class MapAction<in T : Any, out R : Any>(
             val builder = MapActionBuilder<T, R>(
                 data.name,
                 data.meta.toMutableMeta(), // using data meta
-                meta
+                meta,
+                outputType
             ).apply(block)
 
             //getting new name
@@ -67,7 +75,7 @@ internal class MapAction<in T : Any, out R : Any>(
             val newMeta = builder.meta.seal()
 
             @OptIn(DFInternal::class)
-            val newData = Data(outputType, newMeta, dependencies = listOf(data)) {
+            val newData = Data(builder.outputType, newMeta, dependencies = listOf(data)) {
                 builder.result(env, data.await())
             }
             //setting the data node
diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataTree.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataTree.kt
index aab14e25..3ef065e1 100644
--- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataTree.kt
+++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataTree.kt
@@ -73,8 +73,6 @@ public interface DataTree<out T : Any> : DataSet<T> {
     }
 }
 
-public suspend fun <T : Any> DataSet<T>.getData(name: String): Data<T>? = get(Name.parse(name))
-
 /**
  * Get a [DataTreeItem] with given [name] or null if the item does not exist
  */
diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/NamedData.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/NamedData.kt
index 59ae10a8..4c9d4bb3 100644
--- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/NamedData.kt
+++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/NamedData.kt
@@ -9,6 +9,9 @@ public interface NamedData<out T : Any> : Named, Data<T> {
     public val data: Data<T>
 }
 
+public operator fun NamedData<*>.component1(): Name = name
+public operator fun <T: Any> NamedData<T>.component2(): Data<T> = data
+
 private class NamedDataImpl<out T : Any>(
     override val name: Name,
     override val data: Data<T>,
diff --git a/dataforge-data/src/jvmTest/kotlin/space/kscience/dataforge/data/ActionsTest.kt b/dataforge-data/src/jvmTest/kotlin/space/kscience/dataforge/data/ActionsTest.kt
index da7ec0d9..c80f00c4 100644
--- a/dataforge-data/src/jvmTest/kotlin/space/kscience/dataforge/data/ActionsTest.kt
+++ b/dataforge-data/src/jvmTest/kotlin/space/kscience/dataforge/data/ActionsTest.kt
@@ -24,7 +24,7 @@ internal class ActionsTest {
         }
         runBlocking {
             val result = plusOne.execute(data)
-            assertEquals(2, result.getData("1")?.await())
+            assertEquals(2, result["1"]?.await())
         }
     }
 
@@ -36,7 +36,7 @@ internal class ActionsTest {
 
         val datum = runBlocking {
             val result = plusOne.execute(data, scope = this)
-            result.getData("1")?.await()
+            result["1"]?.await()
         }
         assertEquals(2, datum)
     }
diff --git a/dataforge-data/src/jvmTest/kotlin/space/kscience/dataforge/data/DataTreeBuilderTest.kt b/dataforge-data/src/jvmTest/kotlin/space/kscience/dataforge/data/DataTreeBuilderTest.kt
index 3e8d716e..65b43a6f 100644
--- a/dataforge-data/src/jvmTest/kotlin/space/kscience/dataforge/data/DataTreeBuilderTest.kt
+++ b/dataforge-data/src/jvmTest/kotlin/space/kscience/dataforge/data/DataTreeBuilderTest.kt
@@ -19,10 +19,10 @@ internal class DataTreeBuilderTest {
             static("c.f", "c.f")
         }
         runBlocking {
-            assertEquals("a", node.getData("primary.a")?.await())
-            assertEquals("b", node.getData("primary.b")?.await())
-            assertEquals("c.d", node.getData("c.d")?.await())
-            assertEquals("c.f", node.getData("c.f")?.await())
+            assertEquals("a", node["primary.a"]?.await())
+            assertEquals("b", node["primary.b"]?.await())
+            assertEquals("c.d", node["c.d"]?.await())
+            assertEquals("c.f", node["c.f"]?.await())
         }
     }
 
@@ -46,8 +46,8 @@ internal class DataTreeBuilderTest {
         }
 
         runBlocking {
-            assertEquals("a", node.getData("update.a")?.await())
-            assertEquals("a", node.getData("primary.a")?.await())
+            assertEquals("a", node.get("update.a")?.await())
+            assertEquals("a", node.get("primary.a")?.await())
         }
     }
 
@@ -80,7 +80,7 @@ internal class DataTreeBuilderTest {
                     }
                 }
                 updateJob.join()
-                assertEquals(9, rootNode.getData("sub.value")?.await())
+                assertEquals(9, rootNode["sub.value"]?.await())
                 cancel()
             }
         } catch (t: Throwable) {
diff --git a/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/DataPropagationTest.kt b/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/DataPropagationTest.kt
index 07b8fe36..1c1a849e 100644
--- a/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/DataPropagationTest.kt
+++ b/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/DataPropagationTest.kt
@@ -23,7 +23,7 @@ class DataPropagationTestPlugin : WorkspacePlugin() {
 
 
     val singleData by task<Int> {
-        workspace.data.select<Int>().getData("myData[12]")?.let {
+        workspace.data.select<Int>()["myData[12]"]?.let {
             data("result", it)
         }
     }
diff --git a/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/FileDataTest.kt b/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/FileDataTest.kt
index 4dbf3a85..612dad13 100644
--- a/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/FileDataTest.kt
+++ b/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/FileDataTest.kt
@@ -68,8 +68,8 @@ class FileDataTest {
                 writeDataDirectory(dir, dataNode, StringIOFormat)
                 println(dir.toUri().toString())
                 val reconstructed = readDataDirectory(dir, StringFormatResolver)
-                assertEquals(dataNode.getData("dir.a")?.meta, reconstructed.getData("dir.a")?.meta)
-                assertEquals(dataNode.getData("b")?.await(), reconstructed.getData("b")?.await())
+                assertEquals(dataNode["dir.a"]?.meta, reconstructed["dir.a"]?.meta)
+                assertEquals(dataNode["b"]?.await(), reconstructed["b"]?.await())
             }
         }
     }
@@ -84,8 +84,8 @@ class FileDataTest {
                 writeZip(zip, dataNode, StringIOFormat)
                 println(zip.toUri().toString())
                 val reconstructed = readDataDirectory(zip, StringFormatResolver)
-                assertEquals(dataNode.getData("dir.a")?.meta, reconstructed.getData("dir.a")?.meta)
-                assertEquals(dataNode.getData("b")?.await(), reconstructed.getData("b")?.await())
+                assertEquals(dataNode["dir.a"]?.meta, reconstructed["dir.a"]?.meta)
+                assertEquals(dataNode["b"]?.await(), reconstructed["b"]?.await())
             }
         }
     }
diff --git a/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/SimpleWorkspaceTest.kt b/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/SimpleWorkspaceTest.kt
index 3bd09251..2187cbe2 100644
--- a/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/SimpleWorkspaceTest.kt
+++ b/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/SimpleWorkspaceTest.kt
@@ -134,8 +134,8 @@ class SimpleWorkspaceTest {
 
         val delta by task<Int> {
             val averaged = from(averageByGroup)
-            val even = averaged.getData("event")!!
-            val odd = averaged.getData("odd")!!
+            val even = averaged["event"]!!
+            val odd = averaged["odd"]!!
             val res = even.combine(odd) { l, r ->
                 l - r
             }

From f0820a3bedbb2dd08d8bec12b00da3b4443318fd Mon Sep 17 00:00:00 2001
From: Alexander Nozik <altavir@gmail.com>
Date: Sun, 1 May 2022 20:23:37 +0300
Subject: [PATCH 09/19] Reify types for action builders

---
 CHANGELOG.md                                  |  1 +
 .../kscience/dataforge/actions/MapAction.kt   |  9 +++--
 .../dataforge/actions/ReduceAction.kt         | 34 ++++++++++++-------
 .../kscience/dataforge/actions/SplitAction.kt | 19 +++++++----
 4 files changed, 42 insertions(+), 21 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 26208ef7..d3c1b1e4 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,6 +5,7 @@
 - Add `specOrNull` delegate to meta and Scheme
 - Suspended read methods to the `Binary`
 - Synchronously accessed `meta` to all `DataSet`s
+- More fine-grained types in Action builders.
 
 ### Changed
 - `Factory` is now `fun interface` and uses `build` instead of `invoke`. `invoke moved to an extension.
diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/MapAction.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/MapAction.kt
index 08fa97b4..8aae6be8 100644
--- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/MapAction.kt
+++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/MapAction.kt
@@ -31,15 +31,20 @@ public class MapActionBuilder<T, R>(
     public var name: Name,
     public var meta: MutableMeta,
     public val actionMeta: Meta,
-    public var outputType: KType
+    @PublishedApi internal var outputType: KType,
 ) {
 
     public lateinit var result: suspend ActionEnv.(T) -> R
 
+    internal fun <R1 : R> result(outputType: KType, f: suspend ActionEnv.(T) -> R1) {
+        this.outputType = outputType
+        result = f;
+    }
+
     /**
      * Calculate the result of goal
      */
-    public inline fun <reified R1: R> result(noinline f: suspend ActionEnv.(T) -> R1) {
+    public inline fun <reified R1 : R> result(noinline f: suspend ActionEnv.(T) -> R1) {
         outputType = typeOf<R1>()
         result = f;
     }
diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/ReduceAction.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/ReduceAction.kt
index 5bb22bfc..64605ad1 100644
--- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/ReduceAction.kt
+++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/ReduceAction.kt
@@ -14,13 +14,23 @@ import kotlin.reflect.KType
 import kotlin.reflect.typeOf
 
 
-public class JoinGroup<T : Any, R : Any>(public var name: String, internal val set: DataSet<T>) {
+public class JoinGroup<T : Any, R : Any>(
+    public var name: String,
+    internal val set: DataSet<T>,
+    @PublishedApi internal var outputType: KType,
+) {
 
     public var meta: MutableMeta = MutableMeta()
 
     public lateinit var result: suspend ActionEnv.(Map<Name, T>) -> R
 
-    public fun result(f: suspend ActionEnv.(Map<Name, T>) -> R) {
+    internal fun <R1 : R> result(outputType: KType, f: suspend ActionEnv.(Map<Name, T>) -> R1) {
+        this.outputType = outputType
+        this.result = f;
+    }
+
+    public inline fun <reified R1 : R> result(noinline f: suspend ActionEnv.(Map<Name, T>) -> R1) {
+        outputType = typeOf<R1>()
         this.result = f;
     }
 
@@ -28,9 +38,9 @@ public class JoinGroup<T : Any, R : Any>(public var name: String, internal val s
 
 @DFBuilder
 public class ReduceGroupBuilder<T : Any, R : Any>(
-    private val inputType: KType,
     private val scope: CoroutineScope,
     public val actionMeta: Meta,
+    private val outputType: KType
 ) {
     private val groupRules: MutableList<suspend (DataSet<T>) -> List<JoinGroup<T, R>>> = ArrayList();
 
@@ -40,7 +50,7 @@ public class ReduceGroupBuilder<T : Any, R : Any>(
     public fun byValue(tag: String, defaultTag: String = "@default", action: JoinGroup<T, R>.() -> Unit) {
         groupRules += { node ->
             GroupRule.byMetaValue(scope, tag, defaultTag).gather(node).map {
-                JoinGroup<T, R>(it.key, it.value).apply(action)
+                JoinGroup<T, R>(it.key, it.value, outputType).apply(action)
             }
         }
     }
@@ -52,7 +62,7 @@ public class ReduceGroupBuilder<T : Any, R : Any>(
     ) {
         groupRules += { source ->
             listOf(
-                JoinGroup<T, R>(groupName, source.filter(filter)).apply(action)
+                JoinGroup<T, R>(groupName, source.filter(filter), outputType).apply(action)
             )
         }
     }
@@ -62,19 +72,17 @@ public class ReduceGroupBuilder<T : Any, R : Any>(
      */
     public fun result(resultName: String, f: suspend ActionEnv.(Map<Name, T>) -> R) {
         groupRules += { node ->
-            listOf(JoinGroup<T, R>(resultName, node).apply { result(f) })
+            listOf(JoinGroup<T, R>(resultName, node, outputType).apply { result(outputType, f) })
         }
     }
 
-    internal suspend fun buildGroups(input: DataSet<T>): List<JoinGroup<T, R>> {
-        return groupRules.flatMap { it.invoke(input) }
-    }
+    internal suspend fun buildGroups(input: DataSet<T>): List<JoinGroup<T, R>> =
+        groupRules.flatMap { it.invoke(input) }
 
 }
 
 @PublishedApi
 internal class ReduceAction<T : Any, R : Any>(
-    private val inputType: KType,
     outputType: KType,
     private val action: ReduceGroupBuilder<T, R>.() -> Unit,
 ) : CachingAction<T, R>(outputType) {
@@ -82,7 +90,7 @@ internal class ReduceAction<T : Any, R : Any>(
 
 
     override fun CoroutineScope.transform(set: DataSet<T>, meta: Meta, key: Name): Flow<NamedData<R>> = flow {
-        ReduceGroupBuilder<T, R>(inputType, this@transform, meta).apply(action).buildGroups(set).forEach { group ->
+        ReduceGroupBuilder<T, R>(this@transform, meta, outputType).apply(action).buildGroups(set).forEach { group ->
             val dataFlow: Map<Name, Data<T>> = group.set.dataSequence().fold(HashMap()) { acc, value ->
                 acc.apply {
                     acc[value.name] = value.data
@@ -95,7 +103,7 @@ internal class ReduceAction<T : Any, R : Any>(
 
             val env = ActionEnv(Name.parse(groupName), groupMeta, meta)
             @OptIn(DFInternal::class) val res: Data<R> = dataFlow.reduceToData(
-                outputType,
+                group.outputType,
                 meta = groupMeta
             ) { group.result.invoke(env, it) }
 
@@ -111,4 +119,4 @@ internal class ReduceAction<T : Any, R : Any>(
 @Suppress("FunctionName")
 public inline fun <reified T : Any, reified R : Any> Action.Companion.reduce(
     noinline builder: ReduceGroupBuilder<T, R>.() -> Unit,
-): Action<T, R> = ReduceAction(typeOf<T>(), typeOf<R>(), builder)
+): Action<T, R> = ReduceAction(typeOf<R>(), builder)
diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/SplitAction.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/SplitAction.kt
index 88d047cd..0f7446a5 100644
--- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/SplitAction.kt
+++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/SplitAction.kt
@@ -1,7 +1,6 @@
 package space.kscience.dataforge.actions
 
 import kotlinx.coroutines.CoroutineScope
-import kotlinx.coroutines.FlowPreview
 import kotlinx.coroutines.launch
 import space.kscience.dataforge.data.*
 import space.kscience.dataforge.meta.Laminate
@@ -18,10 +17,15 @@ import kotlin.reflect.typeOf
 
 public class SplitBuilder<T : Any, R : Any>(public val name: Name, public val meta: Meta) {
 
-    public class FragmentRule<T : Any, R : Any>(public val name: Name, public var meta: MutableMeta) {
+    public class FragmentRule<T : Any, R : Any>(
+        public val name: Name,
+        public var meta: MutableMeta,
+        @PublishedApi internal var outputType: KType,
+    ) {
         public lateinit var result: suspend (T) -> R
 
-        public fun result(f: suspend (T) -> R) {
+        public inline fun <reified R1 : R> result(noinline f: suspend (T) -> R1) {
+            this.outputType = typeOf<R1>()
             result = f;
         }
     }
@@ -47,7 +51,6 @@ internal class SplitAction<T : Any, R : Any>(
     private val action: SplitBuilder<T, R>.() -> Unit,
 ) : Action<T, R> {
 
-    @OptIn(FlowPreview::class)
     override suspend fun execute(
         dataSet: DataSet<T>,
         meta: Meta,
@@ -62,7 +65,11 @@ internal class SplitAction<T : Any, R : Any>(
 
             // apply individual fragment rules to result
             return split.fragments.entries.asSequence().map { (fragmentName, rule) ->
-                val env = SplitBuilder.FragmentRule<T, R>(fragmentName, laminate.toMutableMeta()).apply(rule)
+                val env = SplitBuilder.FragmentRule<T, R>(
+                    fragmentName,
+                    laminate.toMutableMeta(),
+                    outputType
+                ).apply(rule)
                 //data.map<R>(outputType, meta = env.meta) { env.result(it) }.named(fragmentName)
                 @OptIn(DFInternal::class) Data(outputType, meta = env.meta, dependencies = listOf(data)) {
                     env.result(data.await())
@@ -71,7 +78,7 @@ internal class SplitAction<T : Any, R : Any>(
         }
 
         return ActiveDataTree<R>(outputType) {
-            populateWith(dataSet.dataSequence().flatMap (transform = ::splitOne))
+            populateWith(dataSet.dataSequence().flatMap(transform = ::splitOne))
             scope?.launch {
                 dataSet.updates.collect { name ->
                     //clear old nodes

From bedab0dc86901d8ebabd3981b02fee001e05d81f Mon Sep 17 00:00:00 2001
From: Alexander Nozik <altavir@gmail.com>
Date: Tue, 3 May 2022 17:42:00 +0300
Subject: [PATCH 10/19] Remove experimental flag from YAML

---
 build.gradle.kts                                             | 2 +-
 .../kotlin/space/kscience/dataforge/actions/MapAction.kt     | 5 ++++-
 .../kscience/dataforge/io/yaml/FrontMatterEnvelopeFormat.kt  | 3 +--
 .../space/kscience/dataforge/io/yaml/YamlMetaFormat.kt       | 1 -
 .../kotlin/space/kscience/dataforge/io/yaml/YamlPlugin.kt    | 1 -
 gradle/wrapper/gradle-wrapper.properties                     | 2 +-
 6 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/build.gradle.kts b/build.gradle.kts
index 9db78013..e577b9d2 100644
--- a/build.gradle.kts
+++ b/build.gradle.kts
@@ -4,7 +4,7 @@ plugins {
 
 allprojects {
     group = "space.kscience"
-    version = "0.6.0-dev-3"
+    version = "0.6.0-dev-4"
     repositories{
         mavenCentral()
     }
diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/MapAction.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/MapAction.kt
index 8aae6be8..891e99f0 100644
--- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/MapAction.kt
+++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/MapAction.kt
@@ -36,7 +36,10 @@ public class MapActionBuilder<T, R>(
 
     public lateinit var result: suspend ActionEnv.(T) -> R
 
-    internal fun <R1 : R> result(outputType: KType, f: suspend ActionEnv.(T) -> R1) {
+    /**
+     * Set unsafe [outputType] for the resulting data. Be sure that it is correct.
+     */
+    public fun <R1 : R> result(outputType: KType, f: suspend ActionEnv.(T) -> R1) {
         this.outputType = outputType
         result = f;
     }
diff --git a/dataforge-io/dataforge-io-yaml/src/commonMain/kotlin/space/kscience/dataforge/io/yaml/FrontMatterEnvelopeFormat.kt b/dataforge-io/dataforge-io-yaml/src/commonMain/kotlin/space/kscience/dataforge/io/yaml/FrontMatterEnvelopeFormat.kt
index 2734ebd9..7f705a71 100644
--- a/dataforge-io/dataforge-io-yaml/src/commonMain/kotlin/space/kscience/dataforge/io/yaml/FrontMatterEnvelopeFormat.kt
+++ b/dataforge-io/dataforge-io-yaml/src/commonMain/kotlin/space/kscience/dataforge/io/yaml/FrontMatterEnvelopeFormat.kt
@@ -14,7 +14,6 @@ import space.kscience.dataforge.misc.DFExperimental
 import space.kscience.dataforge.names.Name
 import space.kscience.dataforge.names.plus
 
-@DFExperimental
 public class FrontMatterEnvelopeFormat(
     private val io: IOPlugin,
     private val meta: Meta = Meta.EMPTY,
@@ -101,7 +100,7 @@ public class FrontMatterEnvelopeFormat(
             envelope: Envelope,
             metaFormatFactory: MetaFormatFactory,
             formatMeta: Meta,
-        ): Unit = FrontMatterEnvelopeFormat.default.writeEnvelope(output, envelope, metaFormatFactory, formatMeta)
+        ): Unit = default.writeEnvelope(output, envelope, metaFormatFactory, formatMeta)
 
 
         override fun readObject(input: Input): Envelope = default.readObject(input)
diff --git a/dataforge-io/dataforge-io-yaml/src/commonMain/kotlin/space/kscience/dataforge/io/yaml/YamlMetaFormat.kt b/dataforge-io/dataforge-io-yaml/src/commonMain/kotlin/space/kscience/dataforge/io/yaml/YamlMetaFormat.kt
index 148a2b87..b7c7c3e4 100644
--- a/dataforge-io/dataforge-io-yaml/src/commonMain/kotlin/space/kscience/dataforge/io/yaml/YamlMetaFormat.kt
+++ b/dataforge-io/dataforge-io-yaml/src/commonMain/kotlin/space/kscience/dataforge/io/yaml/YamlMetaFormat.kt
@@ -95,7 +95,6 @@ public fun YamlMap.toMeta(): Meta = YamlMeta(this)
 /**
  * Represent meta as Yaml
  */
-@DFExperimental
 public class YamlMetaFormat(private val meta: Meta) : MetaFormat {
 
     override fun writeMeta(output: Output, meta: Meta, descriptor: MetaDescriptor?) {
diff --git a/dataforge-io/dataforge-io-yaml/src/commonMain/kotlin/space/kscience/dataforge/io/yaml/YamlPlugin.kt b/dataforge-io/dataforge-io-yaml/src/commonMain/kotlin/space/kscience/dataforge/io/yaml/YamlPlugin.kt
index dea3a38c..c26487a5 100644
--- a/dataforge-io/dataforge-io-yaml/src/commonMain/kotlin/space/kscience/dataforge/io/yaml/YamlPlugin.kt
+++ b/dataforge-io/dataforge-io-yaml/src/commonMain/kotlin/space/kscience/dataforge/io/yaml/YamlPlugin.kt
@@ -13,7 +13,6 @@ import space.kscience.dataforge.names.Name
 import space.kscience.dataforge.names.asName
 import kotlin.reflect.KClass
 
-@DFExperimental
 public class YamlPlugin(meta: Meta) : AbstractPlugin(meta) {
     public val io: IOPlugin by require(IOPlugin)
 
diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties
index ffed3a25..aa991fce 100644
--- a/gradle/wrapper/gradle-wrapper.properties
+++ b/gradle/wrapper/gradle-wrapper.properties
@@ -1,5 +1,5 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-7.2-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-7.4.2-bin.zip
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists

From 0622bacc4d26cdfdb47a9e1b95910f19aa67c48c Mon Sep 17 00:00:00 2001
From: Alexander Nozik <altavir@gmail.com>
Date: Wed, 4 May 2022 17:27:56 +0300
Subject: [PATCH 11/19] Refactor DataSet. Remove suspends where it is possible.

---
 CHANGELOG.md                                  |   1 +
 build.gradle.kts                              |   5 +-
 .../kscience/dataforge/actions/Action.kt      |  23 ++--
 .../dataforge/actions/CachingAction.kt        |  53 ++++++++
 .../kscience/dataforge/actions/MapAction.kt   |  27 ++--
 .../dataforge/actions/ReduceAction.kt         |  22 ++--
 .../kscience/dataforge/actions/SplitAction.kt |  26 ++--
 .../kscience/dataforge/data/ActiveDataTree.kt | 103 ---------------
 .../kscience/dataforge/data/CachingAction.kt  |  51 --------
 .../space/kscience/dataforge/data/DataSet.kt  |  15 ++-
 .../kscience/dataforge/data/DataSetBuilder.kt |  60 +++++----
 .../dataforge/data/DataSourceBuilder.kt       | 122 ++++++++++++++++++
 .../kscience/dataforge/data/GroupRule.kt      |  41 +++---
 .../kscience/dataforge/data/StaticDataTree.kt |  29 ++---
 .../kscience/dataforge/data/dataFilter.kt     |  31 +++--
 .../kscience/dataforge/data/dataTransform.kt  |   2 +-
 .../dataforge/data/actionInContext.kt         |   2 +
 .../data/{select.kt => dataFilterJvm.kt}      |  37 +++---
 .../dataforge/data/dataSetBuilderInContext.kt |  10 +-
 .../kscience/dataforge/data/ActionsTest.kt    |  35 ++---
 .../dataforge/data/DataTreeBuilderTest.kt     |   4 +-
 .../dataforge/workspace/WorkspaceBuilder.kt   |  15 ++-
 .../dataforge/workspace/workspaceJvm.kt       |  23 ++--
 .../workspace/DataPropagationTest.kt          |   4 +-
 .../dataforge/workspace/FileDataTest.kt       |  19 ++-
 .../workspace/SimpleWorkspaceTest.kt          |  10 +-
 26 files changed, 418 insertions(+), 352 deletions(-)
 create mode 100644 dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/CachingAction.kt
 delete mode 100644 dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/ActiveDataTree.kt
 delete mode 100644 dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/CachingAction.kt
 create mode 100644 dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSourceBuilder.kt
 create mode 100644 dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/actionInContext.kt
 rename dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/{select.kt => dataFilterJvm.kt} (58%)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index d3c1b1e4..9f73ebf1 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -14,6 +14,7 @@
 - DataSet `getData` is no longer suspended and renamed to `get`
 - DataSet operates with sequences of data instead of flows
 - PartialEnvelope uses `Int` instead `UInt`.
+- `ActiveDataSet` renamed to `DataSource`
 
 ### Deprecated
 
diff --git a/build.gradle.kts b/build.gradle.kts
index e577b9d2..a7140851 100644
--- a/build.gradle.kts
+++ b/build.gradle.kts
@@ -4,10 +4,7 @@ plugins {
 
 allprojects {
     group = "space.kscience"
-    version = "0.6.0-dev-4"
-    repositories{
-        mavenCentral()
-    }
+    version = "0.6.0-dev-5"
 }
 
 subprojects {
diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/Action.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/Action.kt
index 50dc56c8..4fed8e51 100644
--- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/Action.kt
+++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/Action.kt
@@ -1,6 +1,5 @@
 package space.kscience.dataforge.actions
 
-import kotlinx.coroutines.CoroutineScope
 import space.kscience.dataforge.data.DataSet
 import space.kscience.dataforge.meta.Meta
 import space.kscience.dataforge.misc.DFExperimental
@@ -9,13 +8,12 @@ import space.kscience.dataforge.misc.DFExperimental
  * A simple data transformation on a data node. Actions should avoid doing actual dependency evaluation in [execute].
  */
 public interface Action<in T : Any, out R : Any> {
+
     /**
-     * Transform the data in the node, producing a new node. By default it is assumed that all calculations are lazy
+     * Transform the data in the node, producing a new node. By default, it is assumed that all calculations are lazy
      * so not actual computation is started at this moment.
-     *
-     * [scope] context used to compute the initial result, also it is used for updates propagation
      */
-    public suspend fun execute(dataSet: DataSet<T>, meta: Meta = Meta.EMPTY, scope: CoroutineScope? = null): DataSet<R>
+    public fun execute(dataSet: DataSet<T>, meta: Meta = Meta.EMPTY): DataSet<R>
 
     public companion object
 }
@@ -26,16 +24,17 @@ public interface Action<in T : Any, out R : Any> {
 public infix fun <T : Any, I : Any, R : Any> Action<T, I>.then(action: Action<I, R>): Action<T, R> {
     // TODO introduce composite action and add optimize by adding action to the list
     return object : Action<T, R> {
-        override suspend fun execute(dataSet: DataSet<T>, meta: Meta, scope: CoroutineScope?): DataSet<R> {
-            return action.execute(this@then.execute(dataSet, meta, scope), meta, scope)
-        }
+
+        override fun execute(
+            dataSet: DataSet<T>,
+            meta: Meta,
+        ): DataSet<R> = action.execute(this@then.execute(dataSet, meta), meta)
     }
 }
 
 @DFExperimental
-public suspend fun <T : Any, R : Any> DataSet<T>.transformWith(
-    action: Action<T, R>,
+public operator fun <T : Any, R : Any> Action<T, R>.invoke(
+    dataSet: DataSet<T>,
     meta: Meta = Meta.EMPTY,
-    scope: CoroutineScope? = null,
-): DataSet<R> = action.execute(this, meta, scope)
+): DataSet<R> = execute(dataSet, meta)
 
diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/CachingAction.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/CachingAction.kt
new file mode 100644
index 00000000..469b6e60
--- /dev/null
+++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/CachingAction.kt
@@ -0,0 +1,53 @@
+package space.kscience.dataforge.actions
+
+import kotlinx.coroutines.launch
+import space.kscience.dataforge.data.*
+import space.kscience.dataforge.meta.Meta
+import space.kscience.dataforge.names.Name
+import space.kscience.dataforge.names.startsWith
+import kotlin.reflect.KType
+
+/**
+ * Remove all values with keys starting with [name]
+ */
+internal fun MutableMap<Name, *>.removeWhatStartsWith(name: Name) {
+    val toRemove = keys.filter { it.startsWith(name) }
+    toRemove.forEach(::remove)
+}
+
+/**
+ * An action that caches results on-demand and recalculates them on source push
+ */
+public abstract class CachingAction<in T : Any, out R : Any>(
+    public val outputType: KType,
+) : Action<T, R> {
+
+    protected abstract fun transform(
+        set: DataSet<T>,
+        meta: Meta,
+        key: Name = Name.EMPTY,
+    ): Sequence<NamedData<R>>
+
+    override fun execute(
+        dataSet: DataSet<T>,
+        meta: Meta,
+    ): DataSet<R> = if (dataSet is DataSource) {
+        DataSourceBuilder<R>(outputType, dataSet.coroutineContext).apply {
+            populateFrom(transform(dataSet, meta))
+
+            launch {
+                dataSet.updates.collect {
+                    //clear old nodes
+                    remove(it)
+                    //collect new items
+                    populateFrom(transform(dataSet, meta, it))
+                    //FIXME if the target is data, updates are fired twice
+                }
+            }
+        }
+    } else {
+        DataTree<R>(outputType) {
+            populateFrom(transform(dataSet, meta))
+        }
+    }
+}
diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/MapAction.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/MapAction.kt
index 891e99f0..f2165e7d 100644
--- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/MapAction.kt
+++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/MapAction.kt
@@ -1,6 +1,5 @@
 package space.kscience.dataforge.actions
 
-import kotlinx.coroutines.CoroutineScope
 import kotlinx.coroutines.launch
 import space.kscience.dataforge.data.*
 import space.kscience.dataforge.meta.Meta
@@ -59,11 +58,11 @@ internal class MapAction<in T : Any, out R : Any>(
     private val block: MapActionBuilder<T, R>.() -> Unit,
 ) : Action<T, R> {
 
-    override suspend fun execute(
+    override fun execute(
         dataSet: DataSet<T>,
         meta: Meta,
-        scope: CoroutineScope?,
     ): DataSet<R> {
+
         fun mapOne(data: NamedData<T>): NamedData<R> {
             // Creating a new environment for action using **old** name, old meta and task meta
             val env = ActionEnv(data.name, data.meta, meta)
@@ -92,16 +91,22 @@ internal class MapAction<in T : Any, out R : Any>(
 
         val sequence = dataSet.dataSequence().map(::mapOne)
 
-        return ActiveDataTree(outputType) {
-            populateWith(sequence)
-            scope?.launch {
-                dataSet.updates.collect { name ->
-                    //clear old nodes
-                    remove(name)
-                    //collect new items
-                    populateWith(dataSet.children(name).map(::mapOne))
+        return if (dataSet is DataSource ) {
+            ActiveDataTree(outputType, dataSet) {
+                populateFrom(sequence)
+                launch {
+                    dataSet.updates.collect { name ->
+                        //clear old nodes
+                        remove(name)
+                        //collect new items
+                        populateFrom(dataSet.children(name).map(::mapOne))
+                    }
                 }
             }
+        } else {
+            DataTree(outputType) {
+                populateFrom(sequence)
+            }
         }
     }
 }
diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/ReduceAction.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/ReduceAction.kt
index 64605ad1..3e1ec62f 100644
--- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/ReduceAction.kt
+++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/ReduceAction.kt
@@ -1,8 +1,5 @@
 package space.kscience.dataforge.actions
 
-import kotlinx.coroutines.CoroutineScope
-import kotlinx.coroutines.flow.Flow
-import kotlinx.coroutines.flow.flow
 import space.kscience.dataforge.data.*
 import space.kscience.dataforge.meta.Meta
 import space.kscience.dataforge.meta.MutableMeta
@@ -38,18 +35,17 @@ public class JoinGroup<T : Any, R : Any>(
 
 @DFBuilder
 public class ReduceGroupBuilder<T : Any, R : Any>(
-    private val scope: CoroutineScope,
     public val actionMeta: Meta,
-    private val outputType: KType
+    private val outputType: KType,
 ) {
-    private val groupRules: MutableList<suspend (DataSet<T>) -> List<JoinGroup<T, R>>> = ArrayList();
+    private val groupRules: MutableList<(DataSet<T>) -> List<JoinGroup<T, R>>> = ArrayList();
 
     /**
      * introduce grouping by meta value
      */
     public fun byValue(tag: String, defaultTag: String = "@default", action: JoinGroup<T, R>.() -> Unit) {
         groupRules += { node ->
-            GroupRule.byMetaValue(scope, tag, defaultTag).gather(node).map {
+            GroupRule.byMetaValue(tag, defaultTag).gather(node).map {
                 JoinGroup<T, R>(it.key, it.value, outputType).apply(action)
             }
         }
@@ -57,12 +53,12 @@ public class ReduceGroupBuilder<T : Any, R : Any>(
 
     public fun group(
         groupName: String,
-        filter: (Name, Data<T>) -> Boolean,
+        predicate: (Name, Meta) -> Boolean,
         action: JoinGroup<T, R>.() -> Unit,
     ) {
         groupRules += { source ->
             listOf(
-                JoinGroup<T, R>(groupName, source.filter(filter), outputType).apply(action)
+                JoinGroup<T, R>(groupName, source.filter(predicate), outputType).apply(action)
             )
         }
     }
@@ -76,7 +72,7 @@ public class ReduceGroupBuilder<T : Any, R : Any>(
         }
     }
 
-    internal suspend fun buildGroups(input: DataSet<T>): List<JoinGroup<T, R>> =
+    internal fun buildGroups(input: DataSet<T>): List<JoinGroup<T, R>> =
         groupRules.flatMap { it.invoke(input) }
 
 }
@@ -89,8 +85,8 @@ internal class ReduceAction<T : Any, R : Any>(
     //TODO optimize reduction. Currently the whole action recalculates on push
 
 
-    override fun CoroutineScope.transform(set: DataSet<T>, meta: Meta, key: Name): Flow<NamedData<R>> = flow {
-        ReduceGroupBuilder<T, R>(this@transform, meta, outputType).apply(action).buildGroups(set).forEach { group ->
+    override fun transform(set: DataSet<T>, meta: Meta, key: Name): Sequence<NamedData<R>> = sequence {
+        ReduceGroupBuilder<T, R>(meta, outputType).apply(action).buildGroups(set).forEach { group ->
             val dataFlow: Map<Name, Data<T>> = group.set.dataSequence().fold(HashMap()) { acc, value ->
                 acc.apply {
                     acc[value.name] = value.data
@@ -107,7 +103,7 @@ internal class ReduceAction<T : Any, R : Any>(
                 meta = groupMeta
             ) { group.result.invoke(env, it) }
 
-            emit(res.named(env.name))
+            yield(res.named(env.name))
         }
     }
 }
diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/SplitAction.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/SplitAction.kt
index 0f7446a5..471a8057 100644
--- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/SplitAction.kt
+++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/SplitAction.kt
@@ -1,6 +1,5 @@
 package space.kscience.dataforge.actions
 
-import kotlinx.coroutines.CoroutineScope
 import kotlinx.coroutines.launch
 import space.kscience.dataforge.data.*
 import space.kscience.dataforge.meta.Laminate
@@ -51,10 +50,9 @@ internal class SplitAction<T : Any, R : Any>(
     private val action: SplitBuilder<T, R>.() -> Unit,
 ) : Action<T, R> {
 
-    override suspend fun execute(
+    override fun execute(
         dataSet: DataSet<T>,
         meta: Meta,
-        scope: CoroutineScope?,
     ): DataSet<R> {
 
         fun splitOne(data: NamedData<T>): Sequence<NamedData<R>> {
@@ -77,16 +75,22 @@ internal class SplitAction<T : Any, R : Any>(
             }
         }
 
-        return ActiveDataTree<R>(outputType) {
-            populateWith(dataSet.dataSequence().flatMap(transform = ::splitOne))
-            scope?.launch {
-                dataSet.updates.collect { name ->
-                    //clear old nodes
-                    remove(name)
-                    //collect new items
-                    populateWith(dataSet.children(name).flatMap(transform = ::splitOne))
+        return if (dataSet is DataSource) {
+            ActiveDataTree<R>(outputType, dataSet) {
+                populateFrom(dataSet.dataSequence().flatMap(transform = ::splitOne))
+                launch {
+                    dataSet.updates.collect { name ->
+                        //clear old nodes
+                        remove(name)
+                        //collect new items
+                        populateFrom(dataSet.children(name).flatMap(transform = ::splitOne))
+                    }
                 }
             }
+        } else {
+            DataTree<R>(outputType) {
+                populateFrom(dataSet.dataSequence().flatMap(transform = ::splitOne))
+            }
         }
     }
 }
diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/ActiveDataTree.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/ActiveDataTree.kt
deleted file mode 100644
index e403b625..00000000
--- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/ActiveDataTree.kt
+++ /dev/null
@@ -1,103 +0,0 @@
-package space.kscience.dataforge.data
-
-import kotlinx.coroutines.flow.*
-import kotlinx.coroutines.sync.Mutex
-import kotlinx.coroutines.sync.withLock
-import space.kscience.dataforge.meta.*
-import space.kscience.dataforge.names.*
-import kotlin.reflect.KType
-import kotlin.reflect.typeOf
-
-/**
- * A mutable [DataTree].
- */
-public class ActiveDataTree<T : Any>(
-    override val dataType: KType,
-) : DataTree<T>, DataSetBuilder<T>, ActiveDataSet<T> {
-    private val mutex = Mutex()
-    private val treeItems = HashMap<NameToken, DataTreeItem<T>>()
-
-    override val items: Map<NameToken, DataTreeItem<T>>
-        get() = treeItems.filter { !it.key.body.startsWith("@") }
-
-    private val _updates = MutableSharedFlow<Name>()
-
-    override val updates: Flow<Name>
-        get() = _updates
-
-    private suspend fun remove(token: NameToken) = mutex.withLock {
-        if (treeItems.remove(token) != null) {
-            _updates.emit(token.asName())
-        }
-    }
-
-    override suspend fun remove(name: Name) {
-        if (name.isEmpty()) error("Can't remove the root node")
-        (getItem(name.cutLast()).tree as? ActiveDataTree)?.remove(name.lastOrNull()!!)
-    }
-
-    private suspend fun set(token: NameToken, data: Data<T>) = mutex.withLock {
-        treeItems[token] = DataTreeItem.Leaf(data)
-    }
-
-    private suspend fun getOrCreateNode(token: NameToken): ActiveDataTree<T> =
-        (treeItems[token] as? DataTreeItem.Node<T>)?.tree as? ActiveDataTree<T>
-            ?: ActiveDataTree<T>(dataType).also {
-                mutex.withLock {
-                    treeItems[token] = DataTreeItem.Node(it)
-                }
-            }
-
-    private suspend fun getOrCreateNode(name: Name): ActiveDataTree<T> = when (name.length) {
-        0 -> this
-        1 -> getOrCreateNode(name.firstOrNull()!!)
-        else -> getOrCreateNode(name.firstOrNull()!!).getOrCreateNode(name.cutFirst())
-    }
-
-    override suspend fun data(name: Name, data: Data<T>?) {
-        if (data == null) {
-            remove(name)
-        } else {
-            when (name.length) {
-                0 -> error("Can't add data with empty name")
-                1 -> set(name.firstOrNull()!!, data)
-                2 -> getOrCreateNode(name.cutLast()).set(name.lastOrNull()!!, data)
-            }
-        }
-        _updates.emit(name)
-    }
-
-    override suspend fun meta(name: Name, meta: Meta) {
-        val item = getItem(name)
-        if(item is DataTreeItem.Leaf) error("TODO: Can't change meta of existing leaf item.")
-        data(name + DataTree.META_ITEM_NAME_TOKEN, Data.empty(meta))
-    }
-}
-
-/**
- * Create a dynamic tree. Initial data is placed synchronously. Updates are propagated via [updatesScope]
- */
-@Suppress("FunctionName")
-public suspend fun <T : Any> ActiveDataTree(
-    type: KType,
-    block: suspend ActiveDataTree<T>.() -> Unit,
-): ActiveDataTree<T> {
-    val tree = ActiveDataTree<T>(type)
-    tree.block()
-    return tree
-}
-
-@Suppress("FunctionName")
-public suspend inline fun <reified T : Any> ActiveDataTree(
-    crossinline block: suspend ActiveDataTree<T>.() -> Unit,
-): ActiveDataTree<T> = ActiveDataTree<T>(typeOf<T>()).apply { block() }
-
-public suspend inline fun <reified T : Any> ActiveDataTree<T>.emit(
-    name: Name,
-    noinline block: suspend ActiveDataTree<T>.() -> Unit,
-): Unit = node(name, ActiveDataTree(typeOf<T>(), block))
-
-public suspend inline fun <reified T : Any> ActiveDataTree<T>.emit(
-    name: String,
-    noinline block: suspend ActiveDataTree<T>.() -> Unit,
-): Unit = node(Name.parse(name), ActiveDataTree(typeOf<T>(), block))
diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/CachingAction.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/CachingAction.kt
deleted file mode 100644
index 5e7b62bf..00000000
--- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/CachingAction.kt
+++ /dev/null
@@ -1,51 +0,0 @@
-package space.kscience.dataforge.data
-
-import kotlinx.coroutines.CoroutineScope
-import kotlinx.coroutines.coroutineScope
-import kotlinx.coroutines.flow.Flow
-import space.kscience.dataforge.actions.Action
-import space.kscience.dataforge.meta.Meta
-import space.kscience.dataforge.names.Name
-import space.kscience.dataforge.names.startsWith
-import kotlin.reflect.KType
-
-/**
- * Remove all values with keys starting with [name]
- */
-internal fun MutableMap<Name, *>.removeWhatStartsWith(name: Name) {
-    val toRemove = keys.filter { it.startsWith(name) }
-    toRemove.forEach(::remove)
-}
-
-/**
- * An action that caches results on-demand and recalculates them on source push
- */
-public abstract class CachingAction<in T : Any, out R : Any>(
-    public val outputType: KType,
-) : Action<T, R> {
-
-    protected abstract fun CoroutineScope.transform(
-        set: DataSet<T>,
-        meta: Meta,
-        key: Name = Name.EMPTY,
-    ): Flow<NamedData<R>>
-
-    override suspend fun execute(
-        dataSet: DataSet<T>,
-        meta: Meta,
-        scope: CoroutineScope?,
-    ): DataSet<R> = ActiveDataTree<R>(outputType) {
-        coroutineScope {
-            populateWith(transform(dataSet, meta))
-        }
-        scope?.let {
-            dataSet.updates.collect {
-                //clear old nodes
-                remove(it)
-                //collect new items
-                populateWith(scope.transform(dataSet, meta, it))
-                //FIXME if the target is data, updates are fired twice
-            }
-        }
-    }
-}
diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSet.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSet.kt
index 45e60bb4..f468632f 100644
--- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSet.kt
+++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSet.kt
@@ -62,7 +62,11 @@ DataSet<out T : Any> {
 
 public operator fun <T: Any> DataSet<T>.get(name:String): Data<T>? = get(name.parseAsName())
 
-public interface ActiveDataSet<T : Any> : DataSet<T> {
+/**
+ * A [DataSet] with propagated updates.
+ */
+public interface DataSource<T : Any> : DataSet<T>, CoroutineScope {
+
     /**
      * A flow of updated item names. Updates are propagated in a form of [Flow] of names of updated nodes.
      * Those can include new data items and replacement of existing ones. The replaced items could update existing data content
@@ -70,9 +74,16 @@ public interface ActiveDataSet<T : Any> : DataSet<T> {
      *
      */
     public val updates: Flow<Name>
+
+    /**
+     * Stop generating updates from this [DataSource]
+     */
+    public fun close(){
+        coroutineContext[Job]?.cancel()
+    }
 }
 
-public val <T : Any> DataSet<T>.updates: Flow<Name> get() = if (this is ActiveDataSet) updates else emptyFlow()
+public val <T : Any> DataSet<T>.updates: Flow<Name> get() = if (this is DataSource) updates else emptyFlow()
 
 /**
  * Flow all data nodes with names starting with [branchName]
diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSetBuilder.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSetBuilder.kt
index d279e34c..cfcc1c97 100644
--- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSetBuilder.kt
+++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSetBuilder.kt
@@ -1,7 +1,5 @@
 package space.kscience.dataforge.data
 
-import kotlinx.coroutines.coroutineScope
-import kotlinx.coroutines.flow.Flow
 import space.kscience.dataforge.meta.Meta
 import space.kscience.dataforge.meta.MutableMeta
 import space.kscience.dataforge.misc.DFExperimental
@@ -16,14 +14,14 @@ public interface DataSetBuilder<in T : Any> {
     /**
      * Remove all data items starting with [name]
      */
-    public suspend fun remove(name: Name)
+    public fun remove(name: Name)
 
-    public suspend fun data(name: Name, data: Data<T>?)
+    public fun data(name: Name, data: Data<T>?)
 
     /**
      * Set a current state of given [dataSet] into a branch [name]. Does not propagate updates
      */
-    public suspend fun node(name: Name, dataSet: DataSet<T>) {
+    public fun node(name: Name, dataSet: DataSet<T>) {
         //remove previous items
         if (name != Name.EMPTY) {
             remove(name)
@@ -38,19 +36,19 @@ public interface DataSetBuilder<in T : Any> {
     /**
      * Set meta for the given node
      */
-    public suspend fun meta(name: Name, meta: Meta)
+    public fun meta(name: Name, meta: Meta)
 
 }
 
 /**
  * Define meta in this [DataSet]
  */
-public suspend fun <T : Any> DataSetBuilder<T>.meta(value: Meta): Unit = meta(Name.EMPTY, value)
+public fun <T : Any> DataSetBuilder<T>.meta(value: Meta): Unit = meta(Name.EMPTY, value)
 
 /**
  * Define meta in this [DataSet]
  */
-public suspend fun <T : Any> DataSetBuilder<T>.meta(mutableMeta: MutableMeta.() -> Unit): Unit = meta(Meta(mutableMeta))
+public fun <T : Any> DataSetBuilder<T>.meta(mutableMeta: MutableMeta.() -> Unit): Unit = meta(Meta(mutableMeta))
 
 @PublishedApi
 internal class SubSetBuilder<in T : Any>(
@@ -59,52 +57,52 @@ internal class SubSetBuilder<in T : Any>(
 ) : DataSetBuilder<T> {
     override val dataType: KType get() = parent.dataType
 
-    override suspend fun remove(name: Name) {
+    override fun remove(name: Name) {
         parent.remove(branch + name)
     }
 
-    override suspend fun data(name: Name, data: Data<T>?) {
+    override fun data(name: Name, data: Data<T>?) {
         parent.data(branch + name, data)
     }
 
-    override suspend fun node(name: Name, dataSet: DataSet<T>) {
+    override fun node(name: Name, dataSet: DataSet<T>) {
         parent.node(branch + name, dataSet)
     }
 
-    override suspend fun meta(name: Name, meta: Meta) {
+    override fun meta(name: Name, meta: Meta) {
         parent.meta(branch + name, meta)
     }
 }
 
-public suspend inline fun <T : Any> DataSetBuilder<T>.node(
+public inline fun <T : Any> DataSetBuilder<T>.node(
     name: Name,
-    crossinline block: suspend DataSetBuilder<T>.() -> Unit,
+    crossinline block: DataSetBuilder<T>.() -> Unit,
 ) {
     if (name.isEmpty()) block() else SubSetBuilder(this, name).block()
 }
 
 
-public suspend fun <T : Any> DataSetBuilder<T>.data(name: String, value: Data<T>) {
+public fun <T : Any> DataSetBuilder<T>.data(name: String, value: Data<T>) {
     data(Name.parse(name), value)
 }
 
-public suspend fun <T : Any> DataSetBuilder<T>.node(name: String, set: DataSet<T>) {
+public fun <T : Any> DataSetBuilder<T>.node(name: String, set: DataSet<T>) {
     node(Name.parse(name), set)
 }
 
-public suspend inline fun <T : Any> DataSetBuilder<T>.node(
+public inline fun <T : Any> DataSetBuilder<T>.node(
     name: String,
-    crossinline block: suspend DataSetBuilder<T>.() -> Unit,
+    crossinline block: DataSetBuilder<T>.() -> Unit,
 ): Unit = node(Name.parse(name), block)
 
-public suspend fun <T : Any> DataSetBuilder<T>.set(value: NamedData<T>) {
+public fun <T : Any> DataSetBuilder<T>.set(value: NamedData<T>) {
     data(value.name, value.data)
 }
 
 /**
  * Produce lazy [Data] and emit it into the [DataSetBuilder]
  */
-public suspend inline fun <reified T : Any> DataSetBuilder<T>.produce(
+public inline fun <reified T : Any> DataSetBuilder<T>.produce(
     name: String,
     meta: Meta = Meta.EMPTY,
     noinline producer: suspend () -> T,
@@ -113,7 +111,7 @@ public suspend inline fun <reified T : Any> DataSetBuilder<T>.produce(
     data(name, data)
 }
 
-public suspend inline fun <reified T : Any> DataSetBuilder<T>.produce(
+public inline fun <reified T : Any> DataSetBuilder<T>.produce(
     name: Name,
     meta: Meta = Meta.EMPTY,
     noinline producer: suspend () -> T,
@@ -125,19 +123,19 @@ public suspend inline fun <reified T : Any> DataSetBuilder<T>.produce(
 /**
  * Emit a static data with the fixed value
  */
-public suspend inline fun <reified T : Any> DataSetBuilder<T>.static(
+public inline fun <reified T : Any> DataSetBuilder<T>.static(
     name: String,
     data: T,
     meta: Meta = Meta.EMPTY,
 ): Unit = data(name, Data.static(data, meta))
 
-public suspend inline fun <reified T : Any> DataSetBuilder<T>.static(
+public inline fun <reified T : Any> DataSetBuilder<T>.static(
     name: Name,
     data: T,
     meta: Meta = Meta.EMPTY,
 ): Unit = data(name, Data.static(data, meta))
 
-public suspend inline fun <reified T : Any> DataSetBuilder<T>.static(
+public inline fun <reified T : Any> DataSetBuilder<T>.static(
     name: String,
     data: T,
     mutableMeta: MutableMeta.() -> Unit,
@@ -147,20 +145,20 @@ public suspend inline fun <reified T : Any> DataSetBuilder<T>.static(
  * Update data with given node data and meta with node meta.
  */
 @DFExperimental
-public suspend fun <T : Any> DataSetBuilder<T>.populateFrom(tree: DataSet<T>): Unit = coroutineScope {
+public fun <T : Any> DataSetBuilder<T>.populateFrom(tree: DataSet<T>): Unit {
     tree.dataSequence().forEach {
         //TODO check if the place is occupied
         data(it.name, it.data)
     }
 }
 
-public suspend fun <T : Any> DataSetBuilder<T>.populateWith(flow: Flow<NamedData<T>>) {
-    flow.collect {
-        data(it.name, it.data)
-    }
-}
+//public fun <T : Any> DataSetBuilder<T>.populateFrom(flow: Flow<NamedData<T>>) {
+//    flow.collect {
+//        data(it.name, it.data)
+//    }
+//}
 
-public suspend fun <T : Any> DataSetBuilder<T>.populateWith(sequence: Sequence<NamedData<T>>) {
+public fun <T : Any> DataSetBuilder<T>.populateFrom(sequence: Sequence<NamedData<T>>) {
     sequence.forEach {
         data(it.name, it.data)
     }
diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSourceBuilder.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSourceBuilder.kt
new file mode 100644
index 00000000..23b44aa0
--- /dev/null
+++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSourceBuilder.kt
@@ -0,0 +1,122 @@
+package space.kscience.dataforge.data
+
+import kotlinx.coroutines.CoroutineScope
+import kotlinx.coroutines.Job
+import kotlinx.coroutines.flow.MutableSharedFlow
+import kotlinx.coroutines.flow.SharedFlow
+import kotlinx.coroutines.launch
+import space.kscience.dataforge.meta.Meta
+import space.kscience.dataforge.names.*
+import kotlin.collections.set
+import kotlin.coroutines.CoroutineContext
+import kotlin.coroutines.coroutineContext
+import kotlin.jvm.Synchronized
+import kotlin.reflect.KType
+import kotlin.reflect.typeOf
+
+/**
+ * A mutable [DataTree] that propagates updates
+ */
+public class DataSourceBuilder<T : Any>(
+    override val dataType: KType,
+    coroutineContext: CoroutineContext,
+) : DataTree<T>, DataSetBuilder<T>, DataSource<T> {
+    override val coroutineContext: CoroutineContext =
+        coroutineContext + Job(coroutineContext[Job]) + GoalExecutionRestriction()
+
+    private val treeItems = HashMap<NameToken, DataTreeItem<T>>()
+
+    override val items: Map<NameToken, DataTreeItem<T>>
+        get() = treeItems.filter { !it.key.body.startsWith("@") }
+
+    private val _updates = MutableSharedFlow<Name>()
+
+    override val updates: SharedFlow<Name>
+        get() = _updates
+
+    @Synchronized
+    private fun remove(token: NameToken) {
+        if (treeItems.remove(token) != null) {
+            launch {
+                _updates.emit(token.asName())
+            }
+        }
+    }
+
+    override fun remove(name: Name) {
+        if (name.isEmpty()) error("Can't remove the root node")
+        (getItem(name.cutLast()).tree as? DataSourceBuilder)?.remove(name.lastOrNull()!!)
+    }
+
+    @Synchronized
+    private fun set(token: NameToken, data: Data<T>) {
+        treeItems[token] = DataTreeItem.Leaf(data)
+    }
+
+    @Synchronized
+    private fun set(token: NameToken, node: DataTree<T>) {
+        treeItems[token] = DataTreeItem.Node(node)
+    }
+
+    private fun getOrCreateNode(token: NameToken): DataSourceBuilder<T> =
+        (treeItems[token] as? DataTreeItem.Node<T>)?.tree as? DataSourceBuilder<T>
+            ?: DataSourceBuilder<T>(dataType, coroutineContext).also { set(token, it) }
+
+    private fun getOrCreateNode(name: Name): DataSourceBuilder<T> = when (name.length) {
+        0 -> this
+        1 -> getOrCreateNode(name.firstOrNull()!!)
+        else -> getOrCreateNode(name.firstOrNull()!!).getOrCreateNode(name.cutFirst())
+    }
+
+    override fun data(name: Name, data: Data<T>?) {
+        if (data == null) {
+            remove(name)
+        } else {
+            when (name.length) {
+                0 -> error("Can't add data with empty name")
+                1 -> set(name.firstOrNull()!!, data)
+                2 -> getOrCreateNode(name.cutLast()).set(name.lastOrNull()!!, data)
+            }
+        }
+        launch {
+            _updates.emit(name)
+        }
+    }
+
+    override fun meta(name: Name, meta: Meta) {
+        val item = getItem(name)
+        if (item is DataTreeItem.Leaf) error("TODO: Can't change meta of existing leaf item.")
+        data(name + DataTree.META_ITEM_NAME_TOKEN, Data.empty(meta))
+    }
+}
+
+/**
+ * Create a dynamic tree. Initial data is placed synchronously.
+ */
+@Suppress("FunctionName")
+public fun <T : Any> ActiveDataTree(
+    type: KType,
+    parent: CoroutineScope,
+    block: DataSourceBuilder<T>.() -> Unit,
+): DataSourceBuilder<T> {
+    val tree = DataSourceBuilder<T>(type, parent.coroutineContext)
+    tree.block()
+    return tree
+}
+
+@Suppress("FunctionName")
+public suspend inline fun <reified T : Any> ActiveDataTree(
+    crossinline block: DataSourceBuilder<T>.() -> Unit = {},
+): DataSourceBuilder<T> = DataSourceBuilder<T>(typeOf<T>(), coroutineContext).apply { block() }
+
+public inline fun <reified T : Any> DataSourceBuilder<T>.emit(
+    name: Name,
+    parent: CoroutineScope,
+    noinline block: DataSourceBuilder<T>.() -> Unit,
+): Unit = node(name, ActiveDataTree(typeOf<T>(), parent, block))
+
+public inline fun <reified T : Any> DataSourceBuilder<T>.emit(
+    name: String,
+    parent: CoroutineScope,
+    noinline block: DataSourceBuilder<T>.() -> Unit,
+): Unit = node(Name.parse(name), ActiveDataTree(typeOf<T>(), parent, block))
diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/GroupRule.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/GroupRule.kt
index 5ef8a6d5..c00fac8d 100644
--- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/GroupRule.kt
+++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/GroupRule.kt
@@ -15,13 +15,12 @@
  */
 package space.kscience.dataforge.data
 
-import kotlinx.coroutines.CoroutineScope
 import kotlinx.coroutines.launch
 import space.kscience.dataforge.meta.get
 import space.kscience.dataforge.meta.string
 
 public interface GroupRule {
-    public suspend fun <T : Any> gather(set: DataSet<T>): Map<String, DataSet<T>>
+    public fun <T : Any> gather(set: DataSet<T>): Map<String, DataSet<T>>
 
     public companion object {
         /**
@@ -33,31 +32,43 @@ public interface GroupRule {
          * @return
          */
         public fun byMetaValue(
-            scope: CoroutineScope,
             key: String,
             defaultTagValue: String,
         ): GroupRule = object : GroupRule {
 
-            override suspend fun <T : Any> gather(
+            override fun <T : Any> gather(
                 set: DataSet<T>,
             ): Map<String, DataSet<T>> {
-                val map = HashMap<String, ActiveDataTree<T>>()
+                val map = HashMap<String, DataSet<T>>()
 
-                set.dataSequence().forEach { data ->
-                    val tagValue = data.meta[key]?.string ?: defaultTagValue
-                    map.getOrPut(tagValue) { ActiveDataTree(set.dataType) }.data(data.name, data.data)
-                }
+                if (set is DataSource) {
+                    set.dataSequence().forEach { data ->
+                        val tagValue: String = data.meta[key]?.string ?: defaultTagValue
+                        (map.getOrPut(tagValue) { DataSourceBuilder(set.dataType, set.coroutineContext) } as DataSourceBuilder<T>)
+                            .data(data.name, data.data)
 
-                scope.launch {
-                    set.updates.collect { name ->
-                        val data = set.get(name)
+                        set.launch {
+                            set.updates.collect { name ->
+                                val dataUpdate = set[name]
 
-                        @Suppress("NULLABLE_EXTENSION_OPERATOR_WITH_SAFE_CALL_RECEIVER")
-                        val tagValue = data?.meta?.get(key)?.string ?: defaultTagValue
-                        map.getOrPut(tagValue) { ActiveDataTree(set.dataType) }.data(name, data)
+                                val updateTagValue = dataUpdate?.meta?.get(key)?.string ?: defaultTagValue
+                                map.getOrPut(updateTagValue) {
+                                    ActiveDataTree(set.dataType, this) {
+                                        data(name, dataUpdate)
+                                    }
+                                }
+                            }
+                        }
+                    }
+                } else {
+                    set.dataSequence().forEach { data ->
+                        val tagValue: String = data.meta[key]?.string ?: defaultTagValue
+                        (map.getOrPut(tagValue) { StaticDataTree(set.dataType) } as StaticDataTree<T>)
+                            .data(data.name, data.data)
                     }
                 }
 
+
                 return map
             }
         }
diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/StaticDataTree.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/StaticDataTree.kt
index 06b1ff6f..2353f97d 100644
--- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/StaticDataTree.kt
+++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/StaticDataTree.kt
@@ -1,6 +1,5 @@
 package space.kscience.dataforge.data
 
-import kotlinx.coroutines.coroutineScope
 import space.kscience.dataforge.meta.Meta
 import space.kscience.dataforge.misc.DFExperimental
 import space.kscience.dataforge.names.*
@@ -17,7 +16,7 @@ internal class StaticDataTree<T : Any>(
     override val items: Map<NameToken, DataTreeItem<T>>
         get() = _items.filter { !it.key.body.startsWith("@") }
 
-    override suspend fun remove(name: Name) {
+    override fun remove(name: Name) {
         when (name.length) {
             0 -> error("Can't remove root tree node")
             1 -> _items.remove(name.firstOrNull()!!)
@@ -36,7 +35,7 @@ internal class StaticDataTree<T : Any>(
         else -> getOrCreateNode(name.cutLast()).getOrCreateNode(name.lastOrNull()!!.asName())
     }
 
-    private suspend fun set(name: Name, item: DataTreeItem<T>?) {
+    private fun set(name: Name, item: DataTreeItem<T>?) {
         if (name.isEmpty()) error("Can't set top level tree node")
         if (item == null) {
             remove(name)
@@ -45,41 +44,39 @@ internal class StaticDataTree<T : Any>(
         }
     }
 
-    override suspend fun data(name: Name, data: Data<T>?) {
+    override fun data(name: Name, data: Data<T>?) {
         set(name, data?.let { DataTreeItem.Leaf(it) })
     }
 
-    override suspend fun node(name: Name, dataSet: DataSet<T>) {
+    override fun node(name: Name, dataSet: DataSet<T>) {
         if (dataSet is StaticDataTree) {
             set(name, DataTreeItem.Node(dataSet))
         } else {
-            coroutineScope {
-                dataSet.dataSequence().forEach {
-                    data(name + it.name, it.data)
-                }
+            dataSet.dataSequence().forEach {
+                data(name + it.name, it.data)
             }
         }
     }
 
-    override suspend fun meta(name: Name, meta: Meta) {
+    override fun meta(name: Name, meta: Meta) {
         val item = getItem(name)
-        if(item is DataTreeItem.Leaf) TODO("Can't change meta of existing leaf item.")
+        if (item is DataTreeItem.Leaf) TODO("Can't change meta of existing leaf item.")
         data(name + DataTree.META_ITEM_NAME_TOKEN, Data.empty(meta))
     }
 }
 
 @Suppress("FunctionName")
-public suspend fun <T : Any> DataTree(
+public inline fun <T : Any> DataTree(
     dataType: KType,
-    block: suspend DataSetBuilder<T>.() -> Unit,
+    block: DataSetBuilder<T>.() -> Unit,
 ): DataTree<T> = StaticDataTree<T>(dataType).apply { block() }
 
 @Suppress("FunctionName")
-public suspend inline fun <reified T : Any> DataTree(
-    noinline block: suspend DataSetBuilder<T>.() -> Unit,
+public inline fun <reified T : Any> DataTree(
+    noinline block: DataSetBuilder<T>.() -> Unit,
 ): DataTree<T> = DataTree(typeOf<T>(), block)
 
 @OptIn(DFExperimental::class)
-public suspend fun <T : Any> DataSet<T>.seal(): DataTree<T> = DataTree(dataType) {
+public fun <T : Any> DataSet<T>.seal(): DataTree<T> = DataTree(dataType) {
     populateFrom(this@seal)
 }
\ No newline at end of file
diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataFilter.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataFilter.kt
index 5ee7027d..f5037918 100644
--- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataFilter.kt
+++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataFilter.kt
@@ -10,6 +10,8 @@ import space.kscience.dataforge.names.Name
 import space.kscience.dataforge.names.isEmpty
 import space.kscience.dataforge.names.plus
 import space.kscience.dataforge.names.removeHeadOrNull
+import kotlin.coroutines.CoroutineContext
+import kotlin.coroutines.EmptyCoroutineContext
 import kotlin.reflect.KType
 
 
@@ -17,34 +19,42 @@ import kotlin.reflect.KType
  * A stateless filtered [DataSet]
  */
 public fun <T : Any> DataSet<T>.filter(
-    predicate: (Name, Data<T>) -> Boolean,
-): ActiveDataSet<T> = object : ActiveDataSet<T> {
+    predicate: (Name, Meta) -> Boolean,
+): DataSource<T> = object : DataSource<T> {
 
     override val dataType: KType get() = this@filter.dataType
 
+    override val coroutineContext: CoroutineContext
+        get() = (this@filter as? DataSource)?.coroutineContext ?: EmptyCoroutineContext
+
+
     override val meta: Meta get() = this@filter.meta
 
     override fun dataSequence(): Sequence<NamedData<T>> =
-        this@filter.dataSequence().filter { predicate(it.name, it.data) }
+        this@filter.dataSequence().filter { predicate(it.name, it.meta) }
 
     override fun get(name: Name): Data<T>? = this@filter.get(name)?.takeIf {
-        predicate(name, it)
+        predicate(name, it.meta)
     }
 
     override val updates: Flow<Name> = this@filter.updates.filter flowFilter@{ name ->
-        val theData = this@filter.get(name) ?: return@flowFilter false
-        predicate(name, theData)
+        val theData = this@filter[name] ?: return@flowFilter false
+        predicate(name, theData.meta)
     }
 }
 
 /**
  * Generate a wrapper data set with a given name prefix appended to all names
  */
-public fun <T : Any> DataSet<T>.withNamePrefix(prefix: Name): DataSet<T> = if (prefix.isEmpty()) this
-else object : ActiveDataSet<T> {
+public fun <T : Any> DataSet<T>.withNamePrefix(prefix: Name): DataSet<T> = if (prefix.isEmpty()) {
+    this
+} else object : DataSource<T> {
 
     override val dataType: KType get() = this@withNamePrefix.dataType
 
+    override val coroutineContext: CoroutineContext
+        get() = (this@withNamePrefix as? DataSource)?.coroutineContext ?: EmptyCoroutineContext
+
     override val meta: Meta get() = this@withNamePrefix.meta
 
 
@@ -62,9 +72,12 @@ else object : ActiveDataSet<T> {
  */
 public fun <T : Any> DataSet<T>.branch(branchName: Name): DataSet<T> = if (branchName.isEmpty()) {
     this
-} else object : ActiveDataSet<T> {
+} else object : DataSource<T> {
     override val dataType: KType get() = this@branch.dataType
 
+    override val coroutineContext: CoroutineContext
+        get() = (this@branch as? DataSource)?.coroutineContext ?: EmptyCoroutineContext
+
     override val meta: Meta get() = this@branch.meta
 
     override fun dataSequence(): Sequence<NamedData<T>> = this@branch.dataSequence().mapNotNull {
diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataTransform.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataTransform.kt
index c8180eb5..b4e58c2d 100644
--- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataTransform.kt
+++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataTransform.kt
@@ -144,7 +144,7 @@ public suspend fun <T : Any, R : Any> DataSet<T>.map(
     metaTransform: MutableMeta.() -> Unit = {},
     block: suspend (T) -> R,
 ): DataTree<R> = DataTree<R>(outputType) {
-    populateWith(
+    populateFrom(
         dataSequence().map {
             val newMeta = it.meta.toMutableMeta().apply(metaTransform).seal()
             Data(outputType, newMeta, coroutineContext, listOf(it)) {
diff --git a/dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/actionInContext.kt b/dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/actionInContext.kt
new file mode 100644
index 00000000..33731a95
--- /dev/null
+++ b/dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/actionInContext.kt
@@ -0,0 +1,2 @@
+package space.kscience.dataforge.data
+
diff --git a/dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/select.kt b/dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/dataFilterJvm.kt
similarity index 58%
rename from dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/select.kt
rename to dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/dataFilterJvm.kt
index f32ff3f3..5efe6d2a 100644
--- a/dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/select.kt
+++ b/dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/dataFilterJvm.kt
@@ -6,6 +6,8 @@ import space.kscience.dataforge.meta.Meta
 import space.kscience.dataforge.misc.DFExperimental
 import space.kscience.dataforge.names.Name
 import space.kscience.dataforge.names.matches
+import kotlin.coroutines.CoroutineContext
+import kotlin.coroutines.EmptyCoroutineContext
 import kotlin.reflect.KType
 import kotlin.reflect.full.isSubtypeOf
 import kotlin.reflect.typeOf
@@ -28,46 +30,47 @@ private fun <R : Any> Data<*>.castOrNull(type: KType): Data<R>? =
  * Select all data matching given type and filters. Does not modify paths
  *
  * @param namePattern a name match patter according to [Name.matches]
- * @param filter addition filtering condition based on item name and meta. By default, accepts all
+ * @param predicate addition filtering condition based on item name and meta. By default, accepts all
  */
 @OptIn(DFExperimental::class)
-public fun <R : Any> DataSet<*>.select(
+public fun <R : Any> DataSet<*>.filterIsInstance(
     type: KType,
-    namePattern: Name? = null,
-    filter: (name: Name, meta: Meta) -> Boolean = { _, _ -> true },
-): ActiveDataSet<R> = object : ActiveDataSet<R> {
+    predicate: (name: Name, meta: Meta) -> Boolean = { _, _ -> true },
+): DataSource<R> = object : DataSource<R> {
     override val dataType = type
 
-    override val meta: Meta get() = this@select.meta
+    override val coroutineContext: CoroutineContext
+        get() = (this@filterIsInstance as? DataSource)?.coroutineContext ?: EmptyCoroutineContext
+
+    override val meta: Meta get() = this@filterIsInstance.meta
 
     private fun checkDatum(name: Name, datum: Data<*>): Boolean = datum.type.isSubtypeOf(type)
-            && (namePattern == null || name.matches(namePattern))
-            && filter(name, datum.meta)
+            && predicate(name, datum.meta)
 
-    override fun dataSequence(): Sequence<NamedData<R>> = this@select.dataSequence().filter {
+    override fun dataSequence(): Sequence<NamedData<R>> = this@filterIsInstance.dataSequence().filter {
         checkDatum(it.name, it.data)
     }.map {
         @Suppress("UNCHECKED_CAST")
         it as NamedData<R>
     }
 
-    override fun get(name: Name): Data<R>? = this@select[name]?.let { datum ->
+    override fun get(name: Name): Data<R>? = this@filterIsInstance[name]?.let { datum ->
         if (checkDatum(name, datum)) datum.castOrNull(type) else null
     }
 
-    override val updates: Flow<Name> = this@select.updates.filter {
-        val datum = this@select[it] ?: return@filter false
-        checkDatum(it, datum)
+    override val updates: Flow<Name> = this@filterIsInstance.updates.filter { name ->
+        get(name)?.let { datum ->
+            checkDatum(name, datum)
+        } ?: false
     }
 }
 
 /**
  * Select a single datum of the appropriate type
  */
-public inline fun <reified R : Any> DataSet<*>.select(
-    namePattern: Name? = null,
-    noinline filter: (name: Name, meta: Meta) -> Boolean = { _, _ -> true },
-): DataSet<R> = select(typeOf<R>(), namePattern, filter)
+public inline fun <reified R : Any> DataSet<*>.filterIsInstance(
+    noinline predicate: (name: Name, meta: Meta) -> Boolean = { _, _ -> true },
+): DataSet<R> = filterIsInstance(typeOf<R>(), predicate)
 
 /**
  * Select a single datum if it is present and of given [type]
diff --git a/dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/dataSetBuilderInContext.kt b/dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/dataSetBuilderInContext.kt
index 51bfa187..3fc68141 100644
--- a/dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/dataSetBuilderInContext.kt
+++ b/dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/dataSetBuilderInContext.kt
@@ -10,24 +10,24 @@ import space.kscience.dataforge.names.plus
 /**
  * Append data to node
  */
-context(DataSetBuilder<T>) public suspend infix fun <T : Any> String.put(data: Data<T>): Unit =
+context(DataSetBuilder<T>) public infix fun <T : Any> String.put(data: Data<T>): Unit =
     data(Name.parse(this), data)
 
 /**
  * Append node
  */
-context(DataSetBuilder<T>) public suspend infix fun <T : Any> String.put(dataSet: DataSet<T>): Unit =
+context(DataSetBuilder<T>) public infix fun <T : Any> String.put(dataSet: DataSet<T>): Unit =
     node(Name.parse(this), dataSet)
 
 /**
  * Build and append node
  */
-context(DataSetBuilder<T>) public suspend infix fun <T : Any> String.put(
-    block: suspend DataSetBuilder<T>.() -> Unit,
+context(DataSetBuilder<T>) public infix fun <T : Any> String.put(
+    block: DataSetBuilder<T>.() -> Unit,
 ): Unit = node(Name.parse(this), block)
 
 /**
- * Copy given data set and mirror its changes to this [ActiveDataTree] in [this@setAndObserve]. Returns an update [Job]
+ * Copy given data set and mirror its changes to this [DataSourceBuilder] in [this@setAndObserve]. Returns an update [Job]
  */
 context(DataSetBuilder<T>) public fun <T : Any> CoroutineScope.setAndWatch(
     name: Name,
diff --git a/dataforge-data/src/jvmTest/kotlin/space/kscience/dataforge/data/ActionsTest.kt b/dataforge-data/src/jvmTest/kotlin/space/kscience/dataforge/data/ActionsTest.kt
index c80f00c4..9fd05357 100644
--- a/dataforge-data/src/jvmTest/kotlin/space/kscience/dataforge/data/ActionsTest.kt
+++ b/dataforge-data/src/jvmTest/kotlin/space/kscience/dataforge/data/ActionsTest.kt
@@ -1,44 +1,49 @@
 package space.kscience.dataforge.data
 
-import kotlinx.coroutines.runBlocking
+import kotlinx.coroutines.delay
+import kotlinx.coroutines.test.runTest
 import org.junit.jupiter.api.Test
 import space.kscience.dataforge.actions.Action
+import space.kscience.dataforge.actions.invoke
 import space.kscience.dataforge.actions.map
 import space.kscience.dataforge.misc.DFExperimental
 import kotlin.test.assertEquals
 
 @OptIn(DFExperimental::class)
 internal class ActionsTest {
-    private val data: DataTree<Int> = runBlocking {
-        DataTree {
+    @Test
+    fun testStaticMapAction() = runTest {
+        val data: DataTree<Int> = DataTree {
             repeat(10) {
                 static(it.toString(), it)
             }
         }
-    }
 
-    @Test
-    fun testStaticMapAction() {
         val plusOne = Action.map<Int, Int> {
             result { it + 1 }
         }
-        runBlocking {
-            val result = plusOne.execute(data)
-            assertEquals(2, result["1"]?.await())
-        }
+        val result = plusOne(data)
+        assertEquals(2, result["1"]?.await())
     }
 
     @Test
-    fun testDynamicMapAction() {
+    fun testDynamicMapAction() = runTest {
+        val data: DataSourceBuilder<Int> = ActiveDataTree()
+
         val plusOne = Action.map<Int, Int> {
             result { it + 1 }
         }
 
-        val datum = runBlocking {
-            val result = plusOne.execute(data, scope = this)
-            result["1"]?.await()
+        val result = plusOne(data)
+
+        repeat(10) {
+            data.static(it.toString(), it)
         }
-        assertEquals(2, datum)
+
+        delay(20)
+
+        assertEquals(2, result["1"]?.await())
+        data.close()
     }
 
 }
\ No newline at end of file
diff --git a/dataforge-data/src/jvmTest/kotlin/space/kscience/dataforge/data/DataTreeBuilderTest.kt b/dataforge-data/src/jvmTest/kotlin/space/kscience/dataforge/data/DataTreeBuilderTest.kt
index 65b43a6f..9509092e 100644
--- a/dataforge-data/src/jvmTest/kotlin/space/kscience/dataforge/data/DataTreeBuilderTest.kt
+++ b/dataforge-data/src/jvmTest/kotlin/space/kscience/dataforge/data/DataTreeBuilderTest.kt
@@ -46,8 +46,8 @@ internal class DataTreeBuilderTest {
         }
 
         runBlocking {
-            assertEquals("a", node.get("update.a")?.await())
-            assertEquals("a", node.get("primary.a")?.await())
+            assertEquals("a", node["update.a"]?.await())
+            assertEquals("a", node["primary.a"]?.await())
         }
     }
 
diff --git a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/WorkspaceBuilder.kt b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/WorkspaceBuilder.kt
index 6a3608bb..54d027ac 100644
--- a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/WorkspaceBuilder.kt
+++ b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/WorkspaceBuilder.kt
@@ -1,12 +1,10 @@
 package space.kscience.dataforge.workspace
 
+import kotlinx.coroutines.CoroutineScope
 import space.kscience.dataforge.context.Context
 import space.kscience.dataforge.context.ContextBuilder
 import space.kscience.dataforge.context.Global
-import space.kscience.dataforge.data.ActiveDataTree
-import space.kscience.dataforge.data.DataSet
-import space.kscience.dataforge.data.DataSetBuilder
-import space.kscience.dataforge.data.DataTree
+import space.kscience.dataforge.data.*
 import space.kscience.dataforge.meta.Meta
 import space.kscience.dataforge.meta.MetaRepr
 import space.kscience.dataforge.meta.MutableMeta
@@ -17,8 +15,11 @@ import space.kscience.dataforge.misc.DFBuilder
 import space.kscience.dataforge.misc.DFExperimental
 import space.kscience.dataforge.names.Name
 import space.kscience.dataforge.names.asName
+import kotlin.collections.HashMap
+import kotlin.collections.set
 import kotlin.properties.PropertyDelegateProvider
 import kotlin.properties.ReadOnlyProperty
+import kotlin.reflect.typeOf
 
 public data class TaskReference<T : Any>(public val taskName: Name, public val task: Task<T>) : DataSelector<T> {
 
@@ -100,13 +101,13 @@ public class WorkspaceBuilder(private val parentContext: Context = Global) : Tas
     /**
      * Define intrinsic data for the workspace
      */
-    public suspend fun buildData(builder: suspend DataSetBuilder<Any>.() -> Unit) {
+    public fun data(builder: DataSetBuilder<Any>.() -> Unit) {
         data = DataTree(builder)
     }
 
     @DFExperimental
-    public suspend fun buildActiveData(builder: suspend ActiveDataTree<Any>.() -> Unit) {
-        data = ActiveDataTree(builder)
+    public fun buildActiveData(scope: CoroutineScope, builder: DataSourceBuilder<Any>.() -> Unit) {
+        data = ActiveDataTree(typeOf<Any>(), scope, builder)
     }
 
     /**
diff --git a/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/workspaceJvm.kt b/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/workspaceJvm.kt
index ffa397d3..6634a2a0 100644
--- a/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/workspaceJvm.kt
+++ b/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/workspaceJvm.kt
@@ -1,21 +1,24 @@
 package space.kscience.dataforge.workspace
 
-import kotlinx.coroutines.runBlocking
 import space.kscience.dataforge.data.DataSet
-import space.kscience.dataforge.data.DataSetBuilder
-import space.kscience.dataforge.data.select
+import space.kscience.dataforge.data.filterIsInstance
 import space.kscience.dataforge.meta.Meta
 import space.kscience.dataforge.names.Name
+import space.kscience.dataforge.names.matches
 
-public fun WorkspaceBuilder.data(builder: suspend DataSetBuilder<Any>.() -> Unit): Unit = runBlocking {
-    buildData(builder)
-}
+//public fun WorkspaceBuilder.data(builder: DataSetBuilder<Any>.() -> Unit): Unit = runBlocking {
+//    data(builder)
+//}
 
-public inline fun <reified T: Any> TaskResultBuilder<*>.data(namePattern: Name? = null): DataSelector<T> = object : DataSelector<T> {
-    override suspend fun select(workspace: Workspace, meta: Meta): DataSet<T> = workspace.data.select(namePattern)
-}
+public inline fun <reified T : Any> TaskResultBuilder<*>.data(namePattern: Name? = null): DataSelector<T> =
+    object : DataSelector<T> {
+        override suspend fun select(workspace: Workspace, meta: Meta): DataSet<T> =
+            workspace.data.filterIsInstance { name, _ ->
+                namePattern == null || name.matches(namePattern)
+            }
+    }
 
 public suspend inline fun <reified T : Any> TaskResultBuilder<*>.fromTask(
     task: Name,
     taskMeta: Meta = Meta.EMPTY,
-): DataSet<T> = workspace.produce(task, taskMeta).select()
\ No newline at end of file
+): DataSet<T> = workspace.produce(task, taskMeta).filterIsInstance()
\ No newline at end of file
diff --git a/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/DataPropagationTest.kt b/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/DataPropagationTest.kt
index 1c1a849e..f4534bc7 100644
--- a/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/DataPropagationTest.kt
+++ b/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/DataPropagationTest.kt
@@ -14,7 +14,7 @@ class DataPropagationTestPlugin : WorkspacePlugin() {
     override val tag: PluginTag = Companion.tag
 
     val allData by task<Int> {
-        val selectedData = workspace.data.select<Int>()
+        val selectedData = workspace.data.filterIsInstance<Int>()
         val result: Data<Int> = selectedData.dataSequence().foldToData(0) { result, data ->
             result + data.await()
         }
@@ -23,7 +23,7 @@ class DataPropagationTestPlugin : WorkspacePlugin() {
 
 
     val singleData by task<Int> {
-        workspace.data.select<Int>()["myData[12]"]?.let {
+        workspace.data.filterIsInstance<Int>()["myData[12]"]?.let {
             data("result", it)
         }
     }
diff --git a/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/FileDataTest.kt b/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/FileDataTest.kt
index 612dad13..d1bbb606 100644
--- a/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/FileDataTest.kt
+++ b/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/FileDataTest.kt
@@ -20,20 +20,19 @@ import kotlin.test.assertEquals
 
 
 class FileDataTest {
-    val dataNode = runBlocking {
-        DataTree<String> {
-            node("dir") {
-                static("a", "Some string") {
-                    "content" put "Some string"
-                }
-            }
-            static("b", "root data")
-            meta {
-                "content" put "This is root meta node"
+    val dataNode = DataTree<String> {
+        node("dir") {
+            static("a", "Some string") {
+                "content" put "Some string"
             }
         }
+        static("b", "root data")
+        meta {
+            "content" put "This is root meta node"
+        }
     }
 
+
     object StringIOFormat : IOFormat<String> {
 
         override val type: KType = typeOf<String>()
diff --git a/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/SimpleWorkspaceTest.kt b/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/SimpleWorkspaceTest.kt
index 2187cbe2..367e8489 100644
--- a/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/SimpleWorkspaceTest.kt
+++ b/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/SimpleWorkspaceTest.kt
@@ -117,16 +117,16 @@ class SimpleWorkspaceTest {
         }
 
         val averageByGroup by task<Int> {
-            val evenSum = workspace.data.filter { name, _ ->
+            val evenSum = workspace.data.filterIsInstance<Int> { name, _ ->
                 name.toString().toInt() % 2 == 0
-            }.select<Int>().foldToData(0) { l, r ->
+            }.foldToData(0) { l, r ->
                 l + r.await()
             }
 
             data("even", evenSum)
-            val oddSum = workspace.data.filter { name, _ ->
+            val oddSum = workspace.data.filterIsInstance<Int> { name, _ ->
                 name.toString().toInt() % 2 == 1
-            }.select<Int>().foldToData(0) { l, r ->
+            }.foldToData(0) { l, r ->
                 l + r.await()
             }
             data("odd", oddSum)
@@ -143,7 +143,7 @@ class SimpleWorkspaceTest {
         }
 
         val customPipe by task<Int> {
-            workspace.data.select<Int>().forEach { data ->
+            workspace.data.filterIsInstance<Int>().forEach { data ->
                 val meta = data.meta.toMutableMeta().apply {
                     "newValue" put 22
                 }

From 7d9189e15c3484e56b53f25db1c9b4ec9e66a055 Mon Sep 17 00:00:00 2001
From: darksnake <altavir@gmail.com>
Date: Sat, 7 May 2022 18:06:55 +0300
Subject: [PATCH 12/19] Replace T by Pair<Meta, T> in data reducers

---
 .../space/kscience/dataforge/actions/ReduceAction.kt   | 10 +++++-----
 .../space/kscience/dataforge/data/dataTransform.kt     |  8 ++++----
 2 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/ReduceAction.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/ReduceAction.kt
index 3e1ec62f..99da6cd2 100644
--- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/ReduceAction.kt
+++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/ReduceAction.kt
@@ -19,14 +19,14 @@ public class JoinGroup<T : Any, R : Any>(
 
     public var meta: MutableMeta = MutableMeta()
 
-    public lateinit var result: suspend ActionEnv.(Map<Name, T>) -> R
+    public lateinit var result: suspend ActionEnv.(Map<Name, Pair<Meta, T>>) -> R
 
-    internal fun <R1 : R> result(outputType: KType, f: suspend ActionEnv.(Map<Name, T>) -> R1) {
+    internal fun <R1 : R> result(outputType: KType, f: suspend ActionEnv.(Map<Name,  Pair<Meta, T>>) -> R1) {
         this.outputType = outputType
         this.result = f;
     }
 
-    public inline fun <reified R1 : R> result(noinline f: suspend ActionEnv.(Map<Name, T>) -> R1) {
+    public inline fun <reified R1 : R> result(noinline f: suspend ActionEnv.(Map<Name,  Pair<Meta, T>>) -> R1) {
         outputType = typeOf<R1>()
         this.result = f;
     }
@@ -66,7 +66,7 @@ public class ReduceGroupBuilder<T : Any, R : Any>(
     /**
      * Apply transformation to the whole node
      */
-    public fun result(resultName: String, f: suspend ActionEnv.(Map<Name, T>) -> R) {
+    public fun result(resultName: String, f: suspend ActionEnv.(Map<Name,  Pair<Meta, T>>) -> R) {
         groupRules += { node ->
             listOf(JoinGroup<T, R>(resultName, node, outputType).apply { result(outputType, f) })
         }
@@ -82,7 +82,7 @@ internal class ReduceAction<T : Any, R : Any>(
     outputType: KType,
     private val action: ReduceGroupBuilder<T, R>.() -> Unit,
 ) : CachingAction<T, R>(outputType) {
-    //TODO optimize reduction. Currently the whole action recalculates on push
+    //TODO optimize reduction. Currently, the whole action recalculates on push
 
 
     override fun transform(set: DataSet<T>, meta: Meta, key: Name): Sequence<NamedData<R>> = sequence {
diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataTransform.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataTransform.kt
index b4e58c2d..aba33561 100644
--- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataTransform.kt
+++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataTransform.kt
@@ -49,13 +49,13 @@ public inline fun <T1 : Any, T2 : Any, reified R : Any> Data<T1>.combine(
 public inline fun <T : Any, reified R : Any> Collection<Data<T>>.reduceToData(
     coroutineContext: CoroutineContext = EmptyCoroutineContext,
     meta: Meta = Meta.EMPTY,
-    crossinline block: suspend (Collection<T>) -> R,
+    crossinline block: suspend (List<Pair<Meta, T>>) -> R,
 ): Data<R> = Data(
     meta,
     coroutineContext,
     this
 ) {
-    block(map { it.await() })
+    block(map { it.meta to it.await() })
 }
 
 @DFInternal
@@ -63,14 +63,14 @@ public fun <K, T : Any, R : Any> Map<K, Data<T>>.reduceToData(
     outputType: KType,
     coroutineContext: CoroutineContext = EmptyCoroutineContext,
     meta: Meta = Meta.EMPTY,
-    block: suspend (Map<K, T>) -> R,
+    block: suspend (Map<K, Pair<Meta, T>>) -> R,
 ): Data<R> = Data(
     outputType,
     meta,
     coroutineContext,
     this.values
 ) {
-    block(mapValues { it.value.await() })
+    block(mapValues { it.value.meta to it.value.await() })
 }
 
 

From fe92e8fccf796ce5c4d4fffe29987c048d403b16 Mon Sep 17 00:00:00 2001
From: Alexander Nozik <altavir@gmail.com>
Date: Sun, 8 May 2022 20:56:14 +0300
Subject: [PATCH 13/19] Data traversal refactoring

---
 CHANGELOG.md                                  |   1 +
 build.gradle.kts                              |   2 +-
 .../kscience/dataforge/actions/MapAction.kt   |   2 +-
 .../dataforge/actions/ReduceAction.kt         |  10 +-
 .../kscience/dataforge/actions/SplitAction.kt |   4 +-
 .../space/kscience/dataforge/data/DataSet.kt  |  29 ++---
 .../kscience/dataforge/data/DataSetBuilder.kt |   4 +-
 .../space/kscience/dataforge/data/DataTree.kt |  24 ++--
 .../kscience/dataforge/data/GroupRule.kt      |   4 +-
 .../kscience/dataforge/data/StaticDataTree.kt |   2 +-
 .../kscience/dataforge/data/dataFilter.kt     |  10 +-
 .../kscience/dataforge/data/dataTransform.kt  | 105 ++++++++++++------
 .../kscience/dataforge/data/dataFilterJvm.kt  |  26 ++---
 .../dataforge/workspace/TaskResult.kt         |   4 +-
 .../kscience/dataforge/workspace/Workspace.kt |   2 +-
 .../dataforge/workspace/workspaceJvm.kt       |   6 +-
 .../workspace/DataPropagationTest.kt          |  12 +-
 .../workspace/SimpleWorkspaceTest.kt          |  20 ++--
 18 files changed, 148 insertions(+), 119 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 9f73ebf1..58f82326 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -15,6 +15,7 @@
 - DataSet operates with sequences of data instead of flows
 - PartialEnvelope uses `Int` instead `UInt`.
 - `ActiveDataSet` renamed to `DataSource`
+- `selectOne`->`getByType`
 
 ### Deprecated
 
diff --git a/build.gradle.kts b/build.gradle.kts
index a7140851..f8072b6a 100644
--- a/build.gradle.kts
+++ b/build.gradle.kts
@@ -4,7 +4,7 @@ plugins {
 
 allprojects {
     group = "space.kscience"
-    version = "0.6.0-dev-5"
+    version = "0.6.0-dev-6"
 }
 
 subprojects {
diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/MapAction.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/MapAction.kt
index f2165e7d..91f58c95 100644
--- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/MapAction.kt
+++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/MapAction.kt
@@ -89,7 +89,7 @@ internal class MapAction<in T : Any, out R : Any>(
             return newData.named(newName)
         }
 
-        val sequence = dataSet.dataSequence().map(::mapOne)
+        val sequence = dataSet.traverse().map(::mapOne)
 
         return if (dataSet is DataSource ) {
             ActiveDataTree(outputType, dataSet) {
diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/ReduceAction.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/ReduceAction.kt
index 99da6cd2..d3be1ce1 100644
--- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/ReduceAction.kt
+++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/ReduceAction.kt
@@ -19,14 +19,14 @@ public class JoinGroup<T : Any, R : Any>(
 
     public var meta: MutableMeta = MutableMeta()
 
-    public lateinit var result: suspend ActionEnv.(Map<Name, Pair<Meta, T>>) -> R
+    public lateinit var result: suspend ActionEnv.(Map<Name, ValueWithMeta<T>>) -> R
 
-    internal fun <R1 : R> result(outputType: KType, f: suspend ActionEnv.(Map<Name,  Pair<Meta, T>>) -> R1) {
+    internal fun <R1 : R> result(outputType: KType, f: suspend ActionEnv.(Map<Name,  ValueWithMeta<T>>) -> R1) {
         this.outputType = outputType
         this.result = f;
     }
 
-    public inline fun <reified R1 : R> result(noinline f: suspend ActionEnv.(Map<Name,  Pair<Meta, T>>) -> R1) {
+    public inline fun <reified R1 : R> result(noinline f: suspend ActionEnv.(Map<Name,  ValueWithMeta<T>>) -> R1) {
         outputType = typeOf<R1>()
         this.result = f;
     }
@@ -66,7 +66,7 @@ public class ReduceGroupBuilder<T : Any, R : Any>(
     /**
      * Apply transformation to the whole node
      */
-    public fun result(resultName: String, f: suspend ActionEnv.(Map<Name,  Pair<Meta, T>>) -> R) {
+    public fun result(resultName: String, f: suspend ActionEnv.(Map<Name,  ValueWithMeta<T>>) -> R) {
         groupRules += { node ->
             listOf(JoinGroup<T, R>(resultName, node, outputType).apply { result(outputType, f) })
         }
@@ -87,7 +87,7 @@ internal class ReduceAction<T : Any, R : Any>(
 
     override fun transform(set: DataSet<T>, meta: Meta, key: Name): Sequence<NamedData<R>> = sequence {
         ReduceGroupBuilder<T, R>(meta, outputType).apply(action).buildGroups(set).forEach { group ->
-            val dataFlow: Map<Name, Data<T>> = group.set.dataSequence().fold(HashMap()) { acc, value ->
+            val dataFlow: Map<Name, Data<T>> = group.set.traverse().fold(HashMap()) { acc, value ->
                 acc.apply {
                     acc[value.name] = value.data
                 }
diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/SplitAction.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/SplitAction.kt
index 471a8057..ee778f6d 100644
--- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/SplitAction.kt
+++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/SplitAction.kt
@@ -77,7 +77,7 @@ internal class SplitAction<T : Any, R : Any>(
 
         return if (dataSet is DataSource) {
             ActiveDataTree<R>(outputType, dataSet) {
-                populateFrom(dataSet.dataSequence().flatMap(transform = ::splitOne))
+                populateFrom(dataSet.traverse().flatMap(transform = ::splitOne))
                 launch {
                     dataSet.updates.collect { name ->
                         //clear old nodes
@@ -89,7 +89,7 @@ internal class SplitAction<T : Any, R : Any>(
             }
         } else {
             DataTree<R>(outputType) {
-                populateFrom(dataSet.dataSequence().flatMap(transform = ::splitOne))
+                populateFrom(dataSet.traverse().flatMap(transform = ::splitOne))
             }
         }
     }
diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSet.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSet.kt
index f468632f..3a3497b3 100644
--- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSet.kt
+++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSet.kt
@@ -10,8 +10,7 @@ import space.kscience.dataforge.meta.set
 import space.kscience.dataforge.names.*
 import kotlin.reflect.KType
 
-public interface
-DataSet<out T : Any> {
+public interface DataSet<out T : Any> {
 
     /**
      * The minimal common ancestor to all data in the node
@@ -24,23 +23,15 @@ DataSet<out T : Any> {
     public val meta: Meta
 
     /**
-     * Traverse this provider or its child. The order is not guaranteed.
+     * Traverse this [DataSet] returning named data instances. The order is not guaranteed.
      */
-    public fun dataSequence(): Sequence<NamedData<T>>
+    public fun traverse(): Sequence<NamedData<T>>
 
     /**
      * Get data with given name.
      */
     public operator fun get(name: Name): Data<T>?
 
-
-    /**
-     * Get a snapshot of names of top level children of given node. Empty if node does not exist or is a leaf.
-     */
-    public fun listTop(prefix: Name = Name.EMPTY): List<Name> =
-        dataSequence().map { it.name }.filter { it.startsWith(prefix) && (it.length == prefix.length + 1) }.toList()
-    // By default, traverses the whole tree. Could be optimized in descendants
-
     public companion object {
         public val META_KEY: Name = "@meta".asName()
 
@@ -51,16 +42,14 @@ DataSet<out T : Any> {
             override val dataType: KType = TYPE_OF_NOTHING
             override val meta: Meta get() = Meta.EMPTY
 
-            //private val nothing: Nothing get() = error("this is nothing")
-
-            override fun dataSequence(): Sequence<NamedData<Nothing>> = emptySequence()
+            override fun traverse(): Sequence<NamedData<Nothing>> = emptySequence()
 
             override fun get(name: Name): Data<Nothing>? = null
         }
     }
 }
 
-public operator fun <T: Any> DataSet<T>.get(name:String): Data<T>? = get(name.parseAsName())
+public operator fun <T : Any> DataSet<T>.get(name: String): Data<T>? = get(name.parseAsName())
 
 /**
  * A [DataSet] with propagated updates.
@@ -78,7 +67,7 @@ public interface DataSource<T : Any> : DataSet<T>, CoroutineScope {
     /**
      * Stop generating updates from this [DataSource]
      */
-    public fun close(){
+    public fun close() {
         coroutineContext[Job]?.cancel()
     }
 }
@@ -89,7 +78,7 @@ public val <T : Any> DataSet<T>.updates: Flow<Name> get() = if (this is DataSour
  * Flow all data nodes with names starting with [branchName]
  */
 public fun <T : Any> DataSet<T>.children(branchName: Name): Sequence<NamedData<T>> =
-    this@children.dataSequence().filter {
+    this@children.traverse().filter {
         it.name.startsWith(branchName)
     }
 
@@ -97,7 +86,7 @@ public fun <T : Any> DataSet<T>.children(branchName: Name): Sequence<NamedData<T
  * Start computation for all goals in data node and return a job for the whole node
  */
 public fun <T : Any> DataSet<T>.startAll(coroutineScope: CoroutineScope): Job = coroutineScope.launch {
-    dataSequence().map {
+    traverse().map {
         it.launch(this@launch)
     }.toList().joinAll()
 }
@@ -105,7 +94,7 @@ public fun <T : Any> DataSet<T>.startAll(coroutineScope: CoroutineScope): Job =
 public suspend fun <T : Any> DataSet<T>.join(): Unit = coroutineScope { startAll(this).join() }
 
 public suspend fun DataSet<*>.toMeta(): Meta = Meta {
-    dataSequence().forEach {
+    traverse().forEach {
         if (it.name.endsWith(DataSet.META_KEY)) {
             set(it.name, it.meta)
         } else {
diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSetBuilder.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSetBuilder.kt
index cfcc1c97..ca50019a 100644
--- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSetBuilder.kt
+++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSetBuilder.kt
@@ -28,7 +28,7 @@ public interface DataSetBuilder<in T : Any> {
         }
 
         //Set new items
-        dataSet.dataSequence().forEach {
+        dataSet.traverse().forEach {
             data(name + it.name, it.data)
         }
     }
@@ -146,7 +146,7 @@ public inline fun <reified T : Any> DataSetBuilder<T>.static(
  */
 @DFExperimental
 public fun <T : Any> DataSetBuilder<T>.populateFrom(tree: DataSet<T>): Unit {
-    tree.dataSequence().forEach {
+    tree.traverse().forEach {
         //TODO check if the place is occupied
         data(it.name, it.data)
     }
diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataTree.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataTree.kt
index 3ef065e1..0540d6f6 100644
--- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataTree.kt
+++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataTree.kt
@@ -1,9 +1,5 @@
 package space.kscience.dataforge.data
 
-import kotlinx.coroutines.flow.Flow
-import kotlinx.coroutines.flow.emitAll
-import kotlinx.coroutines.flow.flow
-import kotlinx.coroutines.flow.map
 import space.kscience.dataforge.meta.Meta
 import space.kscience.dataforge.misc.Type
 import space.kscience.dataforge.names.*
@@ -43,20 +39,17 @@ public interface DataTree<out T : Any> : DataSet<T> {
 
     override val meta: Meta get() = items[META_ITEM_NAME_TOKEN]?.meta ?: Meta.EMPTY
 
-    override fun dataSequence(): Sequence<NamedData<T>> = sequence {
+    override fun traverse(): Sequence<NamedData<T>> = sequence {
         items.forEach { (token, childItem: DataTreeItem<T>) ->
             if (!token.body.startsWith("@")) {
                 when (childItem) {
                     is DataTreeItem.Leaf -> yield(childItem.data.named(token.asName()))
-                    is DataTreeItem.Node -> yieldAll(childItem.tree.dataSequence().map { it.named(token + it.name) })
+                    is DataTreeItem.Node -> yieldAll(childItem.tree.traverse().map { it.named(token + it.name) })
                 }
             }
         }
     }
 
-    override fun listTop(prefix: Name): List<Name> =
-        getItem(prefix).tree?.items?.keys?.map { prefix + it } ?: emptyList()
-
     override fun get(name: Name): Data<T>? = when (name.length) {
         0 -> null
         1 -> items[name.firstOrNull()!!].data
@@ -73,6 +66,9 @@ public interface DataTree<out T : Any> : DataSet<T> {
     }
 }
 
+public fun <T : Any> DataTree<T>.listChildren(prefix: Name): List<Name> =
+    getItem(prefix).tree?.items?.keys?.map { prefix + it } ?: emptyList()
+
 /**
  * Get a [DataTreeItem] with given [name] or null if the item does not exist
  */
@@ -86,15 +82,15 @@ public val <T : Any> DataTreeItem<T>?.tree: DataTree<T>? get() = (this as? DataT
 public val <T : Any> DataTreeItem<T>?.data: Data<T>? get() = (this as? DataTreeItem.Leaf<T>)?.data
 
 /**
- * Flow of all children including nodes
+ * A [Sequence] of all children including nodes
  */
-public fun <T : Any> DataTree<T>.itemFlow(): Flow<Pair<Name, DataTreeItem<T>>> = flow {
+public fun <T : Any> DataTree<T>.traverseItems(): Sequence<Pair<Name, DataTreeItem<T>>> = sequence {
     items.forEach { (head, item) ->
-        emit(head.asName() to item)
+        yield(head.asName() to item)
         if (item is DataTreeItem.Node) {
-            val subSequence = item.tree.itemFlow()
+            val subSequence = item.tree.traverseItems()
                 .map { (name, data) -> (head.asName() + name) to data }
-            emitAll(subSequence)
+            yieldAll(subSequence)
         }
     }
 }
diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/GroupRule.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/GroupRule.kt
index c00fac8d..1c4787f6 100644
--- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/GroupRule.kt
+++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/GroupRule.kt
@@ -42,7 +42,7 @@ public interface GroupRule {
                 val map = HashMap<String, DataSet<T>>()
 
                 if (set is DataSource) {
-                    set.dataSequence().forEach { data ->
+                    set.traverse().forEach { data ->
                         val tagValue: String = data.meta[key]?.string ?: defaultTagValue
                         (map.getOrPut(tagValue) { DataSourceBuilder(set.dataType, set.coroutineContext) } as DataSourceBuilder<T>)
                             .data(data.name, data.data)
@@ -61,7 +61,7 @@ public interface GroupRule {
                         }
                     }
                 } else {
-                    set.dataSequence().forEach { data ->
+                    set.traverse().forEach { data ->
                         val tagValue: String = data.meta[key]?.string ?: defaultTagValue
                         (map.getOrPut(tagValue) { StaticDataTree(set.dataType) } as StaticDataTree<T>)
                             .data(data.name, data.data)
diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/StaticDataTree.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/StaticDataTree.kt
index 2353f97d..04a1ebee 100644
--- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/StaticDataTree.kt
+++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/StaticDataTree.kt
@@ -52,7 +52,7 @@ internal class StaticDataTree<T : Any>(
         if (dataSet is StaticDataTree) {
             set(name, DataTreeItem.Node(dataSet))
         } else {
-            dataSet.dataSequence().forEach {
+            dataSet.traverse().forEach {
                 data(name + it.name, it.data)
             }
         }
diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataFilter.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataFilter.kt
index f5037918..d186c636 100644
--- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataFilter.kt
+++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataFilter.kt
@@ -30,8 +30,8 @@ public fun <T : Any> DataSet<T>.filter(
 
     override val meta: Meta get() = this@filter.meta
 
-    override fun dataSequence(): Sequence<NamedData<T>> =
-        this@filter.dataSequence().filter { predicate(it.name, it.meta) }
+    override fun traverse(): Sequence<NamedData<T>> =
+        this@filter.traverse().filter { predicate(it.name, it.meta) }
 
     override fun get(name: Name): Data<T>? = this@filter.get(name)?.takeIf {
         predicate(name, it.meta)
@@ -58,8 +58,8 @@ public fun <T : Any> DataSet<T>.withNamePrefix(prefix: Name): DataSet<T> = if (p
     override val meta: Meta get() = this@withNamePrefix.meta
 
 
-    override fun dataSequence(): Sequence<NamedData<T>> =
-        this@withNamePrefix.dataSequence().map { it.data.named(prefix + it.name) }
+    override fun traverse(): Sequence<NamedData<T>> =
+        this@withNamePrefix.traverse().map { it.data.named(prefix + it.name) }
 
     override fun get(name: Name): Data<T>? =
         name.removeHeadOrNull(name)?.let { this@withNamePrefix.get(it) }
@@ -80,7 +80,7 @@ public fun <T : Any> DataSet<T>.branch(branchName: Name): DataSet<T> = if (branc
 
     override val meta: Meta get() = this@branch.meta
 
-    override fun dataSequence(): Sequence<NamedData<T>> = this@branch.dataSequence().mapNotNull {
+    override fun traverse(): Sequence<NamedData<T>> = this@branch.traverse().mapNotNull {
         it.name.removeHeadOrNull(branchName)?.let { name ->
             it.data.named(name)
         }
diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataTransform.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataTransform.kt
index aba33561..287d6383 100644
--- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataTransform.kt
+++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataTransform.kt
@@ -1,12 +1,12 @@
 package space.kscience.dataforge.data
 
-import kotlinx.coroutines.flow.Flow
 import kotlinx.coroutines.flow.map
 import space.kscience.dataforge.meta.Meta
 import space.kscience.dataforge.meta.MutableMeta
 import space.kscience.dataforge.meta.seal
 import space.kscience.dataforge.meta.toMutableMeta
 import space.kscience.dataforge.misc.DFInternal
+import space.kscience.dataforge.names.Name
 import kotlin.contracts.InvocationKind
 import kotlin.contracts.contract
 import kotlin.coroutines.CoroutineContext
@@ -14,6 +14,15 @@ import kotlin.coroutines.EmptyCoroutineContext
 import kotlin.reflect.KType
 import kotlin.reflect.typeOf
 
+public data class ValueWithMeta<T>(val meta: Meta, val value: T)
+
+public suspend fun <T: Any> Data<T>.awaitWithMeta(): ValueWithMeta<T> = ValueWithMeta(meta, await())
+
+public data class NamedValueWithMeta<T>(val name: Name, val meta: Meta, val value: T)
+
+public suspend fun <T: Any> NamedData<T>.awaitWithMeta(): NamedValueWithMeta<T> = NamedValueWithMeta(name, meta, await())
+
+
 /**
  * Lazily transform this data to another data. By convention [block] should not use external data (be pure).
  * @param coroutineContext additional [CoroutineContext] elements used for data computation.
@@ -49,13 +58,13 @@ public inline fun <T1 : Any, T2 : Any, reified R : Any> Data<T1>.combine(
 public inline fun <T : Any, reified R : Any> Collection<Data<T>>.reduceToData(
     coroutineContext: CoroutineContext = EmptyCoroutineContext,
     meta: Meta = Meta.EMPTY,
-    crossinline block: suspend (List<Pair<Meta, T>>) -> R,
+    crossinline block: suspend (List<ValueWithMeta<T>>) -> R,
 ): Data<R> = Data(
     meta,
     coroutineContext,
     this
 ) {
-    block(map { it.meta to it.await() })
+    block(map { it.awaitWithMeta() })
 }
 
 @DFInternal
@@ -63,17 +72,16 @@ public fun <K, T : Any, R : Any> Map<K, Data<T>>.reduceToData(
     outputType: KType,
     coroutineContext: CoroutineContext = EmptyCoroutineContext,
     meta: Meta = Meta.EMPTY,
-    block: suspend (Map<K, Pair<Meta, T>>) -> R,
+    block: suspend (Map<K, ValueWithMeta<T>>) -> R,
 ): Data<R> = Data(
     outputType,
     meta,
     coroutineContext,
     this.values
 ) {
-    block(mapValues { it.value.meta to it.value.await() })
+    block(mapValues { it.value.awaitWithMeta() })
 }
 
-
 /**
  * Lazily reduce a [Map] of [Data] with any static key.
  * @param K type of the map key
@@ -83,58 +91,93 @@ public fun <K, T : Any, R : Any> Map<K, Data<T>>.reduceToData(
 public inline fun <K, T : Any, reified R : Any> Map<K, Data<T>>.reduceToData(
     coroutineContext: CoroutineContext = EmptyCoroutineContext,
     meta: Meta = Meta.EMPTY,
-    noinline block: suspend (Map<K, T>) -> R,
+    crossinline block: suspend (Map<K, ValueWithMeta<T>>) -> R,
 ): Data<R> = Data(
     meta,
     coroutineContext,
     this.values
 ) {
-    block(mapValues { it.value.await() })
+    block(mapValues { it.value.awaitWithMeta() })
 }
 
-//flow operations
+//Iterable operations
 
-/**
- * Transform a [Flow] of [NamedData] to a single [Data].
- */
 @DFInternal
-public inline fun <T : Any, R : Any> Sequence<NamedData<T>>.reduceToData(
+public inline fun <T : Any, R : Any> Iterable<Data<T>>.reduceToData(
     outputType: KType,
     coroutineContext: CoroutineContext = EmptyCoroutineContext,
     meta: Meta = Meta.EMPTY,
-    crossinline transformation: suspend (Sequence<NamedData<T>>) -> R,
+    crossinline transformation: suspend (Collection<ValueWithMeta<T>>) -> R,
 ): Data<R> = Data(
     outputType,
     meta,
     coroutineContext,
     toList()
 ) {
-    transformation(this)
+    transformation(map { it.awaitWithMeta() })
 }
 
 @OptIn(DFInternal::class)
-public inline fun <T : Any, reified R : Any> Sequence<NamedData<T>>.reduceToData(
+public inline fun <T : Any, reified R : Any> Iterable<Data<T>>.reduceToData(
     coroutineContext: CoroutineContext = EmptyCoroutineContext,
     meta: Meta = Meta.EMPTY,
-    crossinline transformation: suspend (Sequence<NamedData<T>>) -> R,
+    crossinline transformation: suspend (Collection<ValueWithMeta<T>>) -> R,
 ): Data<R> = reduceToData(typeOf<R>(), coroutineContext, meta) {
     transformation(it)
 }
 
-/**
- * Fold a flow of named data into a single [Data]
- */
-public inline fun <T : Any, reified R : Any> Sequence<NamedData<T>>.foldToData(
+public inline fun <T : Any, reified R : Any> Iterable<Data<T>>.foldToData(
     initial: R,
     coroutineContext: CoroutineContext = EmptyCoroutineContext,
     meta: Meta = Meta.EMPTY,
-    crossinline block: suspend (result: R, data: NamedData<T>) -> R,
+    crossinline block: suspend (result: R, data: ValueWithMeta<T>) -> R,
 ): Data<R> = reduceToData(
     coroutineContext, meta
 ) {
     it.fold(initial) { acc, t -> block(acc, t) }
 }
 
+/**
+ * Transform an [Iterable] of [NamedData] to a single [Data].
+ */
+@DFInternal
+public inline fun <T : Any, R : Any> Iterable<NamedData<T>>.reduceNamedToData(
+    outputType: KType,
+    coroutineContext: CoroutineContext = EmptyCoroutineContext,
+    meta: Meta = Meta.EMPTY,
+    crossinline transformation: suspend (Collection<NamedValueWithMeta<T>>) -> R,
+): Data<R> = Data(
+    outputType,
+    meta,
+    coroutineContext,
+    toList()
+) {
+    transformation(map { it.awaitWithMeta() })
+}
+
+@OptIn(DFInternal::class)
+public inline fun <T : Any, reified R : Any> Iterable<NamedData<T>>.reduceNamedToData(
+    coroutineContext: CoroutineContext = EmptyCoroutineContext,
+    meta: Meta = Meta.EMPTY,
+    crossinline transformation: suspend (Collection<NamedValueWithMeta<T>>) -> R,
+): Data<R> = reduceNamedToData(typeOf<R>(), coroutineContext, meta) {
+    transformation(it)
+}
+
+/**
+ * Fold a [Iterable] of named data into a single [Data]
+ */
+public inline fun <T : Any, reified R : Any> Iterable<NamedData<T>>.foldNamedToData(
+    initial: R,
+    coroutineContext: CoroutineContext = EmptyCoroutineContext,
+    meta: Meta = Meta.EMPTY,
+    crossinline block: suspend (result: R, data: NamedValueWithMeta<T>) -> R,
+): Data<R> = reduceNamedToData(
+    coroutineContext, meta
+) {
+    it.fold(initial) { acc, t -> block(acc, t) }
+}
+
 //DataSet operations
 
 @DFInternal
@@ -142,13 +185,13 @@ public suspend fun <T : Any, R : Any> DataSet<T>.map(
     outputType: KType,
     coroutineContext: CoroutineContext = EmptyCoroutineContext,
     metaTransform: MutableMeta.() -> Unit = {},
-    block: suspend (T) -> R,
+    block: suspend (NamedValueWithMeta<T>) -> R,
 ): DataTree<R> = DataTree<R>(outputType) {
     populateFrom(
-        dataSequence().map {
+        traverse().map {
             val newMeta = it.meta.toMutableMeta().apply(metaTransform).seal()
             Data(outputType, newMeta, coroutineContext, listOf(it)) {
-                block(it.await())
+                block(it.awaitWithMeta())
             }.named(it.name)
         }
     )
@@ -158,12 +201,12 @@ public suspend fun <T : Any, R : Any> DataSet<T>.map(
 public suspend inline fun <T : Any, reified R : Any> DataSet<T>.map(
     coroutineContext: CoroutineContext = EmptyCoroutineContext,
     noinline metaTransform: MutableMeta.() -> Unit = {},
-    noinline block: suspend (T) -> R,
+    noinline block: suspend (NamedValueWithMeta<T>) -> R,
 ): DataTree<R> = map(typeOf<R>(), coroutineContext, metaTransform, block)
 
 public suspend fun <T : Any> DataSet<T>.forEach(block: suspend (NamedData<T>) -> Unit) {
     contract { callsInPlace(block, InvocationKind.EXACTLY_ONCE) }
-    dataSequence().forEach {
+    traverse().forEach {
         block(it)
     }
 }
@@ -171,12 +214,12 @@ public suspend fun <T : Any> DataSet<T>.forEach(block: suspend (NamedData<T>) ->
 public inline fun <T : Any, reified R : Any> DataSet<T>.reduceToData(
     coroutineContext: CoroutineContext = EmptyCoroutineContext,
     meta: Meta = Meta.EMPTY,
-    crossinline transformation: suspend (Sequence<NamedData<T>>) -> R,
-): Data<R> = dataSequence().reduceToData(coroutineContext, meta, transformation)
+    crossinline transformation: suspend (Iterable<NamedValueWithMeta<T>>) -> R,
+): Data<R> = traverse().asIterable().reduceNamedToData(coroutineContext, meta, transformation)
 
 public inline fun <T : Any, reified R : Any> DataSet<T>.foldToData(
     initial: R,
     coroutineContext: CoroutineContext = EmptyCoroutineContext,
     meta: Meta = Meta.EMPTY,
-    crossinline block: suspend (result: R, data: NamedData<T>) -> R,
-): Data<R> = dataSequence().foldToData(initial, coroutineContext, meta, block)
\ No newline at end of file
+    crossinline block: suspend (result: R, data: NamedValueWithMeta<T>) -> R,
+): Data<R> = traverse().asIterable().foldNamedToData(initial, coroutineContext, meta, block)
\ No newline at end of file
diff --git a/dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/dataFilterJvm.kt b/dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/dataFilterJvm.kt
index 5efe6d2a..d5c5eb56 100644
--- a/dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/dataFilterJvm.kt
+++ b/dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/dataFilterJvm.kt
@@ -33,32 +33,32 @@ private fun <R : Any> Data<*>.castOrNull(type: KType): Data<R>? =
  * @param predicate addition filtering condition based on item name and meta. By default, accepts all
  */
 @OptIn(DFExperimental::class)
-public fun <R : Any> DataSet<*>.filterIsInstance(
+public fun <R : Any> DataSet<*>.filterByType(
     type: KType,
     predicate: (name: Name, meta: Meta) -> Boolean = { _, _ -> true },
 ): DataSource<R> = object : DataSource<R> {
     override val dataType = type
 
     override val coroutineContext: CoroutineContext
-        get() = (this@filterIsInstance as? DataSource)?.coroutineContext ?: EmptyCoroutineContext
+        get() = (this@filterByType as? DataSource)?.coroutineContext ?: EmptyCoroutineContext
 
-    override val meta: Meta get() = this@filterIsInstance.meta
+    override val meta: Meta get() = this@filterByType.meta
 
     private fun checkDatum(name: Name, datum: Data<*>): Boolean = datum.type.isSubtypeOf(type)
             && predicate(name, datum.meta)
 
-    override fun dataSequence(): Sequence<NamedData<R>> = this@filterIsInstance.dataSequence().filter {
+    override fun traverse(): Sequence<NamedData<R>> = this@filterByType.traverse().filter {
         checkDatum(it.name, it.data)
     }.map {
         @Suppress("UNCHECKED_CAST")
         it as NamedData<R>
     }
 
-    override fun get(name: Name): Data<R>? = this@filterIsInstance[name]?.let { datum ->
+    override fun get(name: Name): Data<R>? = this@filterByType[name]?.let { datum ->
         if (checkDatum(name, datum)) datum.castOrNull(type) else null
     }
 
-    override val updates: Flow<Name> = this@filterIsInstance.updates.filter { name ->
+    override val updates: Flow<Name> = this@filterByType.updates.filter { name ->
         get(name)?.let { datum ->
             checkDatum(name, datum)
         } ?: false
@@ -68,18 +68,18 @@ public fun <R : Any> DataSet<*>.filterIsInstance(
 /**
  * Select a single datum of the appropriate type
  */
-public inline fun <reified R : Any> DataSet<*>.filterIsInstance(
+public inline fun <reified R : Any> DataSet<*>.filterByType(
     noinline predicate: (name: Name, meta: Meta) -> Boolean = { _, _ -> true },
-): DataSet<R> = filterIsInstance(typeOf<R>(), predicate)
+): DataSet<R> = filterByType(typeOf<R>(), predicate)
 
 /**
  * Select a single datum if it is present and of given [type]
  */
-public fun <R : Any> DataSet<*>.selectOne(type: KType, name: Name): NamedData<R>? =
+public fun <R : Any> DataSet<*>.getByType(type: KType, name: Name): NamedData<R>? =
     get(name)?.castOrNull<R>(type)?.named(name)
 
-public inline fun <reified R : Any> DataSet<*>.selectOne(name: Name): NamedData<R>? =
-    selectOne(typeOf<R>(), name)
+public inline fun <reified R : Any> DataSet<*>.getByType(name: Name): NamedData<R>? =
+    this@getByType.getByType(typeOf<R>(), name)
 
-public inline fun <reified R : Any> DataSet<*>.selectOne(name: String): NamedData<R>? =
-    selectOne(typeOf<R>(), Name.parse(name))
\ No newline at end of file
+public inline fun <reified R : Any> DataSet<*>.getByType(name: String): NamedData<R>? =
+    this@getByType.getByType(typeOf<R>(), Name.parse(name))
\ No newline at end of file
diff --git a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/TaskResult.kt b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/TaskResult.kt
index a1fb4e84..1c9b59fd 100644
--- a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/TaskResult.kt
+++ b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/TaskResult.kt
@@ -23,7 +23,7 @@ public interface TaskResult<out T : Any> : DataSet<T> {
      */
     public val taskMeta: Meta
 
-    override fun dataSequence(): Sequence<TaskData<T>>
+    override fun traverse(): Sequence<TaskData<T>>
     override fun get(name: Name): TaskData<T>?
 }
 
@@ -34,7 +34,7 @@ private class TaskResultImpl<out T : Any>(
     override val taskMeta: Meta,
 ) : TaskResult<T>, DataSet<T> by dataSet {
 
-    override fun dataSequence(): Sequence<TaskData<T>> = dataSet.dataSequence().map {
+    override fun traverse(): Sequence<TaskData<T>> = dataSet.traverse().map {
         workspace.wrapData(it, it.name, taskName, taskMeta)
     }
 
diff --git a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/Workspace.kt b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/Workspace.kt
index 6b120c76..6fa04c94 100644
--- a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/Workspace.kt
+++ b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/Workspace.kt
@@ -35,7 +35,7 @@ public interface Workspace : ContextAware, Provider {
         return when (target) {
             "target", Meta.TYPE -> targets.mapKeys { Name.parse(it.key)}
             Task.TYPE -> tasks
-            Data.TYPE -> data.dataSequence().associateBy { it.name }
+            Data.TYPE -> data.traverse().associateBy { it.name }
             else -> emptyMap()
         }
     }
diff --git a/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/workspaceJvm.kt b/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/workspaceJvm.kt
index 6634a2a0..06a97869 100644
--- a/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/workspaceJvm.kt
+++ b/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/workspaceJvm.kt
@@ -1,7 +1,7 @@
 package space.kscience.dataforge.workspace
 
 import space.kscience.dataforge.data.DataSet
-import space.kscience.dataforge.data.filterIsInstance
+import space.kscience.dataforge.data.filterByType
 import space.kscience.dataforge.meta.Meta
 import space.kscience.dataforge.names.Name
 import space.kscience.dataforge.names.matches
@@ -13,7 +13,7 @@ import space.kscience.dataforge.names.matches
 public inline fun <reified T : Any> TaskResultBuilder<*>.data(namePattern: Name? = null): DataSelector<T> =
     object : DataSelector<T> {
         override suspend fun select(workspace: Workspace, meta: Meta): DataSet<T> =
-            workspace.data.filterIsInstance { name, _ ->
+            workspace.data.filterByType { name, _ ->
                 namePattern == null || name.matches(namePattern)
             }
     }
@@ -21,4 +21,4 @@ public inline fun <reified T : Any> TaskResultBuilder<*>.data(namePattern: Name?
 public suspend inline fun <reified T : Any> TaskResultBuilder<*>.fromTask(
     task: Name,
     taskMeta: Meta = Meta.EMPTY,
-): DataSet<T> = workspace.produce(task, taskMeta).filterIsInstance()
\ No newline at end of file
+): DataSet<T> = workspace.produce(task, taskMeta).filterByType()
\ No newline at end of file
diff --git a/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/DataPropagationTest.kt b/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/DataPropagationTest.kt
index f4534bc7..a7426d61 100644
--- a/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/DataPropagationTest.kt
+++ b/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/DataPropagationTest.kt
@@ -14,16 +14,16 @@ class DataPropagationTestPlugin : WorkspacePlugin() {
     override val tag: PluginTag = Companion.tag
 
     val allData by task<Int> {
-        val selectedData = workspace.data.filterIsInstance<Int>()
-        val result: Data<Int> = selectedData.dataSequence().foldToData(0) { result, data ->
-            result + data.await()
+        val selectedData = workspace.data.filterByType<Int>()
+        val result: Data<Int> = selectedData.traverse().asIterable().foldToData(0) { result, data ->
+            result + data.value
         }
         data("result", result)
     }
 
 
     val singleData by task<Int> {
-        workspace.data.filterIsInstance<Int>()["myData[12]"]?.let {
+        workspace.data.filterByType<Int>()["myData[12]"]?.let {
             data("result", it)
         }
     }
@@ -57,7 +57,7 @@ class DataPropagationTest {
     fun testAllData() {
         runBlocking {
             val node = testWorkspace.produce("Test.allData")
-            assertEquals(4950, node.dataSequence().single().await())
+            assertEquals(4950, node.traverse().single().await())
         }
     }
 
@@ -65,7 +65,7 @@ class DataPropagationTest {
     fun testSingleData() {
         runBlocking {
             val node = testWorkspace.produce("Test.singleData")
-            assertEquals(12, node.dataSequence().single().await())
+            assertEquals(12, node.traverse().single().await())
         }
     }
 }
\ No newline at end of file
diff --git a/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/SimpleWorkspaceTest.kt b/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/SimpleWorkspaceTest.kt
index 367e8489..64d13f30 100644
--- a/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/SimpleWorkspaceTest.kt
+++ b/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/SimpleWorkspaceTest.kt
@@ -63,7 +63,7 @@ class SimpleWorkspaceTest {
         }
 
         val filterOne by task<Int> {
-            workspace.data.selectOne<Int>("myData[12]")?.let { source ->
+            workspace.data.getByType<Int>("myData[12]")?.let { source ->
                 data(source.name, source.map { it })
             }
         }
@@ -111,23 +111,23 @@ class SimpleWorkspaceTest {
         val sum by task<Int> {
             workspace.logger.info { "Starting sum" }
             val res = from(square).foldToData(0) { l, r ->
-                l + r.await()
+                l + r.value
             }
             data("sum", res)
         }
 
         val averageByGroup by task<Int> {
-            val evenSum = workspace.data.filterIsInstance<Int> { name, _ ->
+            val evenSum = workspace.data.filterByType<Int> { name, _ ->
                 name.toString().toInt() % 2 == 0
             }.foldToData(0) { l, r ->
-                l + r.await()
+                l + r.value
             }
 
             data("even", evenSum)
-            val oddSum = workspace.data.filterIsInstance<Int> { name, _ ->
+            val oddSum = workspace.data.filterByType<Int> { name, _ ->
                 name.toString().toInt() % 2 == 1
             }.foldToData(0) { l, r ->
-                l + r.await()
+                l + r.value
             }
             data("odd", oddSum)
         }
@@ -143,7 +143,7 @@ class SimpleWorkspaceTest {
         }
 
         val customPipe by task<Int> {
-            workspace.data.filterIsInstance<Int>().forEach { data ->
+            workspace.data.filterByType<Int>().forEach { data ->
                 val meta = data.meta.toMutableMeta().apply {
                     "newValue" put 22
                 }
@@ -159,7 +159,7 @@ class SimpleWorkspaceTest {
     fun testWorkspace() {
         runBlocking {
             val node = workspace.runBlocking("sum")
-            val res = node.dataSequence().single()
+            val res = node.traverse().single()
             assertEquals(328350, res.await())
         }
     }
@@ -169,7 +169,7 @@ class SimpleWorkspaceTest {
     fun testMetaPropagation() {
         runBlocking {
             val node = workspace.produce("sum") { "testFlag" put true }
-            val res = node.dataSequence().single().await()
+            val res = node.traverse().single().await()
         }
     }
 
@@ -192,7 +192,7 @@ class SimpleWorkspaceTest {
     fun testFilter() {
         runBlocking {
             val node = workspace.produce("filterOne")
-            assertEquals(12, node.dataSequence().first().await())
+            assertEquals(12, node.traverse().first().await())
         }
     }
 }
\ No newline at end of file

From a546552540ccdad0607b14540dcb8682832e33c6 Mon Sep 17 00:00:00 2001
From: Alexander Nozik <altavir@gmail.com>
Date: Tue, 10 May 2022 14:45:58 +0300
Subject: [PATCH 14/19] Replace sequences by iterators in `DataSet`

---
 CHANGELOG.md                                  |  1 +
 build.gradle.kts                              |  2 +-
 .../{CachingAction.kt => AbstractAction.kt}   | 40 ++++++----
 .../kscience/dataforge/actions/MapAction.kt   | 76 +++++++------------
 .../dataforge/actions/ReduceAction.kt         | 11 ++-
 .../kscience/dataforge/actions/SplitAction.kt | 67 +++++++---------
 .../space/kscience/dataforge/data/DataSet.kt  | 43 +++++++----
 .../kscience/dataforge/data/DataSetBuilder.kt |  4 +-
 .../space/kscience/dataforge/data/DataTree.kt |  4 +-
 ...ataSourceBuilder.kt => DataTreeBuilder.kt} | 54 +++++++------
 .../kscience/dataforge/data/GroupRule.kt      | 10 ++-
 .../kscience/dataforge/data/StaticDataTree.kt |  2 +-
 .../kscience/dataforge/data/dataFilter.kt     | 24 ++++--
 .../kscience/dataforge/data/dataTransform.kt  | 31 ++++----
 .../kscience/dataforge/data/dataFilterJvm.kt  | 14 ++--
 .../dataforge/data/dataSetBuilderInContext.kt |  2 +-
 .../kscience/dataforge/data/ActionsTest.kt    |  5 +-
 .../dataforge/data/DataTreeBuilderTest.kt     |  4 +-
 .../space/kscience/dataforge/meta/Meta.kt     |  1 +
 .../dataforge/workspace/TaskResult.kt         |  9 ++-
 .../kscience/dataforge/workspace/Workspace.kt |  3 +-
 .../dataforge/workspace/WorkspaceBuilder.kt   |  6 +-
 .../dataforge/workspace/workspaceJvm.kt       |  2 +
 .../workspace/DataPropagationTest.kt          | 31 ++++----
 .../workspace/SimpleWorkspaceTest.kt          | 25 +++---
 25 files changed, 242 insertions(+), 229 deletions(-)
 rename dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/{CachingAction.kt => AbstractAction.kt} (52%)
 rename dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/{DataSourceBuilder.kt => DataTreeBuilder.kt} (66%)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 58f82326..17b972cc 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -16,6 +16,7 @@
 - PartialEnvelope uses `Int` instead `UInt`.
 - `ActiveDataSet` renamed to `DataSource`
 - `selectOne`->`getByType`
+- Data traversal in `DataSet` is done via iterator
 
 ### Deprecated
 
diff --git a/build.gradle.kts b/build.gradle.kts
index f8072b6a..a7560698 100644
--- a/build.gradle.kts
+++ b/build.gradle.kts
@@ -4,7 +4,7 @@ plugins {
 
 allprojects {
     group = "space.kscience"
-    version = "0.6.0-dev-6"
+    version = "0.6.0-dev-7"
 }
 
 subprojects {
diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/CachingAction.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/AbstractAction.kt
similarity index 52%
rename from dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/CachingAction.kt
rename to dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/AbstractAction.kt
index 469b6e60..e7bbe6f6 100644
--- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/CachingAction.kt
+++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/AbstractAction.kt
@@ -3,6 +3,7 @@ package space.kscience.dataforge.actions
 import kotlinx.coroutines.launch
 import space.kscience.dataforge.data.*
 import space.kscience.dataforge.meta.Meta
+import space.kscience.dataforge.misc.DFInternal
 import space.kscience.dataforge.names.Name
 import space.kscience.dataforge.names.startsWith
 import kotlin.reflect.KType
@@ -18,36 +19,47 @@ internal fun MutableMap<Name, *>.removeWhatStartsWith(name: Name) {
 /**
  * An action that caches results on-demand and recalculates them on source push
  */
-public abstract class CachingAction<in T : Any, out R : Any>(
+public abstract class AbstractAction<in T : Any, R : Any>(
     public val outputType: KType,
 ) : Action<T, R> {
 
-    protected abstract fun transform(
-        set: DataSet<T>,
+    /**
+     * Generate initial content of the output
+     */
+    protected abstract fun DataSetBuilder<R>.generate(
+        data: DataSet<T>,
         meta: Meta,
-        key: Name = Name.EMPTY,
-    ): Sequence<NamedData<R>>
+    )
 
+    /**
+     * Update part of the data set when given [updateKey] is triggered by the source
+     */
+    protected open fun DataSourceBuilder<R>.update(
+        dataSet: DataSet<T>,
+        meta: Meta,
+        updateKey: Name,
+    ) {
+        // By default, recalculate the whole dataset
+        generate(dataSet, meta)
+    }
+
+    @OptIn(DFInternal::class)
     override fun execute(
         dataSet: DataSet<T>,
         meta: Meta,
     ): DataSet<R> = if (dataSet is DataSource) {
-        DataSourceBuilder<R>(outputType, dataSet.coroutineContext).apply {
-            populateFrom(transform(dataSet, meta))
+        DataSource(outputType, dataSet){
+            generate(dataSet, meta)
 
             launch {
-                dataSet.updates.collect {
-                    //clear old nodes
-                    remove(it)
-                    //collect new items
-                    populateFrom(transform(dataSet, meta, it))
-                    //FIXME if the target is data, updates are fired twice
+                dataSet.updates.collect { name ->
+                    update(dataSet, meta, name)
                 }
             }
         }
     } else {
         DataTree<R>(outputType) {
-            populateFrom(transform(dataSet, meta))
+            generate(dataSet, meta)
         }
     }
 }
diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/MapAction.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/MapAction.kt
index 91f58c95..883b3928 100644
--- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/MapAction.kt
+++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/MapAction.kt
@@ -1,6 +1,5 @@
 package space.kscience.dataforge.actions
 
-import kotlinx.coroutines.launch
 import space.kscience.dataforge.data.*
 import space.kscience.dataforge.meta.Meta
 import space.kscience.dataforge.meta.MutableMeta
@@ -53,61 +52,44 @@ public class MapActionBuilder<T, R>(
 }
 
 @PublishedApi
-internal class MapAction<in T : Any, out R : Any>(
-    private val outputType: KType,
+internal class MapAction<in T : Any, R : Any>(
+    outputType: KType,
     private val block: MapActionBuilder<T, R>.() -> Unit,
-) : Action<T, R> {
+) : AbstractAction<T, R>(outputType) {
 
-    override fun execute(
-        dataSet: DataSet<T>,
-        meta: Meta,
-    ): DataSet<R> {
+    private fun DataSetBuilder<R>.mapOne(name: Name, data: Data<T>, meta: Meta) {
+        // Creating a new environment for action using **old** name, old meta and task meta
+        val env = ActionEnv(name, data.meta, meta)
 
-        fun mapOne(data: NamedData<T>): NamedData<R> {
-            // Creating a new environment for action using **old** name, old meta and task meta
-            val env = ActionEnv(data.name, data.meta, meta)
+        //applying transformation from builder
+        val builder = MapActionBuilder<T, R>(
+            name,
+            data.meta.toMutableMeta(), // using data meta
+            meta,
+            outputType
+        ).apply(block)
 
-            //applying transformation from builder
-            val builder = MapActionBuilder<T, R>(
-                data.name,
-                data.meta.toMutableMeta(), // using data meta
-                meta,
-                outputType
-            ).apply(block)
+        //getting new name
+        val newName = builder.name
 
-            //getting new name
-            val newName = builder.name
+        //getting new meta
+        val newMeta = builder.meta.seal()
 
-            //getting new meta
-            val newMeta = builder.meta.seal()
-
-            @OptIn(DFInternal::class)
-            val newData = Data(builder.outputType, newMeta, dependencies = listOf(data)) {
-                builder.result(env, data.await())
-            }
-            //setting the data node
-            return newData.named(newName)
+        @OptIn(DFInternal::class)
+        val newData = Data(builder.outputType, newMeta, dependencies = listOf(data)) {
+            builder.result(env, data.await())
         }
+        //setting the data node
+        data(newName, newData)
+    }
 
-        val sequence = dataSet.traverse().map(::mapOne)
+    override fun DataSetBuilder<R>.generate(data: DataSet<T>, meta: Meta) {
+        data.forEach { mapOne(it.name, it.data, meta) }
+    }
 
-        return if (dataSet is DataSource ) {
-            ActiveDataTree(outputType, dataSet) {
-                populateFrom(sequence)
-                launch {
-                    dataSet.updates.collect { name ->
-                        //clear old nodes
-                        remove(name)
-                        //collect new items
-                        populateFrom(dataSet.children(name).map(::mapOne))
-                    }
-                }
-            }
-        } else {
-            DataTree(outputType) {
-                populateFrom(sequence)
-            }
-        }
+    override fun DataSourceBuilder<R>.update(dataSet: DataSet<T>, meta: Meta, updateKey: Name) {
+        remove(updateKey)
+        dataSet[updateKey]?.let { mapOne(updateKey, it, meta) }
     }
 }
 
diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/ReduceAction.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/ReduceAction.kt
index d3be1ce1..1af3d5ec 100644
--- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/ReduceAction.kt
+++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/ReduceAction.kt
@@ -81,13 +81,12 @@ public class ReduceGroupBuilder<T : Any, R : Any>(
 internal class ReduceAction<T : Any, R : Any>(
     outputType: KType,
     private val action: ReduceGroupBuilder<T, R>.() -> Unit,
-) : CachingAction<T, R>(outputType) {
+) : AbstractAction<T, R>(outputType) {
     //TODO optimize reduction. Currently, the whole action recalculates on push
 
-
-    override fun transform(set: DataSet<T>, meta: Meta, key: Name): Sequence<NamedData<R>> = sequence {
-        ReduceGroupBuilder<T, R>(meta, outputType).apply(action).buildGroups(set).forEach { group ->
-            val dataFlow: Map<Name, Data<T>> = group.set.traverse().fold(HashMap()) { acc, value ->
+    override fun DataSetBuilder<R>.generate(data: DataSet<T>, meta: Meta) {
+        ReduceGroupBuilder<T, R>(meta, outputType).apply(action).buildGroups(data).forEach { group ->
+            val dataFlow: Map<Name, Data<T>> = group.set.asSequence().fold(HashMap()) { acc, value ->
                 acc.apply {
                     acc[value.name] = value.data
                 }
@@ -103,7 +102,7 @@ internal class ReduceAction<T : Any, R : Any>(
                 meta = groupMeta
             ) { group.result.invoke(env, it) }
 
-            yield(res.named(env.name))
+            data(env.name, res)
         }
     }
 }
diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/SplitAction.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/SplitAction.kt
index ee778f6d..b71c5fc5 100644
--- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/SplitAction.kt
+++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/SplitAction.kt
@@ -1,13 +1,11 @@
 package space.kscience.dataforge.actions
 
-import kotlinx.coroutines.launch
 import space.kscience.dataforge.data.*
 import space.kscience.dataforge.meta.Laminate
 import space.kscience.dataforge.meta.Meta
 import space.kscience.dataforge.meta.MutableMeta
 import space.kscience.dataforge.meta.toMutableMeta
 import space.kscience.dataforge.misc.DFExperimental
-import space.kscience.dataforge.misc.DFInternal
 import space.kscience.dataforge.names.Name
 import kotlin.collections.set
 import kotlin.reflect.KType
@@ -46,53 +44,42 @@ public class SplitBuilder<T : Any, R : Any>(public val name: Name, public val me
  */
 @PublishedApi
 internal class SplitAction<T : Any, R : Any>(
-    private val outputType: KType,
+    outputType: KType,
     private val action: SplitBuilder<T, R>.() -> Unit,
-) : Action<T, R> {
+) : AbstractAction<T, R>(outputType) {
 
-    override fun execute(
-        dataSet: DataSet<T>,
-        meta: Meta,
-    ): DataSet<R> {
+    private fun DataSetBuilder<R>.splitOne(name: Name, data: Data<T>, meta: Meta) {
+        val laminate = Laminate(data.meta, meta)
 
-        fun splitOne(data: NamedData<T>): Sequence<NamedData<R>> {
-            val laminate = Laminate(data.meta, meta)
-
-            val split = SplitBuilder<T, R>(data.name, data.meta).apply(action)
+        val split = SplitBuilder<T, R>(name, data.meta).apply(action)
 
 
-            // apply individual fragment rules to result
-            return split.fragments.entries.asSequence().map { (fragmentName, rule) ->
-                val env = SplitBuilder.FragmentRule<T, R>(
-                    fragmentName,
-                    laminate.toMutableMeta(),
-                    outputType
-                ).apply(rule)
-                //data.map<R>(outputType, meta = env.meta) { env.result(it) }.named(fragmentName)
-                @OptIn(DFInternal::class) Data(outputType, meta = env.meta, dependencies = listOf(data)) {
+        // apply individual fragment rules to result
+        split.fragments.forEach { (fragmentName, rule) ->
+            val env = SplitBuilder.FragmentRule<T, R>(
+                fragmentName,
+                laminate.toMutableMeta(),
+                outputType
+            ).apply(rule)
+            //data.map<R>(outputType, meta = env.meta) { env.result(it) }.named(fragmentName)
+
+            data(
+                fragmentName,
+                @Suppress("OPT_IN_USAGE") Data(outputType, meta = env.meta, dependencies = listOf(data)) {
                     env.result(data.await())
-                }.named(fragmentName)
-            }
-        }
-
-        return if (dataSet is DataSource) {
-            ActiveDataTree<R>(outputType, dataSet) {
-                populateFrom(dataSet.traverse().flatMap(transform = ::splitOne))
-                launch {
-                    dataSet.updates.collect { name ->
-                        //clear old nodes
-                        remove(name)
-                        //collect new items
-                        populateFrom(dataSet.children(name).flatMap(transform = ::splitOne))
-                    }
                 }
-            }
-        } else {
-            DataTree<R>(outputType) {
-                populateFrom(dataSet.traverse().flatMap(transform = ::splitOne))
-            }
+            )
         }
     }
+
+    override fun DataSetBuilder<R>.generate(data: DataSet<T>, meta: Meta) {
+        data.forEach { splitOne(it.name, it.data, meta) }
+    }
+
+    override fun DataSourceBuilder<R>.update(dataSet: DataSet<T>, meta: Meta, updateKey: Name) {
+        remove(updateKey)
+        dataSet[updateKey]?.let { splitOne(updateKey, it, meta) }
+    }
 }
 
 /**
diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSet.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSet.kt
index 3a3497b3..cf327c9b 100644
--- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSet.kt
+++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSet.kt
@@ -7,7 +7,10 @@ import kotlinx.coroutines.flow.mapNotNull
 import space.kscience.dataforge.data.Data.Companion.TYPE_OF_NOTHING
 import space.kscience.dataforge.meta.Meta
 import space.kscience.dataforge.meta.set
-import space.kscience.dataforge.names.*
+import space.kscience.dataforge.names.Name
+import space.kscience.dataforge.names.asName
+import space.kscience.dataforge.names.endsWith
+import space.kscience.dataforge.names.parseAsName
 import kotlin.reflect.KType
 
 public interface DataSet<out T : Any> {
@@ -25,7 +28,7 @@ public interface DataSet<out T : Any> {
     /**
      * Traverse this [DataSet] returning named data instances. The order is not guaranteed.
      */
-    public fun traverse(): Sequence<NamedData<T>>
+    public operator fun iterator(): Iterator<NamedData<T>>
 
     /**
      * Get data with given name.
@@ -42,19 +45,27 @@ public interface DataSet<out T : Any> {
             override val dataType: KType = TYPE_OF_NOTHING
             override val meta: Meta get() = Meta.EMPTY
 
-            override fun traverse(): Sequence<NamedData<Nothing>> = emptySequence()
+            override fun iterator(): Iterator<NamedData<Nothing>> = emptySequence<NamedData<Nothing>>().iterator()
 
             override fun get(name: Name): Data<Nothing>? = null
         }
     }
 }
 
+public fun <T : Any> DataSet<T>.asSequence(): Sequence<NamedData<T>> = object : Sequence<NamedData<T>> {
+    override fun iterator(): Iterator<NamedData<T>> = this@asSequence.iterator()
+}
+
+public fun <T : Any> DataSet<T>.asIterable(): Iterable<NamedData<T>> = object : Iterable<NamedData<T>> {
+    override fun iterator(): Iterator<NamedData<T>> = this@asIterable.iterator()
+}
+
 public operator fun <T : Any> DataSet<T>.get(name: String): Data<T>? = get(name.parseAsName())
 
 /**
  * A [DataSet] with propagated updates.
  */
-public interface DataSource<T : Any> : DataSet<T>, CoroutineScope {
+public interface DataSource<out T : Any> : DataSet<T>, CoroutineScope {
 
     /**
      * A flow of updated item names. Updates are propagated in a form of [Flow] of names of updated nodes.
@@ -73,28 +84,28 @@ public interface DataSource<T : Any> : DataSet<T>, CoroutineScope {
 }
 
 public val <T : Any> DataSet<T>.updates: Flow<Name> get() = if (this is DataSource) updates else emptyFlow()
-
-/**
- * Flow all data nodes with names starting with [branchName]
- */
-public fun <T : Any> DataSet<T>.children(branchName: Name): Sequence<NamedData<T>> =
-    this@children.traverse().filter {
-        it.name.startsWith(branchName)
-    }
+//
+///**
+// * Flow all data nodes with names starting with [branchName]
+// */
+//public fun <T : Any> DataSet<T>.children(branchName: Name): Sequence<NamedData<T>> =
+//    this@children.asSequence().filter {
+//        it.name.startsWith(branchName)
+//    }
 
 /**
  * Start computation for all goals in data node and return a job for the whole node
  */
 public fun <T : Any> DataSet<T>.startAll(coroutineScope: CoroutineScope): Job = coroutineScope.launch {
-    traverse().map {
+    asIterable().map {
         it.launch(this@launch)
-    }.toList().joinAll()
+    }.joinAll()
 }
 
 public suspend fun <T : Any> DataSet<T>.join(): Unit = coroutineScope { startAll(this).join() }
 
-public suspend fun DataSet<*>.toMeta(): Meta = Meta {
-    traverse().forEach {
+public fun DataSet<*>.toMeta(): Meta = Meta {
+    forEach {
         if (it.name.endsWith(DataSet.META_KEY)) {
             set(it.name, it.meta)
         } else {
diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSetBuilder.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSetBuilder.kt
index ca50019a..f9f14f37 100644
--- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSetBuilder.kt
+++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSetBuilder.kt
@@ -28,7 +28,7 @@ public interface DataSetBuilder<in T : Any> {
         }
 
         //Set new items
-        dataSet.traverse().forEach {
+        dataSet.forEach {
             data(name + it.name, it.data)
         }
     }
@@ -146,7 +146,7 @@ public inline fun <reified T : Any> DataSetBuilder<T>.static(
  */
 @DFExperimental
 public fun <T : Any> DataSetBuilder<T>.populateFrom(tree: DataSet<T>): Unit {
-    tree.traverse().forEach {
+    tree.forEach {
         //TODO check if the place is occupied
         data(it.name, it.data)
     }
diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataTree.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataTree.kt
index 0540d6f6..79a44b01 100644
--- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataTree.kt
+++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataTree.kt
@@ -39,12 +39,12 @@ public interface DataTree<out T : Any> : DataSet<T> {
 
     override val meta: Meta get() = items[META_ITEM_NAME_TOKEN]?.meta ?: Meta.EMPTY
 
-    override fun traverse(): Sequence<NamedData<T>> = sequence {
+    override fun iterator(): Iterator<NamedData<T>> = iterator {
         items.forEach { (token, childItem: DataTreeItem<T>) ->
             if (!token.body.startsWith("@")) {
                 when (childItem) {
                     is DataTreeItem.Leaf -> yield(childItem.data.named(token.asName()))
-                    is DataTreeItem.Node -> yieldAll(childItem.tree.traverse().map { it.named(token + it.name) })
+                    is DataTreeItem.Node -> yieldAll(childItem.tree.asSequence().map { it.named(token + it.name) })
                 }
             }
         }
diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSourceBuilder.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataTreeBuilder.kt
similarity index 66%
rename from dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSourceBuilder.kt
rename to dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataTreeBuilder.kt
index 23b44aa0..675d6566 100644
--- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSourceBuilder.kt
+++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataTreeBuilder.kt
@@ -3,9 +3,9 @@ package space.kscience.dataforge.data
 import kotlinx.coroutines.CoroutineScope
 import kotlinx.coroutines.Job
 import kotlinx.coroutines.flow.MutableSharedFlow
-import kotlinx.coroutines.flow.SharedFlow
 import kotlinx.coroutines.launch
 import space.kscience.dataforge.meta.Meta
+import space.kscience.dataforge.misc.DFInternal
 import space.kscience.dataforge.names.*
 import kotlin.collections.set
 import kotlin.coroutines.CoroutineContext
@@ -14,13 +14,19 @@ import kotlin.jvm.Synchronized
 import kotlin.reflect.KType
 import kotlin.reflect.typeOf
 
+public interface DataSourceBuilder<T : Any> : DataSetBuilder<T>, DataSource<T> {
+    override val updates: MutableSharedFlow<Name>
+}
+
 /**
  * A mutable [DataTree] that propagates updates
  */
-public class DataSourceBuilder<T : Any>(
+@PublishedApi
+internal class DataTreeBuilder<T : Any>(
     override val dataType: KType,
     coroutineContext: CoroutineContext,
-) : DataTree<T>, DataSetBuilder<T>, DataSource<T> {
+) : DataTree<T>, DataSourceBuilder<T> {
+
     override val coroutineContext: CoroutineContext =
         coroutineContext + Job(coroutineContext[Job]) + GoalExecutionRestriction()
 
@@ -29,23 +35,20 @@ public class DataSourceBuilder<T : Any>(
     override val items: Map<NameToken, DataTreeItem<T>>
         get() = treeItems.filter { !it.key.body.startsWith("@") }
 
-    private val _updates = MutableSharedFlow<Name>()
-
-    override val updates: SharedFlow<Name>
-        get() = _updates
+    override val updates = MutableSharedFlow<Name>()
 
     @Synchronized
     private fun remove(token: NameToken) {
         if (treeItems.remove(token) != null) {
             launch {
-                _updates.emit(token.asName())
+                updates.emit(token.asName())
             }
         }
     }
 
     override fun remove(name: Name) {
         if (name.isEmpty()) error("Can't remove the root node")
-        (getItem(name.cutLast()).tree as? DataSourceBuilder)?.remove(name.lastOrNull()!!)
+        (getItem(name.cutLast()).tree as? DataTreeBuilder)?.remove(name.lastOrNull()!!)
     }
 
     @Synchronized
@@ -58,11 +61,11 @@ public class DataSourceBuilder<T : Any>(
         treeItems[token] = DataTreeItem.Node(node)
     }
 
-    private fun getOrCreateNode(token: NameToken): DataSourceBuilder<T> =
-        (treeItems[token] as? DataTreeItem.Node<T>)?.tree as? DataSourceBuilder<T>
-            ?: DataSourceBuilder<T>(dataType, coroutineContext).also { set(token, it) }
+    private fun getOrCreateNode(token: NameToken): DataTreeBuilder<T> =
+        (treeItems[token] as? DataTreeItem.Node<T>)?.tree as? DataTreeBuilder<T>
+            ?: DataTreeBuilder<T>(dataType, coroutineContext).also { set(token, it) }
 
-    private fun getOrCreateNode(name: Name): DataSourceBuilder<T> = when (name.length) {
+    private fun getOrCreateNode(name: Name): DataTreeBuilder<T> = when (name.length) {
         0 -> this
         1 -> getOrCreateNode(name.firstOrNull()!!)
         else -> getOrCreateNode(name.firstOrNull()!!).getOrCreateNode(name.cutFirst())
@@ -79,7 +82,7 @@ public class DataSourceBuilder<T : Any>(
             }
         }
         launch {
-            _updates.emit(name)
+            updates.emit(name)
         }
     }
 
@@ -91,32 +94,39 @@ public class DataSourceBuilder<T : Any>(
 }
 
 /**
- * Create a dynamic tree. Initial data is placed synchronously.
+ * Create a dynamic [DataSource]. Initial data is placed synchronously.
  */
+@DFInternal
 @Suppress("FunctionName")
-public fun <T : Any> ActiveDataTree(
+public fun <T : Any> DataSource(
     type: KType,
     parent: CoroutineScope,
     block: DataSourceBuilder<T>.() -> Unit,
-): DataSourceBuilder<T> {
-    val tree = DataSourceBuilder<T>(type, parent.coroutineContext)
+): DataSource<T> {
+    val tree = DataTreeBuilder<T>(type, parent.coroutineContext)
     tree.block()
     return tree
 }
 
+@Suppress("OPT_IN_USAGE","FunctionName")
+public inline fun <reified T : Any> DataSource(
+    parent: CoroutineScope,
+    crossinline block: DataSourceBuilder<T>.() -> Unit,
+): DataSource<T> = DataSource(typeOf<T>(), parent) { block() }
+
 @Suppress("FunctionName")
-public suspend inline fun <reified T : Any> ActiveDataTree(
+public suspend inline fun <reified T : Any> DataSource(
     crossinline block: DataSourceBuilder<T>.() -> Unit = {},
-): DataSourceBuilder<T> = DataSourceBuilder<T>(typeOf<T>(), coroutineContext).apply { block() }
+): DataSourceBuilder<T> = DataTreeBuilder<T>(typeOf<T>(), coroutineContext).apply { block() }
 
 public inline fun <reified T : Any> DataSourceBuilder<T>.emit(
     name: Name,
     parent: CoroutineScope,
     noinline block: DataSourceBuilder<T>.() -> Unit,
-): Unit = node(name, ActiveDataTree(typeOf<T>(), parent, block))
+): Unit = node(name, DataSource(parent, block))
 
 public inline fun <reified T : Any> DataSourceBuilder<T>.emit(
     name: String,
     parent: CoroutineScope,
     noinline block: DataSourceBuilder<T>.() -> Unit,
-): Unit = node(Name.parse(name), ActiveDataTree(typeOf<T>(), parent, block))
+): Unit = node(Name.parse(name), DataSource(parent, block))
diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/GroupRule.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/GroupRule.kt
index 1c4787f6..189087a3 100644
--- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/GroupRule.kt
+++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/GroupRule.kt
@@ -18,6 +18,7 @@ package space.kscience.dataforge.data
 import kotlinx.coroutines.launch
 import space.kscience.dataforge.meta.get
 import space.kscience.dataforge.meta.string
+import space.kscience.dataforge.misc.DFInternal
 
 public interface GroupRule {
     public fun <T : Any> gather(set: DataSet<T>): Map<String, DataSet<T>>
@@ -31,6 +32,7 @@ public interface GroupRule {
          * @param defaultTagValue
          * @return
          */
+        @OptIn(DFInternal::class)
         public fun byMetaValue(
             key: String,
             defaultTagValue: String,
@@ -42,9 +44,9 @@ public interface GroupRule {
                 val map = HashMap<String, DataSet<T>>()
 
                 if (set is DataSource) {
-                    set.traverse().forEach { data ->
+                    set.forEach { data ->
                         val tagValue: String = data.meta[key]?.string ?: defaultTagValue
-                        (map.getOrPut(tagValue) { DataSourceBuilder(set.dataType, set.coroutineContext) } as DataSourceBuilder<T>)
+                        (map.getOrPut(tagValue) { DataTreeBuilder(set.dataType, set.coroutineContext) } as DataTreeBuilder<T>)
                             .data(data.name, data.data)
 
                         set.launch {
@@ -53,7 +55,7 @@ public interface GroupRule {
 
                                 val updateTagValue = dataUpdate?.meta?.get(key)?.string ?: defaultTagValue
                                 map.getOrPut(updateTagValue) {
-                                    ActiveDataTree(set.dataType, this) {
+                                    DataSource(set.dataType, this) {
                                         data(name, dataUpdate)
                                     }
                                 }
@@ -61,7 +63,7 @@ public interface GroupRule {
                         }
                     }
                 } else {
-                    set.traverse().forEach { data ->
+                    set.forEach { data ->
                         val tagValue: String = data.meta[key]?.string ?: defaultTagValue
                         (map.getOrPut(tagValue) { StaticDataTree(set.dataType) } as StaticDataTree<T>)
                             .data(data.name, data.data)
diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/StaticDataTree.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/StaticDataTree.kt
index 04a1ebee..4f0f455e 100644
--- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/StaticDataTree.kt
+++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/StaticDataTree.kt
@@ -52,7 +52,7 @@ internal class StaticDataTree<T : Any>(
         if (dataSet is StaticDataTree) {
             set(name, DataTreeItem.Node(dataSet))
         } else {
-            dataSet.traverse().forEach {
+            dataSet.forEach {
                 data(name + it.name, it.data)
             }
         }
diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataFilter.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataFilter.kt
index d186c636..e405f6c5 100644
--- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataFilter.kt
+++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataFilter.kt
@@ -30,8 +30,13 @@ public fun <T : Any> DataSet<T>.filter(
 
     override val meta: Meta get() = this@filter.meta
 
-    override fun traverse(): Sequence<NamedData<T>> =
-        this@filter.traverse().filter { predicate(it.name, it.meta) }
+    override fun iterator(): Iterator<NamedData<T>> = iterator {
+        for(d in this@filter){
+            if(predicate(d.name, d.meta)){
+                yield(d)
+            }
+        }
+    }
 
     override fun get(name: Name): Data<T>? = this@filter.get(name)?.takeIf {
         predicate(name, it.meta)
@@ -58,8 +63,11 @@ public fun <T : Any> DataSet<T>.withNamePrefix(prefix: Name): DataSet<T> = if (p
     override val meta: Meta get() = this@withNamePrefix.meta
 
 
-    override fun traverse(): Sequence<NamedData<T>> =
-        this@withNamePrefix.traverse().map { it.data.named(prefix + it.name) }
+    override fun iterator(): Iterator<NamedData<T>> = iterator {
+        for(d in this@withNamePrefix){
+            yield(d.data.named(prefix + d.name))
+        }
+    }
 
     override fun get(name: Name): Data<T>? =
         name.removeHeadOrNull(name)?.let { this@withNamePrefix.get(it) }
@@ -80,9 +88,11 @@ public fun <T : Any> DataSet<T>.branch(branchName: Name): DataSet<T> = if (branc
 
     override val meta: Meta get() = this@branch.meta
 
-    override fun traverse(): Sequence<NamedData<T>> = this@branch.traverse().mapNotNull {
-        it.name.removeHeadOrNull(branchName)?.let { name ->
-            it.data.named(name)
+    override fun iterator(): Iterator<NamedData<T>> = iterator {
+        for(d in this@branch){
+            d.name.removeHeadOrNull(branchName)?.let { name ->
+                yield(d.data.named(name))
+            }
         }
     }
 
diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataTransform.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataTransform.kt
index 287d6383..6d683f12 100644
--- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataTransform.kt
+++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataTransform.kt
@@ -7,8 +7,6 @@ import space.kscience.dataforge.meta.seal
 import space.kscience.dataforge.meta.toMutableMeta
 import space.kscience.dataforge.misc.DFInternal
 import space.kscience.dataforge.names.Name
-import kotlin.contracts.InvocationKind
-import kotlin.contracts.contract
 import kotlin.coroutines.CoroutineContext
 import kotlin.coroutines.EmptyCoroutineContext
 import kotlin.reflect.KType
@@ -16,11 +14,12 @@ import kotlin.reflect.typeOf
 
 public data class ValueWithMeta<T>(val meta: Meta, val value: T)
 
-public suspend fun <T: Any> Data<T>.awaitWithMeta(): ValueWithMeta<T> = ValueWithMeta(meta, await())
+public suspend fun <T : Any> Data<T>.awaitWithMeta(): ValueWithMeta<T> = ValueWithMeta(meta, await())
 
 public data class NamedValueWithMeta<T>(val name: Name, val meta: Meta, val value: T)
 
-public suspend fun <T: Any> NamedData<T>.awaitWithMeta(): NamedValueWithMeta<T> = NamedValueWithMeta(name, meta, await())
+public suspend fun <T : Any> NamedData<T>.awaitWithMeta(): NamedValueWithMeta<T> =
+    NamedValueWithMeta(name, meta, await())
 
 
 /**
@@ -187,14 +186,13 @@ public suspend fun <T : Any, R : Any> DataSet<T>.map(
     metaTransform: MutableMeta.() -> Unit = {},
     block: suspend (NamedValueWithMeta<T>) -> R,
 ): DataTree<R> = DataTree<R>(outputType) {
-    populateFrom(
-        traverse().map {
-            val newMeta = it.meta.toMutableMeta().apply(metaTransform).seal()
-            Data(outputType, newMeta, coroutineContext, listOf(it)) {
-                block(it.awaitWithMeta())
-            }.named(it.name)
+    forEach {
+        val newMeta = it.meta.toMutableMeta().apply(metaTransform).seal()
+        val d = Data(outputType, newMeta, coroutineContext, listOf(it)) {
+            block(it.awaitWithMeta())
         }
-    )
+        data(it.name, d)
+    }
 }
 
 @OptIn(DFInternal::class)
@@ -204,10 +202,9 @@ public suspend inline fun <T : Any, reified R : Any> DataSet<T>.map(
     noinline block: suspend (NamedValueWithMeta<T>) -> R,
 ): DataTree<R> = map(typeOf<R>(), coroutineContext, metaTransform, block)
 
-public suspend fun <T : Any> DataSet<T>.forEach(block: suspend (NamedData<T>) -> Unit) {
-    contract { callsInPlace(block, InvocationKind.EXACTLY_ONCE) }
-    traverse().forEach {
-        block(it)
+public inline fun <T : Any> DataSet<T>.forEach(block: (NamedData<T>) -> Unit) {
+    for (d in this) {
+        block(d)
     }
 }
 
@@ -215,11 +212,11 @@ public inline fun <T : Any, reified R : Any> DataSet<T>.reduceToData(
     coroutineContext: CoroutineContext = EmptyCoroutineContext,
     meta: Meta = Meta.EMPTY,
     crossinline transformation: suspend (Iterable<NamedValueWithMeta<T>>) -> R,
-): Data<R> = traverse().asIterable().reduceNamedToData(coroutineContext, meta, transformation)
+): Data<R> = asIterable().reduceNamedToData(coroutineContext, meta, transformation)
 
 public inline fun <T : Any, reified R : Any> DataSet<T>.foldToData(
     initial: R,
     coroutineContext: CoroutineContext = EmptyCoroutineContext,
     meta: Meta = Meta.EMPTY,
     crossinline block: suspend (result: R, data: NamedValueWithMeta<T>) -> R,
-): Data<R> = traverse().asIterable().foldNamedToData(initial, coroutineContext, meta, block)
\ No newline at end of file
+): Data<R> = asIterable().foldNamedToData(initial, coroutineContext, meta, block)
\ No newline at end of file
diff --git a/dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/dataFilterJvm.kt b/dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/dataFilterJvm.kt
index d5c5eb56..74d67d9d 100644
--- a/dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/dataFilterJvm.kt
+++ b/dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/dataFilterJvm.kt
@@ -5,7 +5,6 @@ import kotlinx.coroutines.flow.filter
 import space.kscience.dataforge.meta.Meta
 import space.kscience.dataforge.misc.DFExperimental
 import space.kscience.dataforge.names.Name
-import space.kscience.dataforge.names.matches
 import kotlin.coroutines.CoroutineContext
 import kotlin.coroutines.EmptyCoroutineContext
 import kotlin.reflect.KType
@@ -29,7 +28,6 @@ private fun <R : Any> Data<*>.castOrNull(type: KType): Data<R>? =
 /**
  * Select all data matching given type and filters. Does not modify paths
  *
- * @param namePattern a name match patter according to [Name.matches]
  * @param predicate addition filtering condition based on item name and meta. By default, accepts all
  */
 @OptIn(DFExperimental::class)
@@ -47,11 +45,13 @@ public fun <R : Any> DataSet<*>.filterByType(
     private fun checkDatum(name: Name, datum: Data<*>): Boolean = datum.type.isSubtypeOf(type)
             && predicate(name, datum.meta)
 
-    override fun traverse(): Sequence<NamedData<R>> = this@filterByType.traverse().filter {
-        checkDatum(it.name, it.data)
-    }.map {
-        @Suppress("UNCHECKED_CAST")
-        it as NamedData<R>
+    override fun iterator(): Iterator<NamedData<R>> = iterator {
+        for(d in this@filterByType){
+            if(checkDatum(d.name,d.data)){
+                @Suppress("UNCHECKED_CAST")
+                yield(d as NamedData<R>)
+            }
+        }
     }
 
     override fun get(name: Name): Data<R>? = this@filterByType[name]?.let { datum ->
diff --git a/dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/dataSetBuilderInContext.kt b/dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/dataSetBuilderInContext.kt
index 3fc68141..cb222ea0 100644
--- a/dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/dataSetBuilderInContext.kt
+++ b/dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/dataSetBuilderInContext.kt
@@ -27,7 +27,7 @@ context(DataSetBuilder<T>) public infix fun <T : Any> String.put(
 ): Unit = node(Name.parse(this), block)
 
 /**
- * Copy given data set and mirror its changes to this [DataSourceBuilder] in [this@setAndObserve]. Returns an update [Job]
+ * Copy given data set and mirror its changes to this [DataTreeBuilder] in [this@setAndObserve]. Returns an update [Job]
  */
 context(DataSetBuilder<T>) public fun <T : Any> CoroutineScope.setAndWatch(
     name: Name,
diff --git a/dataforge-data/src/jvmTest/kotlin/space/kscience/dataforge/data/ActionsTest.kt b/dataforge-data/src/jvmTest/kotlin/space/kscience/dataforge/data/ActionsTest.kt
index 9fd05357..3987cd19 100644
--- a/dataforge-data/src/jvmTest/kotlin/space/kscience/dataforge/data/ActionsTest.kt
+++ b/dataforge-data/src/jvmTest/kotlin/space/kscience/dataforge/data/ActionsTest.kt
@@ -1,5 +1,6 @@
 package space.kscience.dataforge.data
 
+import kotlinx.coroutines.ExperimentalCoroutinesApi
 import kotlinx.coroutines.delay
 import kotlinx.coroutines.test.runTest
 import org.junit.jupiter.api.Test
@@ -9,7 +10,7 @@ import space.kscience.dataforge.actions.map
 import space.kscience.dataforge.misc.DFExperimental
 import kotlin.test.assertEquals
 
-@OptIn(DFExperimental::class)
+@OptIn(DFExperimental::class, ExperimentalCoroutinesApi::class)
 internal class ActionsTest {
     @Test
     fun testStaticMapAction() = runTest {
@@ -28,7 +29,7 @@ internal class ActionsTest {
 
     @Test
     fun testDynamicMapAction() = runTest {
-        val data: DataSourceBuilder<Int> = ActiveDataTree()
+        val data: DataSourceBuilder<Int> = DataSource()
 
         val plusOne = Action.map<Int, Int> {
             result { it + 1 }
diff --git a/dataforge-data/src/jvmTest/kotlin/space/kscience/dataforge/data/DataTreeBuilderTest.kt b/dataforge-data/src/jvmTest/kotlin/space/kscience/dataforge/data/DataTreeBuilderTest.kt
index 9509092e..b77f7ea2 100644
--- a/dataforge-data/src/jvmTest/kotlin/space/kscience/dataforge/data/DataTreeBuilderTest.kt
+++ b/dataforge-data/src/jvmTest/kotlin/space/kscience/dataforge/data/DataTreeBuilderTest.kt
@@ -56,7 +56,7 @@ internal class DataTreeBuilderTest {
         try {
             lateinit var updateJob: Job
             supervisorScope {
-                val subNode = ActiveDataTree<Int> {
+                val subNode = DataSource<Int> {
                     updateJob = launch {
                         repeat(10) {
                             delay(10)
@@ -70,7 +70,7 @@ internal class DataTreeBuilderTest {
                         println(it)
                     }
                 }
-                val rootNode = ActiveDataTree<Int> {
+                val rootNode = DataSource<Int> {
                     setAndWatch("sub".asName(), subNode)
                 }
 
diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/Meta.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/Meta.kt
index 48a65c6d..e8f392ac 100644
--- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/Meta.kt
+++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/Meta.kt
@@ -133,6 +133,7 @@ public fun Meta.getIndexed(name: Name): Map<String?, Meta> {
     }
 }
 
+public fun Meta.getIndexed(name: String): Map<String?, Meta>  = getIndexed(name.parseAsName())
 
 /**
  * A meta node that ensures that all of its descendants has at least the same type.
diff --git a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/TaskResult.kt b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/TaskResult.kt
index 1c9b59fd..0c676fb7 100644
--- a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/TaskResult.kt
+++ b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/TaskResult.kt
@@ -1,6 +1,7 @@
 package space.kscience.dataforge.workspace
 
 import space.kscience.dataforge.data.DataSet
+import space.kscience.dataforge.data.forEach
 import space.kscience.dataforge.meta.Meta
 import space.kscience.dataforge.names.Name
 
@@ -23,7 +24,7 @@ public interface TaskResult<out T : Any> : DataSet<T> {
      */
     public val taskMeta: Meta
 
-    override fun traverse(): Sequence<TaskData<T>>
+    override fun iterator(): Iterator<TaskData<T>>
     override fun get(name: Name): TaskData<T>?
 }
 
@@ -34,8 +35,10 @@ private class TaskResultImpl<out T : Any>(
     override val taskMeta: Meta,
 ) : TaskResult<T>, DataSet<T> by dataSet {
 
-    override fun traverse(): Sequence<TaskData<T>> = dataSet.traverse().map {
-        workspace.wrapData(it, it.name, taskName, taskMeta)
+    override fun iterator(): Iterator<TaskData<T>> = iterator {
+        dataSet.forEach {
+            yield(workspace.wrapData(it, it.name, taskName, taskMeta))
+        }
     }
 
     override fun get(name: Name): TaskData<T>? = dataSet.get(name)?.let {
diff --git a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/Workspace.kt b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/Workspace.kt
index 6fa04c94..0053850d 100644
--- a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/Workspace.kt
+++ b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/Workspace.kt
@@ -3,6 +3,7 @@ package space.kscience.dataforge.workspace
 import space.kscience.dataforge.context.ContextAware
 import space.kscience.dataforge.data.Data
 import space.kscience.dataforge.data.DataSet
+import space.kscience.dataforge.data.asSequence
 import space.kscience.dataforge.meta.Meta
 import space.kscience.dataforge.meta.MutableMeta
 import space.kscience.dataforge.misc.Type
@@ -35,7 +36,7 @@ public interface Workspace : ContextAware, Provider {
         return when (target) {
             "target", Meta.TYPE -> targets.mapKeys { Name.parse(it.key)}
             Task.TYPE -> tasks
-            Data.TYPE -> data.traverse().associateBy { it.name }
+            Data.TYPE -> data.asSequence().associateBy { it.name }
             else -> emptyMap()
         }
     }
diff --git a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/WorkspaceBuilder.kt b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/WorkspaceBuilder.kt
index 54d027ac..4a847e64 100644
--- a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/WorkspaceBuilder.kt
+++ b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/WorkspaceBuilder.kt
@@ -15,11 +15,9 @@ import space.kscience.dataforge.misc.DFBuilder
 import space.kscience.dataforge.misc.DFExperimental
 import space.kscience.dataforge.names.Name
 import space.kscience.dataforge.names.asName
-import kotlin.collections.HashMap
 import kotlin.collections.set
 import kotlin.properties.PropertyDelegateProvider
 import kotlin.properties.ReadOnlyProperty
-import kotlin.reflect.typeOf
 
 public data class TaskReference<T : Any>(public val taskName: Name, public val task: Task<T>) : DataSelector<T> {
 
@@ -106,8 +104,8 @@ public class WorkspaceBuilder(private val parentContext: Context = Global) : Tas
     }
 
     @DFExperimental
-    public fun buildActiveData(scope: CoroutineScope, builder: DataSourceBuilder<Any>.() -> Unit) {
-        data = ActiveDataTree(typeOf<Any>(), scope, builder)
+    public fun data(scope: CoroutineScope, builder: DataSourceBuilder<Any>.() -> Unit) {
+        data = DataSource(scope, builder)
     }
 
     /**
diff --git a/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/workspaceJvm.kt b/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/workspaceJvm.kt
index 06a97869..02bf9001 100644
--- a/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/workspaceJvm.kt
+++ b/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/workspaceJvm.kt
@@ -3,6 +3,7 @@ package space.kscience.dataforge.workspace
 import space.kscience.dataforge.data.DataSet
 import space.kscience.dataforge.data.filterByType
 import space.kscience.dataforge.meta.Meta
+import space.kscience.dataforge.misc.DFExperimental
 import space.kscience.dataforge.names.Name
 import space.kscience.dataforge.names.matches
 
@@ -10,6 +11,7 @@ import space.kscience.dataforge.names.matches
 //    data(builder)
 //}
 
+@OptIn(DFExperimental::class)
 public inline fun <reified T : Any> TaskResultBuilder<*>.data(namePattern: Name? = null): DataSelector<T> =
     object : DataSelector<T> {
         override suspend fun select(workspace: Workspace, meta: Meta): DataSet<T> =
diff --git a/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/DataPropagationTest.kt b/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/DataPropagationTest.kt
index a7426d61..b0c2ebf4 100644
--- a/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/DataPropagationTest.kt
+++ b/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/DataPropagationTest.kt
@@ -1,6 +1,9 @@
+@file:OptIn(ExperimentalCoroutinesApi::class)
+
 package space.kscience.dataforge.workspace
 
-import kotlinx.coroutines.runBlocking
+import kotlinx.coroutines.ExperimentalCoroutinesApi
+import kotlinx.coroutines.test.runTest
 import space.kscience.dataforge.context.Context
 import space.kscience.dataforge.context.PluginFactory
 import space.kscience.dataforge.context.PluginTag
@@ -15,7 +18,7 @@ class DataPropagationTestPlugin : WorkspacePlugin() {
 
     val allData by task<Int> {
         val selectedData = workspace.data.filterByType<Int>()
-        val result: Data<Int> = selectedData.traverse().asIterable().foldToData(0) { result, data ->
+        val result: Data<Int> = selectedData.foldToData(0) { result, data ->
             result + data.value
         }
         data("result", result)
@@ -44,28 +47,22 @@ class DataPropagationTest {
         context {
             plugin(DataPropagationTestPlugin)
         }
-        runBlocking {
-            data {
-                repeat(100) {
-                    static("myData[$it]", it)
-                }
+        data {
+            repeat(100) {
+                static("myData[$it]", it)
             }
         }
     }
 
     @Test
-    fun testAllData() {
-        runBlocking {
-            val node = testWorkspace.produce("Test.allData")
-            assertEquals(4950, node.traverse().single().await())
-        }
+    fun testAllData() = runTest {
+        val node = testWorkspace.produce("Test.allData")
+        assertEquals(4950, node.asSequence().single().await())
     }
 
     @Test
-    fun testSingleData() {
-        runBlocking {
-            val node = testWorkspace.produce("Test.singleData")
-            assertEquals(12, node.traverse().single().await())
-        }
+    fun testSingleData() = runTest {
+        val node = testWorkspace.produce("Test.singleData")
+        assertEquals(12, node.asSequence().single().await())
     }
 }
\ No newline at end of file
diff --git a/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/SimpleWorkspaceTest.kt b/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/SimpleWorkspaceTest.kt
index 64d13f30..db231c80 100644
--- a/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/SimpleWorkspaceTest.kt
+++ b/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/SimpleWorkspaceTest.kt
@@ -1,8 +1,11 @@
 @file:Suppress("UNUSED_VARIABLE")
+@file:OptIn(ExperimentalCoroutinesApi::class)
 
 package space.kscience.dataforge.workspace
 
+import kotlinx.coroutines.ExperimentalCoroutinesApi
 import kotlinx.coroutines.runBlocking
+import kotlinx.coroutines.test.runTest
 import org.junit.jupiter.api.Timeout
 import space.kscience.dataforge.context.*
 import space.kscience.dataforge.data.*
@@ -26,7 +29,7 @@ public inline fun <reified P : Plugin> P.toFactory(): PluginFactory<P> = object
     override val type: KClass<out P> = P::class
 }
 
-public fun Workspace.runBlocking(task: String, block: MutableMeta.() -> Unit = {}): DataSet<Any> = runBlocking {
+public fun Workspace.produceBlocking(task: String, block: MutableMeta.() -> Unit = {}): DataSet<Any> = runBlocking {
     produce(task, block)
 }
 
@@ -156,21 +159,17 @@ class SimpleWorkspaceTest {
 
     @Test
     @Timeout(1)
-    fun testWorkspace() {
-        runBlocking {
-            val node = workspace.runBlocking("sum")
-            val res = node.traverse().single()
-            assertEquals(328350, res.await())
-        }
+    fun testWorkspace() = runTest {
+        val node = workspace.produce("sum")
+        val res = node.asSequence().single()
+        assertEquals(328350, res.await())
     }
 
     @Test
     @Timeout(1)
-    fun testMetaPropagation() {
-        runBlocking {
-            val node = workspace.produce("sum") { "testFlag" put true }
-            val res = node.traverse().single().await()
-        }
+    fun testMetaPropagation() = runTest {
+        val node = workspace.produce("sum") { "testFlag" put true }
+        val res = node.asSequence().single().await()
     }
 
     @Test
@@ -192,7 +191,7 @@ class SimpleWorkspaceTest {
     fun testFilter() {
         runBlocking {
             val node = workspace.produce("filterOne")
-            assertEquals(12, node.traverse().first().await())
+            assertEquals(12, node.asSequence().first().await())
         }
     }
 }
\ No newline at end of file

From 0fc219883249d689e149bcbc15a53efddaa977f3 Mon Sep 17 00:00:00 2001
From: Alexander Nozik <altavir@gmail.com>
Date: Mon, 16 May 2022 18:57:48 +0300
Subject: [PATCH 15/19] - Remove all unnecessary properties for `IOFormat` -
 Separate interfaces for `IOReader` and `IOWriter`

---
 CHANGELOG.md                                  |   2 +
 build.gradle.kts                              |   8 ++
 .../kscience/dataforge/provider/Provider.kt   |   6 +-
 .../io/yaml/FrontMatterEnvelopeFormat.kt      |   8 --
 .../dataforge/io/yaml/YamlMetaFormat.kt       |   8 --
 .../kscience/dataforge/io/EnvelopeFormat.kt   |   2 -
 .../kscience/dataforge/io/EnvelopeParts.kt    |  21 +++-
 .../space/kscience/dataforge/io/IOFormat.kt   | 107 +++++-------------
 .../space/kscience/dataforge/io/IOPlugin.kt   |  10 +-
 .../kscience/dataforge/io/JsonMetaFormat.kt   |   9 --
 .../space/kscience/dataforge/io/MetaFormat.kt |   1 -
 .../dataforge/io/TaggedEnvelopeFormat.kt      |  15 +--
 .../dataforge/io/TaglessEnvelopeFormat.kt     |  17 +--
 .../space/kscience/dataforge/io/ioMisc.kt     |   2 +-
 .../kscience/dataforge/io/MultipartTest.kt    |  23 ++--
 .../space/kscience/dataforge/io/fileIO.kt     |  45 +++++---
 .../dataforge/workspace/envelopeData.kt       |  11 +-
 .../kscience/dataforge/workspace/fileData.kt  |  85 ++++----------
 .../dataforge/workspace/FileDataTest.kt       |  21 +---
 gradle.properties                             |   2 +-
 20 files changed, 142 insertions(+), 261 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 17b972cc..95d5c0d5 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -17,6 +17,8 @@
 - `ActiveDataSet` renamed to `DataSource`
 - `selectOne`->`getByType`
 - Data traversal in `DataSet` is done via iterator
+- Remove all unnecessary properties for `IOFormat`
+- Separate interfaces for `IOReader` and `IOWriter`
 
 ### Deprecated
 
diff --git a/build.gradle.kts b/build.gradle.kts
index a7560698..765e242f 100644
--- a/build.gradle.kts
+++ b/build.gradle.kts
@@ -1,3 +1,5 @@
+import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
+
 plugins {
     id("ru.mipt.npm.gradle.project")
 }
@@ -5,6 +7,12 @@ plugins {
 allprojects {
     group = "space.kscience"
     version = "0.6.0-dev-7"
+
+    tasks.withType<KotlinCompile>{
+        kotlinOptions{
+            freeCompilerArgs = freeCompilerArgs + "-Xcontext-receivers"
+        }
+    }
 }
 
 subprojects {
diff --git a/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/provider/Provider.kt b/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/provider/Provider.kt
index e1940958..c91c2e4f 100644
--- a/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/provider/Provider.kt
+++ b/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/provider/Provider.kt
@@ -75,10 +75,8 @@ public inline fun <reified T : Any> Provider.provide(path: String, targetOverrid
 /**
  *  Typed top level content
  */
-public fun <T : Any> Provider.top(target: String, type: KClass<out T>): Map<Name, T> {
-    return content(target).mapValues {
-        type.safeCast(it.value) ?: error("The type of element $it is ${it::class} but $type is expected")
-    }
+public fun <T : Any> Provider.top(target: String, type: KClass<out T>): Map<Name, T> = content(target).mapValues {
+    type.safeCast(it.value) ?: error("The type of element ${it.value} is ${it.value::class} but $type is expected")
 }
 
 /**
diff --git a/dataforge-io/dataforge-io-yaml/src/commonMain/kotlin/space/kscience/dataforge/io/yaml/FrontMatterEnvelopeFormat.kt b/dataforge-io/dataforge-io-yaml/src/commonMain/kotlin/space/kscience/dataforge/io/yaml/FrontMatterEnvelopeFormat.kt
index 7f705a71..244284c8 100644
--- a/dataforge-io/dataforge-io-yaml/src/commonMain/kotlin/space/kscience/dataforge/io/yaml/FrontMatterEnvelopeFormat.kt
+++ b/dataforge-io/dataforge-io-yaml/src/commonMain/kotlin/space/kscience/dataforge/io/yaml/FrontMatterEnvelopeFormat.kt
@@ -7,10 +7,7 @@ import io.ktor.utils.io.core.readBytes
 import space.kscience.dataforge.context.Context
 import space.kscience.dataforge.context.Global
 import space.kscience.dataforge.io.*
-import space.kscience.dataforge.io.IOFormat.Companion.META_KEY
-import space.kscience.dataforge.io.IOFormat.Companion.NAME_KEY
 import space.kscience.dataforge.meta.Meta
-import space.kscience.dataforge.misc.DFExperimental
 import space.kscience.dataforge.names.Name
 import space.kscience.dataforge.names.plus
 
@@ -65,11 +62,6 @@ public class FrontMatterEnvelopeFormat(
         }
     }
 
-    override fun toMeta(): Meta = Meta {
-        NAME_KEY put name.toString()
-        META_KEY put meta
-    }
-
     public companion object : EnvelopeFormatFactory {
         public const val SEPARATOR: String = "---"
 
diff --git a/dataforge-io/dataforge-io-yaml/src/commonMain/kotlin/space/kscience/dataforge/io/yaml/YamlMetaFormat.kt b/dataforge-io/dataforge-io-yaml/src/commonMain/kotlin/space/kscience/dataforge/io/yaml/YamlMetaFormat.kt
index b7c7c3e4..fffaa45b 100644
--- a/dataforge-io/dataforge-io-yaml/src/commonMain/kotlin/space/kscience/dataforge/io/yaml/YamlMetaFormat.kt
+++ b/dataforge-io/dataforge-io-yaml/src/commonMain/kotlin/space/kscience/dataforge/io/yaml/YamlMetaFormat.kt
@@ -4,8 +4,6 @@ import io.ktor.utils.io.core.Input
 import io.ktor.utils.io.core.Output
 import net.mamoe.yamlkt.*
 import space.kscience.dataforge.context.Context
-import space.kscience.dataforge.io.IOFormat.Companion.META_KEY
-import space.kscience.dataforge.io.IOFormat.Companion.NAME_KEY
 import space.kscience.dataforge.io.MetaFormat
 import space.kscience.dataforge.io.MetaFormatFactory
 import space.kscience.dataforge.io.readUtf8String
@@ -14,7 +12,6 @@ import space.kscience.dataforge.meta.Meta
 import space.kscience.dataforge.meta.descriptors.MetaDescriptor
 import space.kscience.dataforge.meta.descriptors.get
 import space.kscience.dataforge.meta.isLeaf
-import space.kscience.dataforge.misc.DFExperimental
 import space.kscience.dataforge.names.NameToken
 import space.kscience.dataforge.names.withIndex
 import space.kscience.dataforge.values.ListValue
@@ -108,11 +105,6 @@ public class YamlMetaFormat(private val meta: Meta) : MetaFormat {
         return yaml.toMeta()
     }
 
-    override fun toMeta(): Meta = Meta {
-        NAME_KEY put FrontMatterEnvelopeFormat.name.toString()
-        META_KEY put meta
-    }
-
     public companion object : MetaFormatFactory {
         override fun build(context: Context, meta: Meta): MetaFormat = YamlMetaFormat(meta)
 
diff --git a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/EnvelopeFormat.kt b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/EnvelopeFormat.kt
index 4dea572b..b705a3c0 100644
--- a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/EnvelopeFormat.kt
+++ b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/EnvelopeFormat.kt
@@ -17,8 +17,6 @@ import kotlin.reflect.typeOf
 public data class PartialEnvelope(val meta: Meta, val dataOffset: Int, val dataSize: ULong?)
 
 public interface EnvelopeFormat : IOFormat<Envelope> {
-    override val type: KType get() = typeOf<Envelope>()
-
     public val defaultMetaFormat: MetaFormatFactory get() = JsonMetaFormat
 
     public fun readPartial(input: Input): PartialEnvelope
diff --git a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/EnvelopeParts.kt b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/EnvelopeParts.kt
index 641247c3..72e9dfd4 100644
--- a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/EnvelopeParts.kt
+++ b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/EnvelopeParts.kt
@@ -1,5 +1,6 @@
 package space.kscience.dataforge.io
 
+import space.kscience.dataforge.context.invoke
 import space.kscience.dataforge.io.Envelope.Companion.ENVELOPE_NODE_KEY
 import space.kscience.dataforge.io.PartDescriptor.Companion.DEFAULT_MULTIPART_DATA_SEPARATOR
 import space.kscience.dataforge.io.PartDescriptor.Companion.MULTIPART_DATA_TYPE
@@ -35,7 +36,7 @@ public typealias EnvelopeParts = List<EnvelopePart>
 
 public fun EnvelopeBuilder.multipart(
     parts: EnvelopeParts,
-    separator: String = DEFAULT_MULTIPART_DATA_SEPARATOR
+    separator: String = DEFAULT_MULTIPART_DATA_SEPARATOR,
 ) {
     dataType = MULTIPART_DATA_TYPE
 
@@ -67,17 +68,25 @@ public fun EnvelopeBuilder.multipart(
     }
 }
 
+/**
+ * Put a list of envelopes as parts of given envelope
+ */
 public fun EnvelopeBuilder.envelopes(
     envelopes: List<Envelope>,
-    format: EnvelopeFormat = TaggedEnvelopeFormat,
-    separator: String = DEFAULT_MULTIPART_DATA_SEPARATOR
+    formatFactory: EnvelopeFormatFactory = TaggedEnvelopeFormat,
+    formatMeta: Meta? = null,
+    separator: String = DEFAULT_MULTIPART_DATA_SEPARATOR,
 ) {
     val parts = envelopes.map {
-        val binary = format.toBinary(it)
+        val format = formatMeta?.let { formatFactory(formatMeta) } ?: formatFactory
+        val binary = Binary(it, format)
         EnvelopePart(binary, null)
     }
-    meta{
-        set(MULTIPART_KEY + PART_FORMAT_KEY, format.toMeta())
+    meta {
+        (MULTIPART_KEY + PART_FORMAT_KEY) put {
+            IOFormatFactory.NAME_KEY put formatFactory.name.toString()
+            formatMeta?.let { IOFormatFactory.META_KEY put formatMeta }
+        }
     }
     multipart(parts, separator)
 }
diff --git a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/IOFormat.kt b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/IOFormat.kt
index 7c3f8988..985a1e69 100644
--- a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/IOFormat.kt
+++ b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/IOFormat.kt
@@ -1,12 +1,13 @@
 package space.kscience.dataforge.io
 
-import io.ktor.utils.io.core.*
+import io.ktor.utils.io.core.Input
+import io.ktor.utils.io.core.Output
+import io.ktor.utils.io.core.readDouble
+import io.ktor.utils.io.core.writeDouble
 import space.kscience.dataforge.context.Context
 import space.kscience.dataforge.context.Factory
-import space.kscience.dataforge.io.IOFormat.Companion.NAME_KEY
 import space.kscience.dataforge.io.IOFormatFactory.Companion.IO_FORMAT_TYPE
 import space.kscience.dataforge.meta.Meta
-import space.kscience.dataforge.meta.MetaRepr
 import space.kscience.dataforge.misc.Named
 import space.kscience.dataforge.misc.Type
 import space.kscience.dataforge.names.Name
@@ -14,92 +15,53 @@ import space.kscience.dataforge.names.asName
 import kotlin.reflect.KType
 import kotlin.reflect.typeOf
 
+public fun interface IOReader<out T : Any> {
+
+    public fun readObject(input: Input): T
+}
+
+public fun interface IOWriter<in T : Any> {
+    public fun writeObject(output: Output, obj: T)
+}
+
+
 /**
  * And interface for reading and writing objects into with IO streams
  */
-public interface IOFormat<T : Any> : MetaRepr {
-    public val type: KType
+public interface IOFormat<T : Any> : IOReader<T>, IOWriter<T>
 
-    public fun writeObject(output: Output, obj: T)
-    public fun readObject(input: Input): T
+public fun <T : Any> Input.readObject(format: IOReader<T>): T = format.readObject(this@readObject)
 
-    public companion object {
-        public val NAME_KEY: Name = "name".asName()
-        public val META_KEY: Name = "meta".asName()
-    }
-}
-
-public fun <T : Any> Input.readWith(format: IOFormat<T>): T = format.readObject(this@readWith)
-
-public fun <T: Any> IOFormat<T>.readObject(binary: Binary): T = binary.read {
+public fun <T : Any> IOFormat<T>.readObjectFrom(binary: Binary): T = binary.read {
     readObject(this)
 }
 
 /**
  * Read given binary as object using given format
  */
-public fun <T : Any> Binary.readWith(format: IOFormat<T>): T = read {
-    readWith(format)
+public fun <T : Any> Binary.readWith(format: IOReader<T>): T = read {
+    readObject(format)
 }
 
-public fun <T : Any> Output.writeWith(format: IOFormat<T>, obj: T): Unit =
-    format.run { writeObject(this@writeWith, obj) }
+public fun <T : Any> Output.writeObject(format: IOWriter<T>, obj: T): Unit =
+    format.run { writeObject(this@writeObject, obj) }
 
-public inline fun <reified T : Any> IOFormat.Companion.listOf(
-    format: IOFormat<T>,
-): IOFormat<List<T>> = object : IOFormat<List<T>> {
-    override val type: KType = typeOf<List<T>>()
-
-    override fun writeObject(output: Output, obj: List<T>) {
-        output.writeInt(obj.size)
-        format.run {
-            obj.forEach {
-                writeObject(output, it)
-            }
-        }
-    }
-
-    override fun readObject(input: Input): List<T> {
-        val size = input.readInt()
-        return format.run {
-            List(size) { readObject(input) }
-        }
-    }
-
-    override fun toMeta(): Meta = Meta {
-        NAME_KEY put "list"
-        "contentFormat" put format.toMeta()
-    }
-
-}
-
-//public fun ObjectPool<Buffer>.fill(block: Buffer.() -> Unit): Buffer {
-//    val buffer = borrow()
-//    return try {
-//        buffer.apply(block)
-//    } catch (ex: Exception) {
-//        //recycle(buffer)
-//        throw ex
-//    }
-//}
 
 @Type(IO_FORMAT_TYPE)
-public interface IOFormatFactory<T : Any> : Factory<IOFormat<T>>, Named, MetaRepr {
+public interface IOFormatFactory<T : Any> : Factory<IOFormat<T>>, Named {
     /**
      * Explicit type for dynamic type checks
      */
     public val type: KType
 
-    override fun toMeta(): Meta = Meta {
-        NAME_KEY put name.toString()
-    }
-
     public companion object {
         public const val IO_FORMAT_TYPE: String = "io.format"
+        public val NAME_KEY: Name = "name".asName()
+        public val META_KEY: Name = "meta".asName()
     }
 }
 
-public fun <T : Any> IOFormat<T>.toBinary(obj: T): Binary = Binary { writeObject(this, obj) }
+public fun <T : Any> Binary(obj: T, format: IOWriter<T>): Binary = Binary { format.writeObject(this, obj) }
 
 public object DoubleIOFormat : IOFormat<Double>, IOFormatFactory<Double> {
     override fun build(context: Context, meta: Meta): IOFormat<Double> = this
@@ -113,21 +75,4 @@ public object DoubleIOFormat : IOFormat<Double>, IOFormatFactory<Double> {
     }
 
     override fun readObject(input: Input): Double = input.readDouble()
-}
-
-//public object ValueIOFormat : IOFormat<Value>, IOFormatFactory<Value> {
-//    override fun invoke(meta: Meta, context: Context): IOFormat<Value> = this
-//
-//    override val name: Name = "value".asName()
-//
-//    override val type: KType get() = typeOf<Value>()
-//
-//    override fun writeObject(output: Output, obj: Value) {
-//        BinaryMetaFormat.run { output.writeValue(obj) }
-//    }
-//
-//    override fun readObject(input: Input): Value {
-//        return (BinaryMetaFormat.run { input.readMetaItem() } as? MetaItemValue)?.value
-//            ?: error("The item is not a value")
-//    }
-//}
\ No newline at end of file
+}
\ No newline at end of file
diff --git a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/IOPlugin.kt b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/IOPlugin.kt
index 9e1f0dd1..dc6541f2 100644
--- a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/IOPlugin.kt
+++ b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/IOPlugin.kt
@@ -2,8 +2,6 @@ package space.kscience.dataforge.io
 
 import space.kscience.dataforge.context.*
 import space.kscience.dataforge.io.EnvelopeFormatFactory.Companion.ENVELOPE_FORMAT_TYPE
-import space.kscience.dataforge.io.IOFormat.Companion.META_KEY
-import space.kscience.dataforge.io.IOFormat.Companion.NAME_KEY
 import space.kscience.dataforge.io.IOFormatFactory.Companion.IO_FORMAT_TYPE
 import space.kscience.dataforge.io.MetaFormatFactory.Companion.META_FORMAT_TYPE
 import space.kscience.dataforge.meta.Meta
@@ -20,12 +18,12 @@ public class IOPlugin(meta: Meta) : AbstractPlugin(meta) {
     }
 
     public fun <T : Any> resolveIOFormat(item: Meta, type: KClass<out T>): IOFormat<T>? {
-        val key = item.string ?: item[NAME_KEY]?.string ?: error("Format name not defined")
+        val key = item.string ?: item[IOFormatFactory.NAME_KEY]?.string ?: error("Format name not defined")
         val name = Name.parse(key)
         return ioFormatFactories.find { it.name == name }?.let {
             @Suppress("UNCHECKED_CAST")
             if (it.type != type) error("Format type ${it.type} is not the same as requested type $type")
-            else it.build(context, item[META_KEY] ?: Meta.EMPTY) as IOFormat<T>
+            else it.build(context, item[IOFormatFactory.META_KEY] ?: Meta.EMPTY) as IOFormat<T>
         }
     }
 
@@ -47,8 +45,8 @@ public class IOPlugin(meta: Meta) : AbstractPlugin(meta) {
         envelopeFormatFactories.find { it.name == name }?.build(context, meta)
 
     public fun resolveEnvelopeFormat(item: Meta): EnvelopeFormat? {
-        val name = item.string ?: item[NAME_KEY]?.string ?: error("Envelope format name not defined")
-        val meta = item[META_KEY] ?: Meta.EMPTY
+        val name = item.string ?: item[IOFormatFactory.NAME_KEY]?.string ?: error("Envelope format name not defined")
+        val meta = item[IOFormatFactory.META_KEY] ?: Meta.EMPTY
         return resolveEnvelopeFormat(Name.parse(name), meta)
     }
 
diff --git a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/JsonMetaFormat.kt b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/JsonMetaFormat.kt
index d5beee22..4bb3c82d 100644
--- a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/JsonMetaFormat.kt
+++ b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/JsonMetaFormat.kt
@@ -8,30 +8,21 @@ import io.ktor.utils.io.core.Output
 import kotlinx.serialization.json.Json
 import kotlinx.serialization.json.JsonObject
 import space.kscience.dataforge.context.Context
-import space.kscience.dataforge.io.IOFormat.Companion.NAME_KEY
 import space.kscience.dataforge.meta.Meta
 import space.kscience.dataforge.meta.descriptors.MetaDescriptor
 import space.kscience.dataforge.meta.toJson
 import space.kscience.dataforge.meta.toMeta
-import kotlin.reflect.KType
-import kotlin.reflect.typeOf
 
 /**
  * A Json format for Meta representation
  */
 public class JsonMetaFormat(private val json: Json = DEFAULT_JSON) : MetaFormat {
 
-    override val type: KType get() = typeOf<Meta>()
-
     override fun writeMeta(output: Output, meta: Meta, descriptor: MetaDescriptor?) {
         val jsonObject = meta.toJson(descriptor)
         output.writeUtf8String(json.encodeToString(JsonObject.serializer(), jsonObject))
     }
 
-    override fun toMeta(): Meta = Meta {
-        NAME_KEY put name.toString()
-    }
-
     override fun readMeta(input: Input, descriptor: MetaDescriptor?): Meta {
         val str = input.readUtf8String()//readByteArray().decodeToString()
         val jsonElement = json.parseToJsonElement(str)
diff --git a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/MetaFormat.kt b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/MetaFormat.kt
index 46d7b540..5e009a12 100644
--- a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/MetaFormat.kt
+++ b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/MetaFormat.kt
@@ -20,7 +20,6 @@ import kotlin.reflect.typeOf
  * A format for meta serialization
  */
 public interface MetaFormat : IOFormat<Meta> {
-    override val type: KType get() = typeOf<Meta>()
 
     override fun writeObject(output: Output, obj: Meta) {
         writeMeta(output, obj, null)
diff --git a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/TaggedEnvelopeFormat.kt b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/TaggedEnvelopeFormat.kt
index 933dc343..2e5d3cc6 100644
--- a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/TaggedEnvelopeFormat.kt
+++ b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/TaggedEnvelopeFormat.kt
@@ -3,8 +3,6 @@ package space.kscience.dataforge.io
 import io.ktor.utils.io.core.*
 import space.kscience.dataforge.context.Context
 import space.kscience.dataforge.context.Global
-import space.kscience.dataforge.io.IOFormat.Companion.META_KEY
-import space.kscience.dataforge.io.IOFormat.Companion.NAME_KEY
 import space.kscience.dataforge.meta.Meta
 import space.kscience.dataforge.meta.enum
 import space.kscience.dataforge.meta.get
@@ -49,7 +47,7 @@ public class TaggedEnvelopeFormat(
         formatMeta: Meta,
     ) {
         val metaFormat = metaFormatFactory.build(this@TaggedEnvelopeFormat.io.context, formatMeta)
-        val metaBytes = metaFormat.toBinary(envelope.meta)
+        val metaBytes = Binary(envelope.meta,metaFormat)
         val actualSize: ULong = (envelope.data?.size ?: 0).toULong()
         val tag = Tag(metaFormatFactory.key, metaBytes.size.toUInt() + 2u, actualSize)
         output.writeBinary(tag.toBinary())
@@ -74,7 +72,7 @@ public class TaggedEnvelopeFormat(
 
         val metaBinary = input.readBinary(tag.metaSize.toInt())
 
-        val meta: Meta = metaFormat.readObject(metaBinary)
+        val meta: Meta = metaFormat.readObjectFrom(metaBinary)
 
         val data = input.readBinary(tag.dataSize.toInt())
 
@@ -89,7 +87,7 @@ public class TaggedEnvelopeFormat(
 
         val metaBinary = input.readBinary(tag.metaSize.toInt())
 
-        val meta: Meta = metaFormat.readObject(metaBinary)
+        val meta: Meta = metaFormat.readObjectFrom(metaBinary)
 
 
         return PartialEnvelope(meta, (version.tagSize + tag.metaSize).toInt(), tag.dataSize)
@@ -106,13 +104,6 @@ public class TaggedEnvelopeFormat(
         DF03(24u)
     }
 
-    override fun toMeta(): Meta = Meta {
-        NAME_KEY put name.toString()
-        META_KEY put {
-            "version" put version
-        }
-    }
-
     public companion object : EnvelopeFormatFactory {
         private const val START_SEQUENCE = "#~"
         private const val END_SEQUENCE = "~#\r\n"
diff --git a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/TaglessEnvelopeFormat.kt b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/TaglessEnvelopeFormat.kt
index 9a0f4a98..4e2bacfc 100644
--- a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/TaglessEnvelopeFormat.kt
+++ b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/TaglessEnvelopeFormat.kt
@@ -3,8 +3,6 @@ package space.kscience.dataforge.io
 import io.ktor.utils.io.core.*
 import space.kscience.dataforge.context.Context
 import space.kscience.dataforge.context.Global
-import space.kscience.dataforge.io.IOFormat.Companion.META_KEY
-import space.kscience.dataforge.io.IOFormat.Companion.NAME_KEY
 import space.kscience.dataforge.meta.Meta
 import space.kscience.dataforge.meta.get
 import space.kscience.dataforge.meta.isEmpty
@@ -50,11 +48,11 @@ public class TaglessEnvelopeFormat(
 
         //Printing meta
         if (!envelope.meta.isEmpty()) {
-            val metaBytes = metaFormat.toBinary(envelope.meta)
+            val metaBinary = Binary(envelope.meta, metaFormat)
             output.writeProperty(META_LENGTH_PROPERTY,
-                metaBytes.size + 2)
+                metaBinary.size + 2)
             output.writeUtf8String(this.metaStart + "\r\n")
-            output.writeBinary(metaBytes)
+            output.writeBinary(metaBinary)
             output.writeRawString("\r\n")
         }
 
@@ -102,7 +100,7 @@ public class TaglessEnvelopeFormat(
             val metaFormat = properties[META_TYPE_PROPERTY]?.let { io.resolveMetaFormat(it) } ?: JsonMetaFormat
             val metaSize = properties[META_LENGTH_PROPERTY]?.toInt()
             meta = if (metaSize != null) {
-                metaFormat.readObject(input.readBinary(metaSize))
+                metaFormat.readObjectFrom(input.readBinary(metaSize))
             } else {
                 error("Can't partially read an envelope with undefined meta size")
             }
@@ -170,7 +168,7 @@ public class TaglessEnvelopeFormat(
             val metaSize = properties[META_LENGTH_PROPERTY]?.toInt()
             meta = if (metaSize != null) {
                 offset += metaSize
-                metaFormat.readObject(input.readBinary(metaSize))
+                metaFormat.readObjectFrom(input.readBinary(metaSize))
             } else {
                 error("Can't partially read an envelope with undefined meta size")
             }
@@ -187,11 +185,6 @@ public class TaglessEnvelopeFormat(
         return PartialEnvelope(meta, offset, dataSize)
     }
 
-    override fun toMeta(): Meta = Meta {
-        NAME_KEY put name.toString()
-        META_KEY put meta
-    }
-
     public companion object : EnvelopeFormatFactory {
 
         private val propertyPattern = "#\\?\\s*([\\w.]*)\\s*:\\s*([^;]*);?".toRegex()
diff --git a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/ioMisc.kt b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/ioMisc.kt
index e72c0eed..5c17e9ac 100644
--- a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/ioMisc.kt
+++ b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/ioMisc.kt
@@ -183,7 +183,7 @@ public fun Input.discardWithSeparator(
     atMost: Int = Int.MAX_VALUE,
     skipUntilEndOfLine: Boolean = false,
 ): Int {
-    val dummy: Output = object :Output(ChunkBuffer.Pool){
+    val dummy: Output = object : Output(ChunkBuffer.Pool) {
         override fun closeDestination() {
             // Do nothing
         }
diff --git a/dataforge-io/src/commonTest/kotlin/space/kscience/dataforge/io/MultipartTest.kt b/dataforge-io/src/commonTest/kotlin/space/kscience/dataforge/io/MultipartTest.kt
index 8ffaef14..6315d744 100644
--- a/dataforge-io/src/commonTest/kotlin/space/kscience/dataforge/io/MultipartTest.kt
+++ b/dataforge-io/src/commonTest/kotlin/space/kscience/dataforge/io/MultipartTest.kt
@@ -32,18 +32,17 @@ class MultipartTest {
 
     @Test
     fun testParts() {
-        TaglessEnvelopeFormat.run {
-            val singleEnvelopeData = toBinary(envelopes[0])
-            val singleEnvelopeSize = singleEnvelopeData.size
-            val bytes = toBinary(partsEnvelope)
-            assertTrue(envelopes.size * singleEnvelopeSize < bytes.size)
-            val reconstructed = bytes.readWith(this)
-            println(reconstructed.meta)
-            val parts = reconstructed.parts()
-            val envelope = parts[2].envelope(io)
-            assertEquals(2, envelope.meta["value"].int)
-            println(reconstructed.data!!.size)
-        }
+        val format = TaglessEnvelopeFormat
+        val singleEnvelopeData = Binary(envelopes[0], format)
+        val singleEnvelopeSize = singleEnvelopeData.size
+        val bytes = Binary(partsEnvelope, format)
+        assertTrue(envelopes.size * singleEnvelopeSize < bytes.size)
+        val reconstructed = bytes.readWith(format)
+        println(reconstructed.meta)
+        val parts = reconstructed.parts()
+        val envelope = parts[2].envelope(io)
+        assertEquals(2, envelope.meta["value"].int)
+        println(reconstructed.data!!.size)
     }
 
 }
\ No newline at end of file
diff --git a/dataforge-io/src/jvmMain/kotlin/space/kscience/dataforge/io/fileIO.kt b/dataforge-io/src/jvmMain/kotlin/space/kscience/dataforge/io/fileIO.kt
index 1f2d3041..64f4907e 100644
--- a/dataforge-io/src/jvmMain/kotlin/space/kscience/dataforge/io/fileIO.kt
+++ b/dataforge-io/src/jvmMain/kotlin/space/kscience/dataforge/io/fileIO.kt
@@ -97,7 +97,36 @@ public val IOPlugin.Companion.DATA_FILE_NAME: String get() = "@data"
 
 /**
  * Read file containing meta using given [formatOverride] or file extension to infer meta type.
- * If [path] is a directory search for file starting with `meta` in it
+ * If [path] is a directory search for file starting with `meta` in it.
+ *
+ * Returns null if meta could not be resolved
+ */
+public fun IOPlugin.readMetaFileOrNull(
+    path: Path,
+    formatOverride: MetaFormat? = null,
+    descriptor: MetaDescriptor? = null,
+): Meta? {
+    if (!Files.exists(path)) return null
+
+    val actualPath: Path = if (Files.isDirectory(path)) {
+        Files.list(path).asSequence().find { it.fileName.startsWith(IOPlugin.META_FILE_NAME) }
+            ?: return null
+    } else {
+        path
+    }
+    val extension = actualPath.fileName.toString().substringAfterLast('.')
+
+    val metaFormat = formatOverride ?: resolveMetaFormat(extension) ?: return null
+    return actualPath.read {
+        metaFormat.readMeta(this, descriptor)
+    }
+}
+
+/**
+ * Read file containing meta using given [formatOverride] or file extension to infer meta type.
+ * If [path] is a directory search for file starting with `meta` in it.
+ *
+ * Fails if nothing works.
  */
 public fun IOPlugin.readMetaFile(
     path: Path,
@@ -120,6 +149,7 @@ public fun IOPlugin.readMetaFile(
     }
 }
 
+
 /**
  * Write meta to file using [metaFormat]. If [path] is a directory, write a file with name equals name of [metaFormat].
  * Like "meta.json"
@@ -196,22 +226,11 @@ public fun IOPlugin.readEnvelopeFile(
         return SimpleEnvelope(meta, data)
     }
 
-    return formatPicker(path)?.let { format ->
-        format.readFile(path)
-    } ?: if (readNonEnvelopes) { // if no format accepts file, read it as binary
+    return formatPicker(path)?.readFile(path) ?: if (readNonEnvelopes) { // if no format accepts file, read it as binary
         SimpleEnvelope(Meta.EMPTY, path.asBinary())
     } else error("Can't infer format for file $path")
 }
 
-/**
- * Write a binary into file. Throws an error if file already exists
- */
-public fun <T : Any> IOFormat<T>.writeToFile(path: Path, obj: T) {
-    path.write {
-        writeObject(this, obj)
-    }
-}
-
 /**
  * Write envelope file to given [path] using [envelopeFormat] and optional [metaFormat]
  */
diff --git a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/envelopeData.kt b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/envelopeData.kt
index 55c98352..ce5be133 100644
--- a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/envelopeData.kt
+++ b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/envelopeData.kt
@@ -4,19 +4,18 @@ import space.kscience.dataforge.data.Data
 import space.kscience.dataforge.data.await
 import space.kscience.dataforge.io.*
 import space.kscience.dataforge.misc.DFInternal
+import kotlin.reflect.typeOf
 
 /**
  * Convert an [Envelope] to a data via given format. The actual parsing is done lazily.
  */
 @OptIn(DFInternal::class)
-public fun <T : Any> Envelope.toData(format: IOFormat<T>): Data<T> {
-    return Data(format.type, meta) {
-        data?.readWith(format) ?: error("Can't convert envelope without data to Data")
-    }
+public inline fun <reified T : Any> Envelope.toData(format: IOReader<T>): Data<T> = Data(typeOf<T>(), meta) {
+    data?.readWith(format) ?: error("Can't convert envelope without data to Data")
 }
 
-public suspend fun <T : Any> Data<T>.toEnvelope(format: IOFormat<T>): Envelope {
+public suspend fun <T : Any> Data<T>.toEnvelope(format: IOWriter<T>): Envelope {
     val obj = await()
-    val binary = format.toBinary(obj)
+    val binary = Binary(obj, format)
     return SimpleEnvelope(meta, binary)
 }
\ No newline at end of file
diff --git a/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/fileData.kt b/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/fileData.kt
index d758c45c..aa3ddede 100644
--- a/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/fileData.kt
+++ b/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/fileData.kt
@@ -2,11 +2,12 @@ package space.kscience.dataforge.workspace
 
 import io.ktor.utils.io.streams.asOutput
 import kotlinx.coroutines.Dispatchers
-import kotlinx.coroutines.runBlocking
 import kotlinx.coroutines.withContext
 import space.kscience.dataforge.data.*
 import space.kscience.dataforge.io.*
-import space.kscience.dataforge.meta.*
+import space.kscience.dataforge.meta.Meta
+import space.kscience.dataforge.meta.get
+import space.kscience.dataforge.meta.string
 import space.kscience.dataforge.misc.DFExperimental
 import java.nio.file.FileSystem
 import java.nio.file.Files
@@ -15,28 +16,13 @@ import java.nio.file.StandardOpenOption
 import java.nio.file.spi.FileSystemProvider
 import java.util.zip.ZipEntry
 import java.util.zip.ZipOutputStream
-import kotlin.reflect.KType
-import kotlin.reflect.typeOf
 import kotlin.streams.toList
 
 //public typealias FileFormatResolver<T> = (Path, Meta) -> IOFormat<T>
 
-public interface FileFormatResolver<T : Any> {
-    public val type: KType
-    public operator fun invoke(path: Path, meta: Meta): IOFormat<T>
-}
+public typealias FileFormatResolver<T> = (path: Path, meta: Meta) -> IOReader<T>
 
 
-@PublishedApi
-internal inline fun <reified T : Any> IOPlugin.formatResolver(): FileFormatResolver<T> =
-    object : FileFormatResolver<T> {
-        override val type: KType = typeOf<T>()
-
-        @OptIn(DFExperimental::class)
-        override fun invoke(path: Path, meta: Meta): IOFormat<T> =
-            resolveIOFormat<T>() ?: error("Can't resolve IO format for ${T::class}")
-    }
-
 private fun newZFS(path: Path): FileSystem {
     val fsProvider = FileSystemProvider.installedProviders().find { it.scheme == "jar" }
         ?: error("Zip file system provider not found")
@@ -46,14 +32,9 @@ private fun newZFS(path: Path): FileSystem {
 /**
  * Read data with supported envelope format and binary format. If envelope format is null, then read binary directly from file.
  * The operation is blocking since it must read meta header. The reading of envelope body is lazy
- * @param type explicit type of data read
- * @param dataFormat binary format
- * @param envelopeFormat the format of envelope. If null, file is read directly
- * @param metaFile the relative file for optional meta override
- * @param metaFileFormat the meta format for override
  */
 @DFExperimental
-public fun <T : Any> IOPlugin.readDataFile(
+public inline fun <reified T : Any> IOPlugin.readDataFile(
     path: Path,
     formatResolver: FileFormatResolver<T>,
 ): Data<T> {
@@ -62,34 +43,26 @@ public fun <T : Any> IOPlugin.readDataFile(
     return envelope.toData(format)
 }
 
-@DFExperimental
-public inline fun <reified T : Any> IOPlugin.readDataFile(path: Path): Data<T> = readDataFile(path, formatResolver())
-
 /**
  * Add file/directory-based data tree item
  */
-@DFExperimental
-public suspend fun <T : Any> DataSetBuilder<T>.file(
-    plugin: IOPlugin,
+context(IOPlugin) @DFExperimental
+public fun DataSetBuilder<Any>.file(
     path: Path,
-    formatResolver: FileFormatResolver<T>,
+    formatResolver: FileFormatResolver<Any>,
 ) {
     //If path is a single file or a special directory, read it as single datum
     if (!Files.isDirectory(path) || Files.list(path).allMatch { it.fileName.toString().startsWith("@") }) {
-        plugin.run {
-            val data = readDataFile(path, formatResolver)
-            val name = data.meta[Envelope.ENVELOPE_NAME_KEY].string
-                ?: path.fileName.toString().replace(".df", "")
-            data(name, data)
-        }
+        val data = readDataFile(path, formatResolver)
+        val name = data.meta[Envelope.ENVELOPE_NAME_KEY].string
+            ?: path.fileName.toString().replace(".df", "")
+        data(name, data)
     } else {
         //otherwise, read as directory
-        plugin.run {
-            val data = readDataDirectory(path, formatResolver)
-            val name = data.meta[Envelope.ENVELOPE_NAME_KEY].string
-                ?: path.fileName.toString().replace(".df", "")
-            node(name, data)
-        }
+        val data = readDataDirectory(path, formatResolver)
+        val name = data.meta[Envelope.ENVELOPE_NAME_KEY].string
+            ?: path.fileName.toString().replace(".df", "")
+        node(name, data)
     }
 }
 
@@ -97,10 +70,10 @@ public suspend fun <T : Any> DataSetBuilder<T>.file(
  * Read the directory as a data node. If [path] is a zip archive, read it as directory
  */
 @DFExperimental
-public suspend fun <T : Any> IOPlugin.readDataDirectory(
+public fun IOPlugin.readDataDirectory(
     path: Path,
-    formatResolver: FileFormatResolver<T>,
-): DataTree<T> {
+    formatResolver: FileFormatResolver<Any>,
+): DataTree<Any> {
     //read zipped data node
     if (path.fileName != null && path.fileName.toString().endsWith(".zip")) {
         //Using explicit Zip file system to avoid bizarre compatibility bugs
@@ -108,24 +81,18 @@ public suspend fun <T : Any> IOPlugin.readDataDirectory(
         return readDataDirectory(fs.rootDirectories.first(), formatResolver)
     }
     if (!Files.isDirectory(path)) error("Provided path $path is not a directory")
-    return DataTree(formatResolver.type) {
+    return DataTree<Any> {
         Files.list(path).toList().forEach { path ->
             val fileName = path.fileName.toString()
             if (fileName.startsWith(IOPlugin.META_FILE_NAME)) {
                 meta(readMetaFile(path))
             } else if (!fileName.startsWith("@")) {
-                runBlocking {
-                    file(this@readDataDirectory, path, formatResolver)
-                }
+                file(path, formatResolver)
             }
         }
     }
 }
 
-@DFExperimental
-public suspend inline fun <reified T : Any> IOPlugin.readDataDirectory(path: Path): DataTree<T> =
-    readDataDirectory(path, formatResolver())
-
 /**
  * Write data tree to existing directory or create a new one using default [java.nio.file.FileSystem] provider
  */
@@ -133,7 +100,7 @@ public suspend inline fun <reified T : Any> IOPlugin.readDataDirectory(path: Pat
 public suspend fun <T : Any> IOPlugin.writeDataDirectory(
     path: Path,
     tree: DataTree<T>,
-    format: IOFormat<T>,
+    format: IOWriter<T>,
     envelopeFormat: EnvelopeFormat? = null,
     metaFormat: MetaFormatFactory? = null,
 ) {
@@ -179,11 +146,9 @@ private suspend fun <T : Any> ZipOutputStream.writeNode(
                 val envelope = treeItem.data.toEnvelope(dataFormat)
                 val entry = ZipEntry(name)
                 putNextEntry(entry)
-                envelopeFormat.run {
-                    asOutput().run {
-                        writeEnvelope(this, envelope)
-                        flush()
-                    }
+                asOutput().run {
+                    envelopeFormat.writeEnvelope(this, envelope)
+                    flush()
                 }
             }
             is DataTreeItem.Node -> {
diff --git a/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/FileDataTest.kt b/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/FileDataTest.kt
index d1bbb606..a47a648d 100644
--- a/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/FileDataTest.kt
+++ b/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/FileDataTest.kt
@@ -9,12 +9,8 @@ import space.kscience.dataforge.io.IOFormat
 import space.kscience.dataforge.io.io
 import space.kscience.dataforge.io.readUtf8String
 import space.kscience.dataforge.io.writeUtf8String
-import space.kscience.dataforge.meta.Meta
 import space.kscience.dataforge.misc.DFExperimental
 import java.nio.file.Files
-import java.nio.file.Path
-import kotlin.reflect.KType
-import kotlin.reflect.typeOf
 import kotlin.test.Test
 import kotlin.test.assertEquals
 
@@ -35,8 +31,6 @@ class FileDataTest {
 
     object StringIOFormat : IOFormat<String> {
 
-        override val type: KType = typeOf<String>()
-
         override fun writeObject(output: Output, obj: String) {
             output.writeUtf8String(obj)
         }
@@ -45,17 +39,6 @@ class FileDataTest {
             return input.readUtf8String()
         }
 
-        override fun toMeta(): Meta = Meta {
-            IOFormat.NAME_KEY put "string"
-        }
-
-    }
-
-    object StringFormatResolver : FileFormatResolver<String> {
-        override val type: KType = typeOf<String>()
-
-        override fun invoke(path: Path, meta: Meta): IOFormat<String> = StringIOFormat
-
     }
 
     @Test
@@ -66,7 +49,7 @@ class FileDataTest {
             runBlocking {
                 writeDataDirectory(dir, dataNode, StringIOFormat)
                 println(dir.toUri().toString())
-                val reconstructed = readDataDirectory(dir, StringFormatResolver)
+                val reconstructed = readDataDirectory(dir) { _, _ -> StringIOFormat }
                 assertEquals(dataNode["dir.a"]?.meta, reconstructed["dir.a"]?.meta)
                 assertEquals(dataNode["b"]?.await(), reconstructed["b"]?.await())
             }
@@ -82,7 +65,7 @@ class FileDataTest {
             runBlocking {
                 writeZip(zip, dataNode, StringIOFormat)
                 println(zip.toUri().toString())
-                val reconstructed = readDataDirectory(zip, StringFormatResolver)
+                val reconstructed = readDataDirectory(zip) { _, _ -> StringIOFormat }
                 assertEquals(dataNode["dir.a"]?.meta, reconstructed["dir.a"]?.meta)
                 assertEquals(dataNode["b"]?.await(), reconstructed["b"]?.await())
             }
diff --git a/gradle.properties b/gradle.properties
index 0de6ab36..1aeadb9b 100644
--- a/gradle.properties
+++ b/gradle.properties
@@ -4,4 +4,4 @@ kotlin.code.style=official
 
 kotlin.mpp.stability.nowarn=true
 
-toolsVersion=0.11.4-kotlin-1.6.20
+toolsVersion=0.11.5-kotlin-1.6.21

From f5d32ba5111b8a6427a2208442548770250304ac Mon Sep 17 00:00:00 2001
From: Alexander Nozik <altavir@gmail.com>
Date: Tue, 17 May 2022 12:24:15 +0300
Subject: [PATCH 16/19] Update build version

---
 build.gradle.kts                                           | 2 +-
 .../kotlin/space/kscience/dataforge/io/IOFormat.kt         | 2 +-
 .../kotlin/space/kscience/dataforge/io/IOPlugin.kt         | 7 +++++--
 3 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/build.gradle.kts b/build.gradle.kts
index 765e242f..cec3fa57 100644
--- a/build.gradle.kts
+++ b/build.gradle.kts
@@ -6,7 +6,7 @@ plugins {
 
 allprojects {
     group = "space.kscience"
-    version = "0.6.0-dev-7"
+    version = "0.6.0-dev-8"
 
     tasks.withType<KotlinCompile>{
         kotlinOptions{
diff --git a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/IOFormat.kt b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/IOFormat.kt
index 985a1e69..9ca6a4b0 100644
--- a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/IOFormat.kt
+++ b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/IOFormat.kt
@@ -44,7 +44,7 @@ public fun <T : Any> Binary.readWith(format: IOReader<T>): T = read {
 }
 
 public fun <T : Any> Output.writeObject(format: IOWriter<T>, obj: T): Unit =
-    format.run { writeObject(this@writeObject, obj) }
+    format.writeObject(this@writeObject, obj)
 
 
 @Type(IO_FORMAT_TYPE)
diff --git a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/IOPlugin.kt b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/IOPlugin.kt
index dc6541f2..7b1a6b2c 100644
--- a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/IOPlugin.kt
+++ b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/IOPlugin.kt
@@ -53,13 +53,16 @@ public class IOPlugin(meta: Meta) : AbstractPlugin(meta) {
     override fun content(target: String): Map<Name, Any> = when (target) {
         META_FORMAT_TYPE -> defaultMetaFormats.toMap()
         ENVELOPE_FORMAT_TYPE -> defaultEnvelopeFormats.toMap()
+        IO_FORMAT_TYPE -> content(META_FORMAT_TYPE) + content(ENVELOPE_FORMAT_TYPE)
         else -> super.content(target)
     }
 
     public companion object : PluginFactory<IOPlugin> {
         public val defaultMetaFormats: List<MetaFormatFactory> = listOf(JsonMetaFormat)
-        public val defaultEnvelopeFormats: List<EnvelopeFormatFactory> =
-            listOf(TaggedEnvelopeFormat, TaglessEnvelopeFormat)
+        public val defaultEnvelopeFormats: List<EnvelopeFormatFactory> = listOf(
+            TaggedEnvelopeFormat,
+            TaglessEnvelopeFormat
+        )
 
         override val tag: PluginTag = PluginTag("io", group = PluginTag.DATAFORGE_GROUP)
 

From 6bd8a7acbc22957b26f10a76f69be8f14ed7745c Mon Sep 17 00:00:00 2001
From: Alexander Nozik <altavir@gmail.com>
Date: Sat, 21 May 2022 10:38:53 +0300
Subject: [PATCH 17/19] return type to IOReader

---
 build.gradle.kts                              |  10 +-
 .../io/yaml/FrontMatterEnvelopeFormatTest.kt  |   3 +
 .../kscience/dataforge/io/EnvelopeFormat.kt   |   3 +
 .../space/kscience/dataforge/io/IOFormat.kt   |  17 +-
 .../space/kscience/dataforge/io/MetaFormat.kt |   2 +
 .../space/kscience/dataforge/meta/Meta.kt     |   6 +-
 .../kscience/dataforge/names/NameToken.kt     |  15 ++
 .../dataforge/workspace/envelopeData.kt       |  11 +-
 .../kscience/dataforge/workspace/fileData.kt  | 209 ++++++++++++++----
 .../dataforge/workspace/FileDataTest.kt       |  15 +-
 gradle.properties                             |   3 +-
 11 files changed, 225 insertions(+), 69 deletions(-)

diff --git a/build.gradle.kts b/build.gradle.kts
index cec3fa57..e3961065 100644
--- a/build.gradle.kts
+++ b/build.gradle.kts
@@ -6,7 +6,11 @@ plugins {
 
 allprojects {
     group = "space.kscience"
-    version = "0.6.0-dev-8"
+    version = "0.6.0-dev-9"
+}
+
+subprojects {
+    apply(plugin = "maven-publish")
 
     tasks.withType<KotlinCompile>{
         kotlinOptions{
@@ -15,10 +19,6 @@ allprojects {
     }
 }
 
-subprojects {
-    apply(plugin = "maven-publish")
-}
-
 readme {
     readmeTemplate = file("docs/templates/README-TEMPLATE.md")
 }
diff --git a/dataforge-io/dataforge-io-yaml/src/commonTest/kotlin/space/kscience/dataforge/io/yaml/FrontMatterEnvelopeFormatTest.kt b/dataforge-io/dataforge-io-yaml/src/commonTest/kotlin/space/kscience/dataforge/io/yaml/FrontMatterEnvelopeFormatTest.kt
index 0c242597..7eb69913 100644
--- a/dataforge-io/dataforge-io-yaml/src/commonTest/kotlin/space/kscience/dataforge/io/yaml/FrontMatterEnvelopeFormatTest.kt
+++ b/dataforge-io/dataforge-io-yaml/src/commonTest/kotlin/space/kscience/dataforge/io/yaml/FrontMatterEnvelopeFormatTest.kt
@@ -1,3 +1,5 @@
+@file:OptIn(DFExperimental::class)
+
 package space.kscience.dataforge.io.yaml
 
 import space.kscience.dataforge.context.Context
@@ -6,6 +8,7 @@ import space.kscience.dataforge.io.readEnvelope
 import space.kscience.dataforge.io.toByteArray
 import space.kscience.dataforge.meta.get
 import space.kscience.dataforge.meta.string
+import space.kscience.dataforge.misc.DFExperimental
 import kotlin.test.Test
 import kotlin.test.assertEquals
 
diff --git a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/EnvelopeFormat.kt b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/EnvelopeFormat.kt
index b705a3c0..502bcba6 100644
--- a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/EnvelopeFormat.kt
+++ b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/EnvelopeFormat.kt
@@ -17,6 +17,9 @@ import kotlin.reflect.typeOf
 public data class PartialEnvelope(val meta: Meta, val dataOffset: Int, val dataSize: ULong?)
 
 public interface EnvelopeFormat : IOFormat<Envelope> {
+
+    override val type: KType get() = typeOf<Envelope>()
+    
     public val defaultMetaFormat: MetaFormatFactory get() = JsonMetaFormat
 
     public fun readPartial(input: Input): PartialEnvelope
diff --git a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/IOFormat.kt b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/IOFormat.kt
index 9ca6a4b0..75c53e70 100644
--- a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/IOFormat.kt
+++ b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/IOFormat.kt
@@ -15,12 +15,25 @@ import space.kscience.dataforge.names.asName
 import kotlin.reflect.KType
 import kotlin.reflect.typeOf
 
-public fun interface IOReader<out T : Any> {
+/**
+ * Reader of a custom object from input
+ */
+public interface IOReader<out T> {
+    /**
+     * The type of object being read
+     */
+    public val type: KType
 
     public fun readObject(input: Input): T
 }
 
-public fun interface IOWriter<in T : Any> {
+public inline fun <reified T : Any> IOReader(crossinline read: Input.() -> T): IOReader<T> = object : IOReader<T> {
+    override val type: KType = typeOf<T>()
+
+    override fun readObject(input: Input): T = input.read()
+}
+
+public fun interface IOWriter<in T> {
     public fun writeObject(output: Output, obj: T)
 }
 
diff --git a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/MetaFormat.kt b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/MetaFormat.kt
index 5e009a12..c902c2e6 100644
--- a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/MetaFormat.kt
+++ b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/MetaFormat.kt
@@ -21,6 +21,8 @@ import kotlin.reflect.typeOf
  */
 public interface MetaFormat : IOFormat<Meta> {
 
+    override val type: KType get() = typeOf<Meta>()
+
     override fun writeObject(output: Output, obj: Meta) {
         writeMeta(output, obj, null)
     }
diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/Meta.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/Meta.kt
index e8f392ac..01a5ebf2 100644
--- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/Meta.kt
+++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/Meta.kt
@@ -102,12 +102,14 @@ public operator fun Meta.get(token: NameToken): Meta? = items[token]
  *
  * If [name] is empty return current [Meta]
  */
-public operator fun Meta.get(name: Name): Meta? = getMeta(name)
+public operator fun Meta.get(name: Name): Meta? = this.getMeta(name)
+
+//TODO allow nullable receivers after Kotlin 1.7
 
 /**
  * Parse [Name] from [key] using full name notation and pass it to [Meta.get]
  */
-public operator fun Meta.get(key: String): Meta? = this[Name.parse(key)]
+public operator fun Meta.get(key: String): Meta? = this.get(Name.parse(key))
 
 /**
  * Get all items matching given name. The index of the last element, if present is used as a [Regex],
diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/names/NameToken.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/names/NameToken.kt
index 25d8c28b..9a4ac79c 100644
--- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/names/NameToken.kt
+++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/names/NameToken.kt
@@ -1,6 +1,7 @@
 package space.kscience.dataforge.names
 
 import kotlinx.serialization.Serializable
+import space.kscience.dataforge.misc.DFExperimental
 
 /**
  * A single name token. Body is not allowed to be empty.
@@ -25,6 +26,20 @@ public data class NameToken(val body: String, val index: String? = null) {
     } else {
         body.escape()
     }
+
+    public companion object {
+
+        /**
+         * Parse name token from a string
+         */
+        @DFExperimental
+        public fun parse(string: String): NameToken {
+            val body = string.substringBefore('[')
+            val index = string.substringAfter('[', "")
+            if (index.isNotEmpty() && index.endsWith(']')) error("NameToken with index must end with ']'")
+            return NameToken(body,index.removeSuffix("]"))
+        }
+    }
 }
 
 /**
diff --git a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/envelopeData.kt b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/envelopeData.kt
index ce5be133..d53a8979 100644
--- a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/envelopeData.kt
+++ b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/envelopeData.kt
@@ -4,15 +4,20 @@ import space.kscience.dataforge.data.Data
 import space.kscience.dataforge.data.await
 import space.kscience.dataforge.io.*
 import space.kscience.dataforge.misc.DFInternal
+import kotlin.reflect.KType
 import kotlin.reflect.typeOf
 
+
+@DFInternal
+public fun <T : Any> Envelope.toData(type: KType, format: IOReader<T>): Data<T> = Data(type, meta) {
+    data?.readWith(format) ?: error("Can't convert envelope without data to Data")
+}
+
 /**
  * Convert an [Envelope] to a data via given format. The actual parsing is done lazily.
  */
 @OptIn(DFInternal::class)
-public inline fun <reified T : Any> Envelope.toData(format: IOReader<T>): Data<T> = Data(typeOf<T>(), meta) {
-    data?.readWith(format) ?: error("Can't convert envelope without data to Data")
-}
+public inline fun <reified T : Any> Envelope.toData(format: IOReader<T>): Data<T> = toData(typeOf<T>(), format)
 
 public suspend fun <T : Any> Data<T>.toEnvelope(format: IOWriter<T>): Envelope {
     val obj = await()
diff --git a/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/fileData.kt b/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/fileData.kt
index aa3ddede..c15972b8 100644
--- a/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/fileData.kt
+++ b/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/fileData.kt
@@ -2,67 +2,90 @@ package space.kscience.dataforge.workspace
 
 import io.ktor.utils.io.streams.asOutput
 import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.isActive
+import kotlinx.coroutines.launch
 import kotlinx.coroutines.withContext
 import space.kscience.dataforge.data.*
 import space.kscience.dataforge.io.*
 import space.kscience.dataforge.meta.Meta
+import space.kscience.dataforge.meta.copy
 import space.kscience.dataforge.meta.get
 import space.kscience.dataforge.meta.string
 import space.kscience.dataforge.misc.DFExperimental
-import java.nio.file.FileSystem
-import java.nio.file.Files
-import java.nio.file.Path
-import java.nio.file.StandardOpenOption
+import space.kscience.dataforge.misc.DFInternal
+import space.kscience.dataforge.names.Name
+import space.kscience.dataforge.names.NameToken
+import space.kscience.dataforge.names.asName
+import space.kscience.dataforge.names.plus
+import java.nio.file.*
+import java.nio.file.attribute.BasicFileAttributes
 import java.nio.file.spi.FileSystemProvider
 import java.util.zip.ZipEntry
 import java.util.zip.ZipOutputStream
+import kotlin.io.path.extension
+import kotlin.io.path.nameWithoutExtension
+import kotlin.io.path.readAttributes
+import kotlin.reflect.KType
+import kotlin.reflect.typeOf
 import kotlin.streams.toList
 
+
 //public typealias FileFormatResolver<T> = (Path, Meta) -> IOFormat<T>
 
 public typealias FileFormatResolver<T> = (path: Path, meta: Meta) -> IOReader<T>
 
-
-private fun newZFS(path: Path): FileSystem {
-    val fsProvider = FileSystemProvider.installedProviders().find { it.scheme == "jar" }
-        ?: error("Zip file system provider not found")
-    return fsProvider.newFileSystem(path, mapOf("create" to "true"))
+public object FileData {
+    public val META_FILE_KEY: Name = "file".asName()
+    public val META_FILE_PATH_KEY: Name = META_FILE_KEY + "path"
+    public val META_FILE_EXTENSION_KEY: Name = META_FILE_KEY + "extension"
+    public val META_FILE_CREATE_TIME_KEY: Name = META_FILE_KEY + "created"
+    public val META_FILE_UPDATE_TIME_KEY: Name = META_FILE_KEY + "update"
 }
 
-/**
- * Read data with supported envelope format and binary format. If envelope format is null, then read binary directly from file.
- * The operation is blocking since it must read meta header. The reading of envelope body is lazy
- */
+
+@DFInternal
 @DFExperimental
-public inline fun <reified T : Any> IOPlugin.readDataFile(
+public fun <T : Any> IOPlugin.readDataFile(
+    type: KType,
     path: Path,
     formatResolver: FileFormatResolver<T>,
 ): Data<T> {
     val envelope = readEnvelopeFile(path, true)
     val format = formatResolver(path, envelope.meta)
-    return envelope.toData(format)
+    val updatedMeta = envelope.meta.copy {
+        FileData.META_FILE_PATH_KEY put path.toString()
+        FileData.META_FILE_EXTENSION_KEY put path.extension
+
+        val attributes = path.readAttributes<BasicFileAttributes>()
+        FileData.META_FILE_UPDATE_TIME_KEY put attributes.lastModifiedTime().toInstant().toString()
+        FileData.META_FILE_CREATE_TIME_KEY put attributes.creationTime().toInstant().toString()
+    }
+    return Data(type, updatedMeta) {
+        envelope.data?.readWith(format) ?: error("Can't convert envelope without content to Data")
+    }
 }
 
+
 /**
- * Add file/directory-based data tree item
+ * Read data with supported envelope format and binary format. If envelope format is null, then read binary directly from file.
+ * The operation is blocking since it must read meta header. The reading of envelope body is lazy
  */
-context(IOPlugin) @DFExperimental
-public fun DataSetBuilder<Any>.file(
+@OptIn(DFInternal::class)
+@DFExperimental
+public inline fun <reified T : Any> IOPlugin.readDataFile(
     path: Path,
-    formatResolver: FileFormatResolver<Any>,
-) {
-    //If path is a single file or a special directory, read it as single datum
-    if (!Files.isDirectory(path) || Files.list(path).allMatch { it.fileName.toString().startsWith("@") }) {
-        val data = readDataFile(path, formatResolver)
-        val name = data.meta[Envelope.ENVELOPE_NAME_KEY].string
-            ?: path.fileName.toString().replace(".df", "")
-        data(name, data)
-    } else {
-        //otherwise, read as directory
-        val data = readDataDirectory(path, formatResolver)
-        val name = data.meta[Envelope.ENVELOPE_NAME_KEY].string
-            ?: path.fileName.toString().replace(".df", "")
-        node(name, data)
+    noinline formatResolver: FileFormatResolver<T>,
+): Data<T> = readDataFile(typeOf<T>(), path, formatResolver)
+
+context(IOPlugin) @DFExperimental
+private fun <T : Any> DataSetBuilder<T>.directory(path: Path, formatResolver: FileFormatResolver<T>) {
+    Files.list(path).toList().forEach { childPath ->
+        val fileName = childPath.fileName.toString()
+        if (fileName.startsWith(IOPlugin.META_FILE_NAME)) {
+            meta(readMetaFile(childPath))
+        } else if (!fileName.startsWith("@")) {
+            file(childPath, formatResolver)
+        }
     }
 }
 
@@ -70,29 +93,94 @@ public fun DataSetBuilder<Any>.file(
  * Read the directory as a data node. If [path] is a zip archive, read it as directory
  */
 @DFExperimental
-public fun IOPlugin.readDataDirectory(
+@DFInternal
+public fun <T : Any> IOPlugin.readDataDirectory(
+    type: KType,
     path: Path,
-    formatResolver: FileFormatResolver<Any>,
-): DataTree<Any> {
+    formatResolver: FileFormatResolver<T>,
+): DataTree<T> {
     //read zipped data node
     if (path.fileName != null && path.fileName.toString().endsWith(".zip")) {
         //Using explicit Zip file system to avoid bizarre compatibility bugs
-        val fs = newZFS(path)
-        return readDataDirectory(fs.rootDirectories.first(), formatResolver)
+        val fsProvider = FileSystemProvider.installedProviders().find { it.scheme == "jar" }
+            ?: error("Zip file system provider not found")
+        val fs = fsProvider.newFileSystem(path, mapOf("create" to "true"))
+
+        return readDataDirectory(type, fs.rootDirectories.first(), formatResolver)
     }
     if (!Files.isDirectory(path)) error("Provided path $path is not a directory")
-    return DataTree<Any> {
-        Files.list(path).toList().forEach { path ->
-            val fileName = path.fileName.toString()
-            if (fileName.startsWith(IOPlugin.META_FILE_NAME)) {
-                meta(readMetaFile(path))
-            } else if (!fileName.startsWith("@")) {
-                file(path, formatResolver)
-            }
+    return DataTree(type) {
+        directory(path, formatResolver)
+    }
+}
+
+@OptIn(DFInternal::class)
+@DFExperimental
+public inline fun <reified T : Any> IOPlugin.readDataDirectory(
+    path: Path,
+    noinline formatResolver: FileFormatResolver<T>,
+): DataTree<Any> = readDataDirectory(typeOf<T>(), path, formatResolver)
+
+
+
+@OptIn(DFExperimental::class)
+private fun Path.toName() = Name(map { NameToken.parse(it.nameWithoutExtension) })
+
+@DFInternal
+@DFExperimental
+public fun <T : Any> IOPlugin.monitorDataDirectory(
+    type: KType,
+    path: Path,
+    formatResolver: FileFormatResolver<T>,
+): DataSource<T> {
+    if (path.fileName.toString().endsWith(".zip")) error("Monitoring not supported for ZipFS")
+    if (!Files.isDirectory(path)) error("Provided path $path is not a directory")
+    return DataSource(type, context) {
+        directory(path, formatResolver)
+        launch(Dispatchers.IO) {
+            val watchService = path.fileSystem.newWatchService()
+
+            path.register(
+                watchService,
+                StandardWatchEventKinds.ENTRY_DELETE,
+                StandardWatchEventKinds.ENTRY_MODIFY,
+                StandardWatchEventKinds.ENTRY_CREATE
+            )
+
+            do {
+                val key = watchService.take()
+                if (key != null) {
+                    for (event: WatchEvent<*> in key.pollEvents()) {
+                        val eventPath = event.context() as Path
+                        if (event.kind() == StandardWatchEventKinds.ENTRY_DELETE) {
+                            remove(eventPath.toName())
+                        } else {
+                            val fileName = eventPath.fileName.toString()
+                            if (fileName.startsWith(IOPlugin.META_FILE_NAME)) {
+                                meta(readMetaFile(eventPath))
+                            } else if (!fileName.startsWith("@")) {
+                                file(eventPath, formatResolver)
+                            }
+                        }
+                    }
+                    key.reset()
+                }
+            } while (isActive && key != null)
         }
     }
 }
 
+
+/**
+ * Start monitoring given directory ([path]) as a [DataSource].
+ */
+@OptIn(DFInternal::class)
+@DFExperimental
+public inline fun <reified T : Any> IOPlugin.monitorDataDirectory(
+    path: Path,
+    noinline formatResolver: FileFormatResolver<T>,
+): DataSource<T> = monitorDataDirectory(typeOf<T>(), path, formatResolver)
+
 /**
  * Write data tree to existing directory or create a new one using default [java.nio.file.FileSystem] provider
  */
@@ -164,9 +252,8 @@ private suspend fun <T : Any> ZipOutputStream.writeNode(
     }
 }
 
-@Suppress("BlockingMethodInNonBlockingContext")
 @DFExperimental
-public suspend fun <T : Any> IOPlugin.writeZip(
+public suspend fun <T : Any> FileData.writeZip(
     path: Path,
     tree: DataTree<T>,
     format: IOFormat<T>,
@@ -178,10 +265,12 @@ public suspend fun <T : Any> IOPlugin.writeZip(
         } else {
             path.resolveSibling(path.fileName.toString() + ".zip")
         }
-        val fos = Files.newOutputStream(actualFile,
+        val fos = Files.newOutputStream(
+            actualFile,
             StandardOpenOption.WRITE,
             StandardOpenOption.CREATE,
-            StandardOpenOption.TRUNCATE_EXISTING)
+            StandardOpenOption.TRUNCATE_EXISTING
+        )
         val zos = ZipOutputStream(fos)
         zos.use {
             it.writeNode("", DataTreeItem.Node(tree), format, envelopeFormat)
@@ -189,3 +278,25 @@ public suspend fun <T : Any> IOPlugin.writeZip(
     }
 }
 
+/**
+ * Add file/directory-based data tree item
+ */
+context(IOPlugin) @OptIn(DFInternal::class)
+@DFExperimental
+public fun <T : Any> DataSetBuilder<T>.file(
+    path: Path,
+    formatResolver: FileFormatResolver<out T>,
+) {
+    //If path is a single file or a special directory, read it as single datum
+    if (!Files.isDirectory(path) || Files.list(path).allMatch { it.fileName.toString().startsWith("@") }) {
+        val data = readDataFile(dataType, path, formatResolver)
+        val name = data.meta[Envelope.ENVELOPE_NAME_KEY].string ?: path.nameWithoutExtension
+        data(name, data)
+    } else {
+        //otherwise, read as directory
+        val data = readDataDirectory(dataType, path, formatResolver)
+        val name = data.meta[Envelope.ENVELOPE_NAME_KEY].string ?: path.nameWithoutExtension
+        node(name, data)
+    }
+}
+
diff --git a/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/FileDataTest.kt b/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/FileDataTest.kt
index a47a648d..a048f3fc 100644
--- a/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/FileDataTest.kt
+++ b/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/FileDataTest.kt
@@ -9,8 +9,11 @@ import space.kscience.dataforge.io.IOFormat
 import space.kscience.dataforge.io.io
 import space.kscience.dataforge.io.readUtf8String
 import space.kscience.dataforge.io.writeUtf8String
+import space.kscience.dataforge.meta.get
 import space.kscience.dataforge.misc.DFExperimental
 import java.nio.file.Files
+import kotlin.reflect.KType
+import kotlin.reflect.typeOf
 import kotlin.test.Test
 import kotlin.test.assertEquals
 
@@ -30,15 +33,13 @@ class FileDataTest {
 
 
     object StringIOFormat : IOFormat<String> {
+        override val type: KType get() = typeOf<String>()
 
         override fun writeObject(output: Output, obj: String) {
             output.writeUtf8String(obj)
         }
 
-        override fun readObject(input: Input): String {
-            return input.readUtf8String()
-        }
-
+        override fun readObject(input: Input): String = input.readUtf8String()
     }
 
     @Test
@@ -50,7 +51,7 @@ class FileDataTest {
                 writeDataDirectory(dir, dataNode, StringIOFormat)
                 println(dir.toUri().toString())
                 val reconstructed = readDataDirectory(dir) { _, _ -> StringIOFormat }
-                assertEquals(dataNode["dir.a"]?.meta, reconstructed["dir.a"]?.meta)
+                assertEquals(dataNode["dir.a"]?.meta?.get("content"), reconstructed["dir.a"]?.meta?.get("content"))
                 assertEquals(dataNode["b"]?.await(), reconstructed["b"]?.await())
             }
         }
@@ -63,10 +64,10 @@ class FileDataTest {
         Global.io.run {
             val zip = Files.createTempFile("df_data_node", ".zip")
             runBlocking {
-                writeZip(zip, dataNode, StringIOFormat)
+                FileData.writeZip(zip, dataNode, StringIOFormat)
                 println(zip.toUri().toString())
                 val reconstructed = readDataDirectory(zip) { _, _ -> StringIOFormat }
-                assertEquals(dataNode["dir.a"]?.meta, reconstructed["dir.a"]?.meta)
+                assertEquals(dataNode["dir.a"]?.meta?.get("content"), reconstructed["dir.a"]?.meta?.get("content"))
                 assertEquals(dataNode["b"]?.await(), reconstructed["b"]?.await())
             }
         }
diff --git a/gradle.properties b/gradle.properties
index 1aeadb9b..93001b21 100644
--- a/gradle.properties
+++ b/gradle.properties
@@ -1,7 +1,8 @@
 org.gradle.parallel=true
+org.gradle.jvmargs=-Xmx4096m
 
 kotlin.code.style=official
-
 kotlin.mpp.stability.nowarn=true
+#kotlin.incremental.js.ir=true
 
 toolsVersion=0.11.5-kotlin-1.6.21

From 48331288578371c544ef22d3f4f04811add0519b Mon Sep 17 00:00:00 2001
From: Alexander Nozik <altavir@gmail.com>
Date: Sat, 21 May 2022 11:08:59 +0300
Subject: [PATCH 18/19] refactor file reading

---
 .../space/kscience/dataforge/data/Data.kt     |   2 +-
 .../space/kscience/dataforge/io/IOFormat.kt   |   4 +-
 .../kscience/dataforge/workspace/fileData.kt  | 128 +++++-------------
 .../kscience/dataforge/workspace/zipData.kt   |  72 ++++++++++
 .../dataforge/workspace/FileDataTest.kt       |   2 +-
 5 files changed, 111 insertions(+), 97 deletions(-)
 create mode 100644 dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/zipData.kt

diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/Data.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/Data.kt
index 41182882..c484927f 100644
--- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/Data.kt
+++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/Data.kt
@@ -15,7 +15,7 @@ import kotlin.reflect.typeOf
  * A data element characterized by its meta
  */
 @Type(Data.TYPE)
-public interface Data<out T : Any> : Goal<T>, MetaRepr {
+public interface Data<out T> : Goal<T>, MetaRepr {
     /**
      * Type marker for the data. The type is known before the calculation takes place so it could be checked.
      */
diff --git a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/IOFormat.kt b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/IOFormat.kt
index 75c53e70..3a0d4eea 100644
--- a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/IOFormat.kt
+++ b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/IOFormat.kt
@@ -27,7 +27,7 @@ public interface IOReader<out T> {
     public fun readObject(input: Input): T
 }
 
-public inline fun <reified T : Any> IOReader(crossinline read: Input.() -> T): IOReader<T> = object : IOReader<T> {
+public inline fun <reified T> IOReader(crossinline read: Input.() -> T): IOReader<T> = object : IOReader<T> {
     override val type: KType = typeOf<T>()
 
     override fun readObject(input: Input): T = input.read()
@@ -41,7 +41,7 @@ public fun interface IOWriter<in T> {
 /**
  * And interface for reading and writing objects into with IO streams
  */
-public interface IOFormat<T : Any> : IOReader<T>, IOWriter<T>
+public interface IOFormat<T> : IOReader<T>, IOWriter<T>
 
 public fun <T : Any> Input.readObject(format: IOReader<T>): T = format.readObject(this@readObject)
 
diff --git a/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/fileData.kt b/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/fileData.kt
index c15972b8..67f1ca12 100644
--- a/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/fileData.kt
+++ b/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/fileData.kt
@@ -1,6 +1,5 @@
 package space.kscience.dataforge.workspace
 
-import io.ktor.utils.io.streams.asOutput
 import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.isActive
 import kotlinx.coroutines.launch
@@ -17,11 +16,13 @@ import space.kscience.dataforge.names.Name
 import space.kscience.dataforge.names.NameToken
 import space.kscience.dataforge.names.asName
 import space.kscience.dataforge.names.plus
-import java.nio.file.*
+import java.nio.file.Files
+import java.nio.file.Path
+import java.nio.file.StandardWatchEventKinds
+import java.nio.file.WatchEvent
 import java.nio.file.attribute.BasicFileAttributes
 import java.nio.file.spi.FileSystemProvider
-import java.util.zip.ZipEntry
-import java.util.zip.ZipOutputStream
+import java.time.Instant
 import kotlin.io.path.extension
 import kotlin.io.path.nameWithoutExtension
 import kotlin.io.path.readAttributes
@@ -34,34 +35,20 @@ import kotlin.streams.toList
 
 public typealias FileFormatResolver<T> = (path: Path, meta: Meta) -> IOReader<T>
 
-public object FileData {
-    public val META_FILE_KEY: Name = "file".asName()
-    public val META_FILE_PATH_KEY: Name = META_FILE_KEY + "path"
-    public val META_FILE_EXTENSION_KEY: Name = META_FILE_KEY + "extension"
-    public val META_FILE_CREATE_TIME_KEY: Name = META_FILE_KEY + "created"
-    public val META_FILE_UPDATE_TIME_KEY: Name = META_FILE_KEY + "update"
-}
+public class FileData<T> internal constructor(private val data: Data<T>) : Data<T> by data {
 
+    public val path: String? get() = meta[META_FILE_PATH_KEY].string
+    public val extension: String? get() = meta[META_FILE_EXTENSION_KEY].string
 
-@DFInternal
-@DFExperimental
-public fun <T : Any> IOPlugin.readDataFile(
-    type: KType,
-    path: Path,
-    formatResolver: FileFormatResolver<T>,
-): Data<T> {
-    val envelope = readEnvelopeFile(path, true)
-    val format = formatResolver(path, envelope.meta)
-    val updatedMeta = envelope.meta.copy {
-        FileData.META_FILE_PATH_KEY put path.toString()
-        FileData.META_FILE_EXTENSION_KEY put path.extension
+    public val createdTime: Instant? get() = meta[META_FILE_CREATE_TIME_KEY].string?.let { Instant.parse(it) }
+    public val updatedTime: Instant? get() = meta[META_FILE_UPDATE_TIME_KEY].string?.let { Instant.parse(it) }
 
-        val attributes = path.readAttributes<BasicFileAttributes>()
-        FileData.META_FILE_UPDATE_TIME_KEY put attributes.lastModifiedTime().toInstant().toString()
-        FileData.META_FILE_CREATE_TIME_KEY put attributes.creationTime().toInstant().toString()
-    }
-    return Data(type, updatedMeta) {
-        envelope.data?.readWith(format) ?: error("Can't convert envelope without content to Data")
+    public companion object {
+        public val META_FILE_KEY: Name = "file".asName()
+        public val META_FILE_PATH_KEY: Name = META_FILE_KEY + "path"
+        public val META_FILE_EXTENSION_KEY: Name = META_FILE_KEY + "extension"
+        public val META_FILE_CREATE_TIME_KEY: Name = META_FILE_KEY + "created"
+        public val META_FILE_UPDATE_TIME_KEY: Name = META_FILE_KEY + "update"
     }
 }
 
@@ -72,10 +59,25 @@ public fun <T : Any> IOPlugin.readDataFile(
  */
 @OptIn(DFInternal::class)
 @DFExperimental
-public inline fun <reified T : Any> IOPlugin.readDataFile(
+public fun <T : Any> IOPlugin.readDataFile(
     path: Path,
-    noinline formatResolver: FileFormatResolver<T>,
-): Data<T> = readDataFile(typeOf<T>(), path, formatResolver)
+    formatResolver: FileFormatResolver<T>,
+): FileData<T> {
+    val envelope = readEnvelopeFile(path, true)
+    val format = formatResolver(path, envelope.meta)
+    val updatedMeta = envelope.meta.copy {
+        FileData.META_FILE_PATH_KEY put path.toString()
+        FileData.META_FILE_EXTENSION_KEY put path.extension
+
+        val attributes = path.readAttributes<BasicFileAttributes>()
+        FileData.META_FILE_UPDATE_TIME_KEY put attributes.lastModifiedTime().toInstant().toString()
+        FileData.META_FILE_CREATE_TIME_KEY put attributes.creationTime().toInstant().toString()
+    }
+    return FileData(Data(format.type, updatedMeta) {
+        envelope.data?.readWith(format) ?: error("Can't convert envelope without content to Data")
+    })
+}
+
 
 context(IOPlugin) @DFExperimental
 private fun <T : Any> DataSetBuilder<T>.directory(path: Path, formatResolver: FileFormatResolver<T>) {
@@ -122,7 +124,6 @@ public inline fun <reified T : Any> IOPlugin.readDataDirectory(
 ): DataTree<Any> = readDataDirectory(typeOf<T>(), path, formatResolver)
 
 
-
 @OptIn(DFExperimental::class)
 private fun Path.toName() = Name(map { NameToken.parse(it.nameWithoutExtension) })
 
@@ -219,65 +220,6 @@ public suspend fun <T : Any> IOPlugin.writeDataDirectory(
     }
 }
 
-
-@Suppress("BlockingMethodInNonBlockingContext")
-private suspend fun <T : Any> ZipOutputStream.writeNode(
-    name: String,
-    treeItem: DataTreeItem<T>,
-    dataFormat: IOFormat<T>,
-    envelopeFormat: EnvelopeFormat = TaggedEnvelopeFormat,
-) {
-    withContext(Dispatchers.IO) {
-        when (treeItem) {
-            is DataTreeItem.Leaf -> {
-                //TODO add directory-based envelope writer
-                val envelope = treeItem.data.toEnvelope(dataFormat)
-                val entry = ZipEntry(name)
-                putNextEntry(entry)
-                asOutput().run {
-                    envelopeFormat.writeEnvelope(this, envelope)
-                    flush()
-                }
-            }
-            is DataTreeItem.Node -> {
-                val entry = ZipEntry("$name/")
-                putNextEntry(entry)
-                closeEntry()
-                treeItem.tree.items.forEach { (token, item) ->
-                    val childName = "$name/$token"
-                    writeNode(childName, item, dataFormat, envelopeFormat)
-                }
-            }
-        }
-    }
-}
-
-@DFExperimental
-public suspend fun <T : Any> FileData.writeZip(
-    path: Path,
-    tree: DataTree<T>,
-    format: IOFormat<T>,
-    envelopeFormat: EnvelopeFormat = TaggedEnvelopeFormat,
-) {
-    withContext(Dispatchers.IO) {
-        val actualFile = if (path.toString().endsWith(".zip")) {
-            path
-        } else {
-            path.resolveSibling(path.fileName.toString() + ".zip")
-        }
-        val fos = Files.newOutputStream(
-            actualFile,
-            StandardOpenOption.WRITE,
-            StandardOpenOption.CREATE,
-            StandardOpenOption.TRUNCATE_EXISTING
-        )
-        val zos = ZipOutputStream(fos)
-        zos.use {
-            it.writeNode("", DataTreeItem.Node(tree), format, envelopeFormat)
-        }
-    }
-}
-
 /**
  * Add file/directory-based data tree item
  */
@@ -289,7 +231,7 @@ public fun <T : Any> DataSetBuilder<T>.file(
 ) {
     //If path is a single file or a special directory, read it as single datum
     if (!Files.isDirectory(path) || Files.list(path).allMatch { it.fileName.toString().startsWith("@") }) {
-        val data = readDataFile(dataType, path, formatResolver)
+        val data = readDataFile(path, formatResolver)
         val name = data.meta[Envelope.ENVELOPE_NAME_KEY].string ?: path.nameWithoutExtension
         data(name, data)
     } else {
diff --git a/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/zipData.kt b/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/zipData.kt
new file mode 100644
index 00000000..b596d1cb
--- /dev/null
+++ b/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/zipData.kt
@@ -0,0 +1,72 @@
+package space.kscience.dataforge.workspace
+
+import io.ktor.utils.io.streams.asOutput
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.withContext
+import space.kscience.dataforge.data.DataTree
+import space.kscience.dataforge.data.DataTreeItem
+import space.kscience.dataforge.io.EnvelopeFormat
+import space.kscience.dataforge.io.IOFormat
+import space.kscience.dataforge.io.TaggedEnvelopeFormat
+import space.kscience.dataforge.misc.DFExperimental
+import java.nio.file.Files
+import java.nio.file.Path
+import java.nio.file.StandardOpenOption
+import java.util.zip.ZipEntry
+import java.util.zip.ZipOutputStream
+
+
+private suspend fun <T : Any> ZipOutputStream.writeNode(
+    name: String,
+    treeItem: DataTreeItem<T>,
+    dataFormat: IOFormat<T>,
+    envelopeFormat: EnvelopeFormat = TaggedEnvelopeFormat,
+): Unit = withContext(Dispatchers.IO) {
+    when (treeItem) {
+        is DataTreeItem.Leaf -> {
+            //TODO add directory-based envelope writer
+            val envelope = treeItem.data.toEnvelope(dataFormat)
+            val entry = ZipEntry(name)
+            putNextEntry(entry)
+            asOutput().run {
+                envelopeFormat.writeEnvelope(this, envelope)
+                flush()
+            }
+        }
+        is DataTreeItem.Node -> {
+            val entry = ZipEntry("$name/")
+            putNextEntry(entry)
+            closeEntry()
+            treeItem.tree.items.forEach { (token, item) ->
+                val childName = "$name/$token"
+                writeNode(childName, item, dataFormat, envelopeFormat)
+            }
+        }
+    }
+}
+
+/**
+ * Write this [DataTree] as a zip archive
+ */
+@DFExperimental
+public suspend fun <T : Any> DataTree<T>.writeZip(
+    path: Path,
+    format: IOFormat<T>,
+    envelopeFormat: EnvelopeFormat = TaggedEnvelopeFormat,
+): Unit = withContext(Dispatchers.IO) {
+    val actualFile = if (path.toString().endsWith(".zip")) {
+        path
+    } else {
+        path.resolveSibling(path.fileName.toString() + ".zip")
+    }
+    val fos = Files.newOutputStream(
+        actualFile,
+        StandardOpenOption.WRITE,
+        StandardOpenOption.CREATE,
+        StandardOpenOption.TRUNCATE_EXISTING
+    )
+    val zos = ZipOutputStream(fos)
+    zos.use {
+        it.writeNode("", DataTreeItem.Node(this@writeZip), format, envelopeFormat)
+    }
+}
\ No newline at end of file
diff --git a/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/FileDataTest.kt b/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/FileDataTest.kt
index a048f3fc..6bfde195 100644
--- a/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/FileDataTest.kt
+++ b/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/FileDataTest.kt
@@ -64,7 +64,7 @@ class FileDataTest {
         Global.io.run {
             val zip = Files.createTempFile("df_data_node", ".zip")
             runBlocking {
-                FileData.writeZip(zip, dataNode, StringIOFormat)
+                dataNode.writeZip(zip, StringIOFormat)
                 println(zip.toUri().toString())
                 val reconstructed = readDataDirectory(zip) { _, _ -> StringIOFormat }
                 assertEquals(dataNode["dir.a"]?.meta?.get("content"), reconstructed["dir.a"]?.meta?.get("content"))

From b8869570cea07412ecf391906813f8bb48509ad4 Mon Sep 17 00:00:00 2001
From: Alexander Nozik <altavir@gmail.com>
Date: Wed, 25 May 2022 19:00:12 +0300
Subject: [PATCH 19/19] refactor file reading

---
 .../space/kscience/dataforge/io/IOFormat.kt   |  3 +-
 .../kscience/dataforge/workspace/fileData.kt  | 29 ++++++++++++-------
 2 files changed, 19 insertions(+), 13 deletions(-)

diff --git a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/IOFormat.kt b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/IOFormat.kt
index 3a0d4eea..f4735878 100644
--- a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/IOFormat.kt
+++ b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/IOFormat.kt
@@ -37,7 +37,6 @@ public fun interface IOWriter<in T> {
     public fun writeObject(output: Output, obj: T)
 }
 
-
 /**
  * And interface for reading and writing objects into with IO streams
  */
@@ -83,7 +82,7 @@ public object DoubleIOFormat : IOFormat<Double>, IOFormatFactory<Double> {
 
     override val type: KType get() = typeOf<Double>()
 
-    override fun writeObject(output: Output, obj: kotlin.Double) {
+    override fun writeObject(output: Output, obj: Double) {
         output.writeDouble(obj)
     }
 
diff --git a/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/fileData.kt b/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/fileData.kt
index 67f1ca12..21151113 100644
--- a/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/fileData.kt
+++ b/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/fileData.kt
@@ -4,6 +4,8 @@ import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.isActive
 import kotlinx.coroutines.launch
 import kotlinx.coroutines.withContext
+import space.kscience.dataforge.context.error
+import space.kscience.dataforge.context.logger
 import space.kscience.dataforge.data.*
 import space.kscience.dataforge.io.*
 import space.kscience.dataforge.meta.Meta
@@ -48,7 +50,7 @@ public class FileData<T> internal constructor(private val data: Data<T>) : Data<
         public val META_FILE_PATH_KEY: Name = META_FILE_KEY + "path"
         public val META_FILE_EXTENSION_KEY: Name = META_FILE_KEY + "extension"
         public val META_FILE_CREATE_TIME_KEY: Name = META_FILE_KEY + "created"
-        public val META_FILE_UPDATE_TIME_KEY: Name = META_FILE_KEY + "update"
+        public val META_FILE_UPDATE_TIME_KEY: Name = META_FILE_KEY + "updated"
     }
 }
 
@@ -229,16 +231,21 @@ public fun <T : Any> DataSetBuilder<T>.file(
     path: Path,
     formatResolver: FileFormatResolver<out T>,
 ) {
-    //If path is a single file or a special directory, read it as single datum
-    if (!Files.isDirectory(path) || Files.list(path).allMatch { it.fileName.toString().startsWith("@") }) {
-        val data = readDataFile(path, formatResolver)
-        val name = data.meta[Envelope.ENVELOPE_NAME_KEY].string ?: path.nameWithoutExtension
-        data(name, data)
-    } else {
-        //otherwise, read as directory
-        val data = readDataDirectory(dataType, path, formatResolver)
-        val name = data.meta[Envelope.ENVELOPE_NAME_KEY].string ?: path.nameWithoutExtension
-        node(name, data)
+    try {
+
+        //If path is a single file or a special directory, read it as single datum
+        if (!Files.isDirectory(path) || Files.list(path).allMatch { it.fileName.toString().startsWith("@") }) {
+            val data = readDataFile(path, formatResolver)
+            val name = data.meta[Envelope.ENVELOPE_NAME_KEY].string ?: path.nameWithoutExtension
+            data(name, data)
+        } else {
+            //otherwise, read as directory
+            val data = readDataDirectory(dataType, path, formatResolver)
+            val name = data.meta[Envelope.ENVELOPE_NAME_KEY].string ?: path.nameWithoutExtension
+            node(name, data)
+        }
+    } catch (ex: Exception) {
+        logger.error { "Failed to read file or directory at $path: ${ex.message}" }
     }
 }