Merge pull request #23 from mipt-npm/dev

0.1.4

build.gradle.kts

@@ -1,9 +1,10 @@
 plugins {
-    id("scientifik.mpp") version "0.1.4" apply false
+    id("scientifik.mpp") version "0.2.1" apply false
-    id("scientifik.publish") version "0.1.4" apply false
+    id("scientifik.jvm") version "0.2.1" apply false
+    id("scientifik.publish") version "0.2.1" apply false
 }
 
-val dataforgeVersion by extra("0.1.3")
+val dataforgeVersion by extra("0.1.4")
 
 val bintrayRepo by extra("dataforge")
 val githubProject by extra("dataforge-core")

dataforge-context/src/commonMain/kotlin/hep/dataforge/context/AbstractPlugin.kt

@@ -3,10 +3,13 @@ package hep.dataforge.context
 import hep.dataforge.meta.EmptyMeta
 import hep.dataforge.meta.Meta
 import hep.dataforge.names.Name
-import hep.dataforge.names.toName
+import kotlin.properties.ReadOnlyProperty
+import kotlin.reflect.KClass
+import kotlin.reflect.KProperty
 
 abstract class AbstractPlugin(override val meta: Meta = EmptyMeta) : Plugin {
     private var _context: Context? = null
+    private val dependencies = ArrayList<PluginFactory<*>>()
 
     override val context: Context
         get() = _context ?: error("Plugin $tag is not attached")
@@ -19,9 +22,23 @@ abstract class AbstractPlugin(override val meta: Meta = EmptyMeta) : Plugin {
         this._context = null
     }
 
-    override fun provideTop(target: String): Map<Name, Any>  = emptyMap()
+    final override fun dependsOn(): List<PluginFactory<*>> = dependencies
 
-    companion object{
+    /**
-        fun <T: Named> Collection<T>.toMap(): Map<Name, T> = associate { it.name.toName() to it }
+     * Register plugin dependency and return a delegate which provides lazily initialized reference to dependent plugin
+     */
+    protected fun <P : Plugin> require(factory: PluginFactory<P>): ReadOnlyProperty<AbstractPlugin, P> {
+        dependencies.add(factory)
+        return PluginDependencyDelegate(factory.type)
+    }
+
+    override fun provideTop(target: String): Map<Name, Any> = emptyMap()
+}
+
+fun <T : Named> Collection<T>.toMap(): Map<Name, T> = associate { it.name to it }
+
+private class PluginDependencyDelegate<P : Plugin>(val type: KClass<out P>) : ReadOnlyProperty<AbstractPlugin, P> {
+    override fun getValue(thisRef: AbstractPlugin, property: KProperty<*>): P {
+        return thisRef.context.plugins[type] ?: error("Plugin with type $type not found")
     }
 }

dataforge-context/src/commonMain/kotlin/hep/dataforge/context/Context.kt

@@ -2,8 +2,8 @@ package hep.dataforge.context
 
 import hep.dataforge.meta.*
 import hep.dataforge.names.Name
-import hep.dataforge.names.appendLeft
+import hep.dataforge.names.asName
-import hep.dataforge.names.toName
+import hep.dataforge.names.plus
 import hep.dataforge.provider.Provider
 import hep.dataforge.provider.top
 import hep.dataforge.values.Value
@@ -27,7 +27,7 @@ import kotlin.jvm.JvmName
  * @author Alexander Nozik
  */
 open class Context(
-    final override val name: String,
+    final override val name: Name,
     val parent: Context? = Global
 ) : Named, MetaRepr, Provider, CoroutineScope {
 
@@ -45,7 +45,7 @@ open class Context(
     /**
      * Context logger
      */
-    val logger: KLogger = KotlinLogging.logger(name)
+    val logger: KLogger = KotlinLogging.logger(name.toString())
 
     /**
      * A [PluginManager] for current context
@@ -64,7 +64,7 @@ open class Context(
     override fun provideTop(target: String): Map<Name, Any> {
         return when (target) {
             Value.TYPE -> properties.sequence().toMap()
-            Plugin.PLUGIN_TARGET -> plugins.sequence(true).associateBy { it.name.toName() }
+            Plugin.PLUGIN_TARGET -> plugins.sequence(true).associateBy { it.name }
             else -> emptyMap()
         }
     }
@@ -105,8 +105,8 @@ open class Context(
 
     override fun toMeta(): Meta = buildMeta {
         "parent" to parent?.name
-        "properties" to properties.seal()
+        "properties" put properties.seal()
-        "plugins" to plugins.map { it.toMeta() }
+        "plugins" put plugins.map { it.toMeta() }
     }
 }
 
@@ -118,14 +118,14 @@ fun Context.content(target: String): Map<Name, Any> = content<Any>(target)
 @JvmName("typedContent")
 inline fun <reified T : Any> Context.content(target: String): Map<Name, T> =
     plugins.flatMap { plugin ->
-        plugin.top<T>(target).entries.map { (it.key.appendLeft(plugin.name)) to it.value }
+        plugin.top<T>(target).entries.map { (plugin.name + it.key) to it.value }
     }.associate { it }
 
 
 /**
  * A global root context. Closing [Global] terminates the framework.
  */
-object Global : Context("GLOBAL", null) {
+object Global : Context("GLOBAL".asName(), null) {
     /**
      * Closing all contexts
      *
@@ -173,7 +173,7 @@ interface ContextAware {
 
     val logger: KLogger
         get() = if (this is Named) {
-            KotlinLogging.logger(context.name + "." + (this as Named).name)
+            KotlinLogging.logger((context.name + this.name).toString())
         } else {
             context.logger
         }

dataforge-context/src/commonMain/kotlin/hep/dataforge/context/ContextBuilder.kt

@@ -2,11 +2,12 @@ package hep.dataforge.context
 
 import hep.dataforge.meta.MetaBuilder
 import hep.dataforge.meta.buildMeta
+import hep.dataforge.names.toName
 
 /**
  * A convenience builder for context
  */
-class ContextBuilder(var name: String = "@anonimous", val parent: Context = Global) {
+class ContextBuilder(var name: String = "@anonymous", val parent: Context = Global) {
     private val plugins = ArrayList<Plugin>()
     private var meta = MetaBuilder()
 
@@ -31,7 +32,7 @@ class ContextBuilder(var name: String = "@anonimous", val parent: Context = Glob
     }
 
     fun build(): Context {
-        return Context(name, parent).apply {
+        return Context(name.toName(), parent).apply {
             this@ContextBuilder.plugins.forEach {
                 plugins.load(it)
             }

dataforge-context/src/commonMain/kotlin/hep/dataforge/context/Factory.kt

@@ -0,0 +1,8 @@
+package hep.dataforge.context
+
+import hep.dataforge.meta.EmptyMeta
+import hep.dataforge.meta.Meta
+
+interface Factory<out T : Any> {
+    operator fun invoke(meta: Meta = EmptyMeta, context: Context = Global): T
+}

dataforge-context/src/commonMain/kotlin/hep/dataforge/context/Named.kt

@@ -15,6 +15,10 @@
  */
 package hep.dataforge.context
 
+import hep.dataforge.names.Name
+import hep.dataforge.names.asName
+import hep.dataforge.names.isEmpty
+
 /**
  * Any object that have name
  *
@@ -27,10 +31,9 @@ interface Named {
      *
      * @return
      */
-    val name: String
+    val name: Name
 
     companion object {
-        const val ANONYMOUS = ""
 
         /**
          * Get the name of given object. If object is Named its name is used,
@@ -39,11 +42,11 @@ interface Named {
          * @param obj
          * @return
          */
-        fun nameOf(obj: Any): String {
+        fun nameOf(obj: Any): Name {
             return if (obj is Named) {
                 obj.name
             } else {
-                obj.toString()
+                obj.toString().asName()
             }
         }
     }
@@ -54,4 +57,4 @@ interface Named {
  * @return
  */
 val Named.isAnonymous: Boolean
-    get() = this.name == Named.ANONYMOUS
+    get() = this.name.isEmpty()

dataforge-context/src/commonMain/kotlin/hep/dataforge/context/Plugin.kt

@@ -3,6 +3,8 @@ package hep.dataforge.context
 import hep.dataforge.meta.Meta
 import hep.dataforge.meta.MetaRepr
 import hep.dataforge.meta.buildMeta
+import hep.dataforge.names.Name
+import hep.dataforge.names.toName
 import hep.dataforge.provider.Provider
 
 /**
@@ -37,7 +39,7 @@ interface Plugin : Named, ContextAware, Provider, MetaRepr {
      *
      * @return
      */
-    override val name: String get() = tag.name
+    override val name: Name get() = tag.name.toName()
 
     /**
      * Plugin dependencies which are required to attach this plugin. Plugin
@@ -46,7 +48,7 @@ interface Plugin : Named, ContextAware, Provider, MetaRepr {
      *
      * @return
      */
-    fun dependsOn(): List<PluginFactory<*>> = emptyList()
+    fun dependsOn(): Collection<PluginFactory<*>>
 
     /**
      * Start this plugin and attach registration info to the context. This method
@@ -64,10 +66,10 @@ interface Plugin : Named, ContextAware, Provider, MetaRepr {
     fun detach()
 
     override fun toMeta(): Meta = buildMeta {
-        "context" to context.name
+        "context" put context.name.toString()
         "type" to this::class.simpleName
-        "tag" to tag
+        "tag" put tag
-        "meta" to meta
+        "meta" put meta
     }
 
     companion object {

dataforge-context/src/commonMain/kotlin/hep/dataforge/context/PluginManager.kt

@@ -38,7 +38,7 @@ class PluginManager(override val context: Context) : ContextAware, Iterable<Plug
      * @param recursive search for parent [PluginManager] plugins
      * @param predicate condition for the plugin
      */
-    fun get(recursive: Boolean = true, predicate: (Plugin) -> Boolean): Plugin? = sequence(recursive).find(predicate)
+    fun find(recursive: Boolean = true, predicate: (Plugin) -> Boolean): Plugin? = sequence(recursive).find(predicate)
 
 
     /**
@@ -47,7 +47,7 @@ class PluginManager(override val context: Context) : ContextAware, Iterable<Plug
      * @param tag
      * @return
      */
-    operator fun get(tag: PluginTag, recursive: Boolean = true): Plugin? = get(recursive) { tag.matches(it.tag) }
+    operator fun get(tag: PluginTag, recursive: Boolean = true): Plugin? = find(recursive) { tag.matches(it.tag) }
 
 
     /**
@@ -63,11 +63,13 @@ class PluginManager(override val context: Context) : ContextAware, Iterable<Plug
      */
     @Suppress("UNCHECKED_CAST")
     operator fun <T : Any> get(type: KClass<T>, tag: PluginTag? = null, recursive: Boolean = true): T? =
-        get(recursive) { type.isInstance(it) && (tag == null || tag.matches(it.tag)) } as T?
+        find(recursive) { type.isInstance(it) && (tag == null || tag.matches(it.tag)) } as T?
 
-    inline fun <reified T : Any> get(tag: PluginTag? = null, recursive: Boolean = true): T? =
+    inline operator fun <reified T : Any> get(tag: PluginTag? = null, recursive: Boolean = true): T? =
         get(T::class, tag, recursive)
 
+    inline operator fun <reified T : Plugin> get(factory: PluginFactory<T>, recursive: Boolean = true): T? =
+        get(factory.type, factory.tag, recursive)
 
     /**
      * Load given plugin into this manager and return loaded instance.
@@ -97,7 +99,7 @@ class PluginManager(override val context: Context) : ContextAware, Iterable<Plug
      * Load a plugin using its factory
      */
     fun <T : Plugin> load(factory: PluginFactory<T>, meta: Meta = EmptyMeta): T =
-        load(factory(meta))
+        load(factory(meta, context))
 
     fun <T : Plugin> load(factory: PluginFactory<T>, metaBuilder: MetaBuilder.() -> Unit): T =
         load(factory, buildMeta(metaBuilder))
@@ -122,7 +124,7 @@ class PluginManager(override val context: Context) : ContextAware, Iterable<Plug
     fun <T : Plugin> fetch(factory: PluginFactory<T>, recursive: Boolean = true, meta: Meta = EmptyMeta): T {
         val loaded = get(factory.type, factory.tag, recursive)
         return when {
-            loaded == null -> load(factory(meta))
+            loaded == null -> load(factory(meta, context))
             loaded.meta == meta -> loaded // if meta is the same, return existing plugin
             else -> throw RuntimeException("Can't load plugin with tag ${factory.tag}. Plugin with this tag and different configuration already exists in context.")
         }

dataforge-context/src/commonMain/kotlin/hep/dataforge/context/PluginRepository.kt

@@ -4,10 +4,9 @@ import hep.dataforge.meta.EmptyMeta
 import hep.dataforge.meta.Meta
 import kotlin.reflect.KClass
 
-interface PluginFactory<T : Plugin> {
+interface PluginFactory<T : Plugin> : Factory<T> {
     val tag: PluginTag
     val type: KClass<out T>
-    operator fun invoke(meta: Meta = EmptyMeta): T
 }
 
 expect object PluginRepository {
@@ -25,25 +24,26 @@ expect object PluginRepository {
  * Fetch specific plugin and instantiate it with given meta
  */
 fun PluginRepository.fetch(tag: PluginTag, meta: Meta = EmptyMeta): Plugin =
-    list().find { it.tag.matches(tag) }?.invoke(meta) ?: error("Plugin with tag $tag not found in the repository")
+    list().find { it.tag.matches(tag) }?.invoke(meta = meta)
+        ?: error("Plugin with tag $tag not found in the repository")
 
 fun <T : Plugin> PluginRepository.register(
     tag: PluginTag,
     type: KClass<out T>,
-    constructor: (Meta) -> T
+    constructor: (Context, Meta) -> T
 ): PluginFactory<T> {
     val factory = object : PluginFactory<T> {
         override val tag: PluginTag = tag
         override val type: KClass<out T> = type
 
-        override fun invoke(meta: Meta): T = constructor(meta)
+        override fun invoke(meta: Meta, context: Context): T = constructor(context, meta)
 
     }
     register(factory)
     return factory
 }
 
-inline fun <reified T : Plugin> PluginRepository.register(tag: PluginTag, noinline constructor: (Meta) -> T) =
+inline fun <reified T : Plugin> PluginRepository.register(tag: PluginTag, noinline constructor: (Context, Meta) -> T) =
     register(tag, T::class, constructor)
 
-fun PluginRepository.register(plugin: Plugin) = register(plugin.tag, plugin::class) { plugin }
+fun PluginRepository.register(plugin: Plugin) = register(plugin.tag, plugin::class) { _, _ -> plugin }

dataforge-context/src/commonMain/kotlin/hep/dataforge/context/PluginTag.kt

@@ -37,9 +37,9 @@ data class PluginTag(
     override fun toString(): String = listOf(group, name, version).joinToString(separator = ":")
 
     override fun toMeta(): Meta = buildMeta {
-        "name" to name
+        "name" put name
-        "group" to group
+        "group" put group
-        "version" to version
+        "version" put version
     }
 
     companion object {

dataforge-context/src/commonTest/kotlin/hep/dataforge/context/ContextTest.kt

@@ -5,7 +5,6 @@ import hep.dataforge.names.appendLeft
 import hep.dataforge.names.toName
 import kotlin.test.Test
 import kotlin.test.assertEquals
-import kotlin.test.assertTrue
 
 
 class ContextTest {
@@ -26,7 +25,7 @@ class ContextTest {
         val members = Global.content<Name>("test")
         assertEquals(3, members.count())
         members.forEach {
-            assertTrue{it.key == it.value.appendLeft("test")}
+            assertEquals(it.key, it.value.appendLeft("test"))
         }
     }
 

dataforge-data/src/commonMain/kotlin/hep/dataforge/data/Data.kt

@@ -1,8 +1,6 @@
 package hep.dataforge.data
 
-import hep.dataforge.meta.EmptyMeta
+import hep.dataforge.meta.*
-import hep.dataforge.meta.Meta
-import hep.dataforge.meta.MetaRepr
 import kotlinx.coroutines.CoroutineScope
 import kotlin.coroutines.CoroutineContext
 import kotlin.coroutines.EmptyCoroutineContext
@@ -11,7 +9,7 @@ import kotlin.reflect.KClass
 /**
  * A data element characterized by its meta
  */
-interface Data<out T : Any> : Goal<T>, MetaRepr {
+interface Data<out T : Any> : Goal<T>, MetaRepr{
     /**
      * Type marker for the data. The type is known before the calculation takes place so it could be checked.
      */
@@ -21,7 +19,12 @@ interface Data<out T : Any> : Goal<T>, MetaRepr {
      */
     val meta: Meta
 
-    override fun toMeta(): Meta = meta
+    override fun toMeta(): Meta  = buildMeta {
+        "type" put (type.simpleName?:"undefined")
+        if(!meta.isEmpty()) {
+            "meta" put meta
+        }
+    }
 
     companion object {
         const val TYPE = "data"
@@ -34,7 +37,7 @@ interface Data<out T : Any> : Goal<T>, MetaRepr {
             block: suspend CoroutineScope.() -> T
         ): Data<T> = DynamicData(type, meta, context, dependencies, block)
 
-        operator inline fun <reified T : Any> invoke(
+        inline operator fun <reified T : Any> invoke(
             meta: Meta = EmptyMeta,
             context: CoroutineContext = EmptyCoroutineContext,
             dependencies: Collection<Data<*>> = emptyList(),
@@ -50,7 +53,7 @@ interface Data<out T : Any> : Goal<T>, MetaRepr {
             block: suspend CoroutineScope.() -> T
         ): Data<T> = NamedData(name, invoke(type, meta, context, dependencies, block))
 
-        operator inline fun <reified T : Any> invoke(
+        inline operator fun <reified T : Any> invoke(
             name: String,
             meta: Meta = EmptyMeta,
             context: CoroutineContext = EmptyCoroutineContext,
@@ -65,18 +68,6 @@ interface Data<out T : Any> : Goal<T>, MetaRepr {
 }
 
 
-fun <R : Any, T : R> Data<T>.cast(type: KClass<R>): Data<R> {
-    return object : Data<R> by this {
-        override val type: KClass<out R> = type
-    }
-}
-
-/**
- * Upcast a [Data] to a supertype
- */
-inline fun <reified R : Any, T : R> Data<T>.cast(): Data<R> = cast(R::class)
-
-
 class DynamicData<T : Any>(
     override val type: KClass<out T>,
     override val meta: Meta = EmptyMeta,
@@ -94,7 +85,7 @@ class StaticData<T : Any>(
 
 class NamedData<out T : Any>(val name: String, data: Data<T>) : Data<T> by data
 
-fun <T : Any, R : Any> Data<T>.pipe(
+fun <T : Any, R : Any> Data<T>.map(
     outputType: KClass<out R>,
     coroutineContext: CoroutineContext = EmptyCoroutineContext,
     meta: Meta = this.meta,
@@ -107,7 +98,7 @@ fun <T : Any, R : Any> Data<T>.pipe(
 /**
  * Create a data pipe
  */
-inline fun <T : Any, reified R : Any> Data<T>.pipe(
+inline fun <T : Any, reified R : Any> Data<T>.map(
     coroutineContext: CoroutineContext = EmptyCoroutineContext,
     meta: Meta = this.meta,
     noinline block: suspend CoroutineScope.(T) -> R
@@ -118,7 +109,7 @@ inline fun <T : Any, reified R : Any> Data<T>.pipe(
 /**
  * Create a joined data.
  */
-inline fun <T : Any, reified R : Any> Collection<Data<T>>.join(
+inline fun <T : Any, reified R : Any> Collection<Data<T>>.reduce(
     coroutineContext: CoroutineContext = EmptyCoroutineContext,
     meta: Meta,
     noinline block: suspend CoroutineScope.(Collection<T>) -> R
@@ -128,10 +119,10 @@ inline fun <T : Any, reified R : Any> Collection<Data<T>>.join(
     coroutineContext,
     this
 ) {
-    block(map { this.run { it.await(this) } })
+    block(map { run { it.await(this) } })
 }
 
-fun <K, T : Any, R : Any> Map<K, Data<T>>.join(
+fun <K, T : Any, R : Any> Map<K, Data<T>>.reduce(
     outputType: KClass<out R>,
     coroutineContext: CoroutineContext = EmptyCoroutineContext,
     meta: Meta,
@@ -152,7 +143,7 @@ fun <K, T : Any, R : Any> Map<K, Data<T>>.join(
  * @param T type of the input goal
  * @param R type of the result goal
  */
-inline fun <K, T : Any, reified R : Any> Map<K, Data<T>>.join(
+inline fun <K, T : Any, reified R : Any> Map<K, Data<T>>.reduce(
     coroutineContext: CoroutineContext = EmptyCoroutineContext,
     meta: Meta,
     noinline block: suspend CoroutineScope.(Map<K, T>) -> R

dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataFilter.kt

@@ -5,12 +5,23 @@ import hep.dataforge.names.toName
 
 
 class DataFilter(override val config: Config) : Specific {
+    /**
+     * A source node for the filter
+     */
     var from by string()
+    /**
+     * A target placement for the filtered node
+     */
     var to by string()
-    var pattern by string("*.")
+    /**
+     * A regular expression pattern for the filter
+     */
+    var pattern by string(".*")
 //    val prefix by string()
 //    val suffix by string()
 
+    fun isEmpty(): Boolean = config.isEmpty()
+
     companion object : Specification<DataFilter> {
         override fun wrap(config: Config): DataFilter = DataFilter(config)
     }

dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataNode.kt

@@ -1,5 +1,6 @@
 package hep.dataforge.data
 
+import hep.dataforge.meta.*
 import hep.dataforge.names.*
 import kotlinx.coroutines.CoroutineScope
 import kotlinx.coroutines.Job
@@ -9,22 +10,26 @@ import kotlin.collections.component2
 import kotlin.collections.set
 import kotlin.reflect.KClass
 
-sealed class DataItem<out T : Any> {
+sealed class DataItem<out T : Any> : MetaRepr {
     abstract val type: KClass<out T>
 
     class Node<out T : Any>(val value: DataNode<T>) : DataItem<T>() {
         override val type: KClass<out T> get() = value.type
+
+        override fun toMeta(): Meta = value.toMeta()
     }
 
     class Leaf<out T : Any>(val value: Data<T>) : DataItem<T>() {
         override val type: KClass<out T> get() = value.type
+
+        override fun toMeta(): Meta = value.toMeta()
     }
 }
 
 /**
  * A tree-like data structure grouped into the node. All data inside the node must inherit its type
  */
-interface DataNode<out T : Any> {
+interface DataNode<out T : Any> : MetaRepr {
 
     /**
      * The minimal common ancestor to all data in the node
@@ -33,12 +38,24 @@ interface DataNode<out T : Any> {
 
     val items: Map<NameToken, DataItem<T>>
 
+    override fun toMeta(): Meta = buildMeta {
+        "type" put (type.simpleName ?: "undefined")
+        "items" put {
+            this@DataNode.items.forEach {
+                it.key.toString() put it.value.toMeta()
+            }
+        }
+    }
+
     companion object {
         const val TYPE = "dataNode"
 
-        fun <T : Any> build(type: KClass<out T>, block: DataTreeBuilder<T>.() -> Unit) =
+        operator fun <T : Any> invoke(type: KClass<out T>, block: DataTreeBuilder<T>.() -> Unit) =
             DataTreeBuilder(type).apply(block).build()
 
+        inline operator fun <reified T : Any> invoke(noinline block: DataTreeBuilder<T>.() -> Unit) =
+            DataTreeBuilder(T::class).apply(block).build()
+
         fun <T : Any> builder(type: KClass<out T>) = DataTreeBuilder(type)
     }
 }
@@ -68,10 +85,12 @@ fun DataNode<*>.joinAll(scope: CoroutineScope): Job = scope.launch {
 
 operator fun <T : Any> DataNode<T>.get(name: Name): DataItem<T>? = when (name.length) {
     0 -> error("Empty name")
-    1 -> (items[name.first()] as? DataItem.Leaf)
+    1 -> items[name.first()]
     else -> get(name.first()!!.asName()).node?.get(name.cutFirst())
 }
 
+operator fun <T : Any> DataNode<T>.get(name: String): DataItem<T>? = get(name.toName())
+
 /**
  * Sequence of all children including nodes
  */
@@ -108,7 +127,9 @@ class DataTree<out T : Any> internal constructor(
     override val type: KClass<out T>,
     override val items: Map<NameToken, DataItem<T>>
 ) : DataNode<T> {
-    //TODO add node-level meta?
+    override fun toString(): String {
+        return super.toString()
+    }
 }
 
 private sealed class DataTreeBuilderItem<out T : Any> {
@@ -119,10 +140,11 @@ private sealed class DataTreeBuilderItem<out T : Any> {
 /**
  * A builder for a DataTree.
  */
-class DataTreeBuilder<T : Any>(private val type: KClass<out T>) {
+@DFBuilder
+class DataTreeBuilder<T : Any>(val type: KClass<out T>) {
     private val map = HashMap<NameToken, DataTreeBuilderItem<T>>()
 
-    operator fun set(token: NameToken, node: DataTreeBuilder<T>) {
+    operator fun set(token: NameToken, node: DataTreeBuilder<out T>) {
         if (map.containsKey(token)) error("Tree entry with name $token is not empty")
         map[token] = DataTreeBuilderItem.Node(node)
     }
@@ -134,9 +156,9 @@ class DataTreeBuilder<T : Any>(private val type: KClass<out T>) {
 
     private fun buildNode(token: NameToken): DataTreeBuilder<T> {
         return if (!map.containsKey(token)) {
-            DataTreeBuilder<T>(type).also { map[token] = DataTreeBuilderItem.Node(it) }
+            DataTreeBuilder(type).also { map[token] = DataTreeBuilderItem.Node(it) }
         } else {
-            (map[token] as? DataTreeBuilderItem.Node ?: error("The node with name $token is occupied by leaf")).tree
+            (map[token] as? DataTreeBuilderItem.Node<T> ?: error("The node with name $token is occupied by leaf")).tree
         }
     }
 
@@ -156,7 +178,7 @@ class DataTreeBuilder<T : Any>(private val type: KClass<out T>) {
         }
     }
 
-    operator fun set(name: Name, node: DataTreeBuilder<T>) {
+    operator fun set(name: Name, node: DataTreeBuilder<out T>) {
         when (name.length) {
             0 -> error("Can't add data with empty name")
             1 -> set(name.first()!!, node)
@@ -174,19 +196,19 @@ class DataTreeBuilder<T : Any>(private val type: KClass<out T>) {
     /**
      * Append data to node
      */
-    infix fun String.to(data: Data<T>) = set(toName(), data)
+    infix fun String.put(data: Data<T>) = set(toName(), data)
 
     /**
      * Append node
      */
-    infix fun String.to(node: DataNode<T>) = set(toName(), node)
+    infix fun String.put(node: DataNode<T>) = set(toName(), node)
 
-    infix fun String.to(item: DataItem<T>) = set(toName(), item)
+    infix fun String.put(item: DataItem<T>) = set(toName(), item)
 
     /**
      * Build and append node
      */
-    infix fun String.to(block: DataTreeBuilder<T>.() -> Unit) = set(toName(), DataTreeBuilder<T>(type).apply(block))
+    infix fun String.put(block: DataTreeBuilder<T>.() -> Unit) = set(toName(), DataTreeBuilder(type).apply(block))
 
 
     fun update(node: DataNode<T>) {
@@ -207,6 +229,42 @@ class DataTreeBuilder<T : Any>(private val type: KClass<out T>) {
     }
 }
 
+fun <T : Any> DataTreeBuilder<T>.datum(name: Name, data: Data<T>) {
+    this[name] = data
+}
+
+fun <T : Any> DataTreeBuilder<T>.datum(name: String, data: Data<T>) {
+    this[name.toName()] = data
+}
+
+fun <T : Any> DataTreeBuilder<T>.static(name: Name, data: T, meta: Meta = EmptyMeta) {
+    this[name] = Data.static(data, meta)
+}
+
+fun <T : Any> DataTreeBuilder<T>.static(name: Name, data: T, block: MetaBuilder.() -> Unit = {}) {
+    this[name] = Data.static(data, buildMeta(block))
+}
+
+fun <T : Any> DataTreeBuilder<T>.static(name: String, data: T, block: MetaBuilder.() -> Unit = {}) {
+    this[name.toName()] = Data.static(data, buildMeta(block))
+}
+
+fun <T : Any> DataTreeBuilder<T>.node(name: Name, node: DataNode<T>) {
+    this[name] = node
+}
+
+fun <T : Any> DataTreeBuilder<T>.node(name: String, node: DataNode<T>) {
+    this[name.toName()] = node
+}
+
+inline fun <reified T : Any> DataTreeBuilder<T>.node(name: Name, noinline block: DataTreeBuilder<T>.() -> Unit) {
+    this[name] = DataNode(T::class, block)
+}
+
+inline fun <reified T : Any> DataTreeBuilder<T>.node(name: String, noinline block: DataTreeBuilder<T>.() -> Unit) {
+    this[name.toName()] = DataNode(T::class, block)
+}
+
 /**
  * Generate a mutable builder from this node. Node content is not changed
  */
@@ -214,7 +272,7 @@ fun <T : Any> DataNode<T>.builder(): DataTreeBuilder<T> = DataTreeBuilder(type).
     dataSequence().forEach { (name, data) -> this[name] = data }
 }
 
-fun <T : Any> DataNode<T>.filter(predicate: (Name, Data<T>) -> Boolean): DataNode<T> = DataNode.build(type) {
+fun <T : Any> DataNode<T>.filter(predicate: (Name, Data<T>) -> Boolean): DataNode<T> = DataNode.invoke(type) {
     dataSequence().forEach { (name, data) ->
         if (predicate(name, data)) {
             this[name] = data
@@ -222,9 +280,4 @@ fun <T : Any> DataNode<T>.filter(predicate: (Name, Data<T>) -> Boolean): DataNod
     }
 }
 
-fun <T : Any> DataNode<T>.first(): Data<T>? = dataSequence().first().second
+fun <T : Any> DataNode<T>.first(): Data<T>? = dataSequence().first().second
-
-/**
- * Check that node is compatible with given type meaning that each element could be cast to the type
- */
-expect fun DataNode<*>.checkType(type: KClass<*>)

dataforge-data/src/commonMain/kotlin/hep/dataforge/data/Goal.kt

@@ -85,7 +85,7 @@ open class DynamicGoal<T>(
 /**
  * Create a one-to-one goal based on existing goal
  */
-fun <T, R> Goal<T>.pipe(
+fun <T, R> Goal<T>.map(
     coroutineContext: CoroutineContext = EmptyCoroutineContext,
     block: suspend CoroutineScope.(T) -> R
 ): Goal<R> = DynamicGoal(coroutineContext, listOf(this)) {
@@ -95,11 +95,11 @@ fun <T, R> Goal<T>.pipe(
 /**
  * Create a joining goal.
  */
-fun <T, R> Collection<Goal<T>>.join(
+fun <T, R> Collection<Goal<T>>.reduce(
     coroutineContext: CoroutineContext = EmptyCoroutineContext,
     block: suspend CoroutineScope.(Collection<T>) -> R
 ): Goal<R> = DynamicGoal(coroutineContext, this) {
-    block(map { this.run { it.await(this) } })
+    block(map { run { it.await(this) } })
 }
 
 /**
@@ -108,7 +108,7 @@ fun <T, R> Collection<Goal<T>>.join(
  * @param T type of the input goal
  * @param R type of the result goal
  */
-fun <K, T, R> Map<K, Goal<T>>.join(
+fun <K, T, R> Map<K, Goal<T>>.reduce(
     coroutineContext: CoroutineContext = EmptyCoroutineContext,
     block: suspend CoroutineScope.(Map<K, T>) -> R
 ): Goal<R> = DynamicGoal(coroutineContext, this.values) {

dataforge-data/src/commonMain/kotlin/hep/dataforge/data/MapAction.kt

@@ -0,0 +1,78 @@
+package hep.dataforge.data
+
+import hep.dataforge.meta.Meta
+import hep.dataforge.meta.MetaBuilder
+import hep.dataforge.meta.builder
+import hep.dataforge.meta.seal
+import hep.dataforge.names.Name
+import kotlin.reflect.KClass
+
+/**
+ * Action environment includes data name, data meta and action configuration meta
+ */
+data class ActionEnv(
+    val name: Name,
+    val meta: Meta,
+    val actionMeta: Meta
+)
+
+
+/**
+ * Action environment
+ */
+class MapActionBuilder<T, R>(var name: Name, var meta: MetaBuilder, val actionMeta: Meta) {
+    lateinit var result: suspend ActionEnv.(T) -> R
+
+    /**
+     * Calculate the result of goal
+     */
+    fun result(f: suspend ActionEnv.(T) -> R) {
+        result = f;
+    }
+}
+
+
+class MapAction<T : Any, out R : Any>(
+    val inputType: KClass<out T>,
+    val outputType: KClass<out R>,
+    private val block: MapActionBuilder<T, R>.() -> Unit
+) : Action<T, R> {
+
+    override fun invoke(node: DataNode<T>, meta: Meta): DataNode<R> {
+        node.ensureType(inputType)
+
+        return DataNode.invoke(outputType) {
+            node.dataSequence().forEach { (name, data) ->
+                /*
+                 * Creating a new environment for action using **old** name, old meta and task meta
+                 */
+                val env = ActionEnv(name, data.meta, meta)
+
+                //applying transformation from builder
+                val builder = MapActionBuilder<T, R>(
+                    name,
+                    data.meta.builder(), // using data meta
+                    meta
+                ).apply(block)
+
+                //getting new name
+                val newName = builder.name
+
+                //getting new meta
+                val newMeta = builder.meta.seal()
+
+                val newData = data.map(outputType, meta = newMeta) { builder.result(env, it) }
+                //setting the data node
+                this[newName] = newData
+            }
+        }
+    }
+}
+
+inline fun <reified T : Any, reified R : Any> DataNode<T>.map(
+    meta: Meta,
+    noinline action: MapActionBuilder<in T, out R>.() -> Unit
+): DataNode<R> = MapAction(T::class, R::class, action).invoke(this, meta)
+
+
+

dataforge-data/src/commonMain/kotlin/hep/dataforge/data/PipeAction.kt

@@ -1,60 +0,0 @@
-package hep.dataforge.data
-
-import hep.dataforge.meta.*
-import hep.dataforge.names.Name
-import kotlin.reflect.KClass
-
-class ActionEnv(val name: Name, val meta: Meta)
-
-
-/**
- * Action environment
- */
-class PipeBuilder<T, R>(var name: Name, var meta: MetaBuilder) {
-    lateinit var result: suspend ActionEnv.(T) -> R
-
-    /**
-     * Calculate the result of goal
-     */
-    fun result(f: suspend ActionEnv.(T) -> R) {
-        result = f;
-    }
-}
-
-
-class PipeAction<T : Any, R : Any>(
-    val inputType: KClass<T>,
-    val outputType: KClass<R>,
-    private val block: PipeBuilder<T, R>.() -> Unit
-) : Action<T, R> {
-
-    override fun invoke(node: DataNode<T>, meta: Meta): DataNode<R> {
-        node.checkType(inputType)
-
-        return DataNode.build(outputType) {
-            node.dataSequence().forEach { (name, data) ->
-                //merging data meta with action meta (data meta is primary)
-                val oldMeta = meta.builder().apply { update(data.meta) }
-                // creating environment from old meta and name
-                val env = ActionEnv(name, oldMeta)
-                //applying transformation from builder
-                val builder = PipeBuilder<T, R>(name, oldMeta).apply(block)
-                //getting new name
-                val newName = builder.name
-                //getting new meta
-                val newMeta = builder.meta.seal()
-                val newData = data.pipe(outputType, meta = newMeta) { builder.result(env, it) }
-                //setting the data node
-                this[newName] = newData
-            }
-        }
-    }
-}
-
-inline fun <reified T : Any, reified R : Any> DataNode<T>.pipe(
-    meta: Meta,
-    noinline action: PipeBuilder<T, R>.() -> Unit
-): DataNode<R> = PipeAction(T::class, R::class, action).invoke(this, meta)
-
-
-

dataforge-data/src/commonMain/kotlin/hep/dataforge/data/ReduceAction.kt

@@ -1,9 +1,7 @@
 package hep.dataforge.data
 
-import hep.dataforge.meta.Laminate
 import hep.dataforge.meta.Meta
 import hep.dataforge.meta.MetaBuilder
-import hep.dataforge.meta.builder
 import hep.dataforge.names.Name
 import hep.dataforge.names.toName
 import kotlin.reflect.KClass
@@ -21,7 +19,7 @@ class JoinGroup<T : Any, R : Any>(var name: String, internal val node: DataNode<
 
 }
 
-class JoinGroupBuilder<T : Any, R : Any>(val actionMeta: Meta) {
+class ReduceGroupBuilder<T : Any, R : Any>(val actionMeta: Meta) {
     private val groupRules: MutableList<(DataNode<T>) -> List<JoinGroup<T, R>>> = ArrayList();
 
     /**
@@ -73,26 +71,31 @@ class JoinGroupBuilder<T : Any, R : Any>(val actionMeta: Meta) {
 /**
  * The same rules as for KPipe
  */
-class JoinAction<T : Any, R : Any>(
+class ReduceAction<T : Any, R : Any>(
-    val inputType: KClass<T>,
+    val inputType: KClass<out T>,
-    val outputType: KClass<R>,
+    val outputType: KClass<out R>,
-    private val action: JoinGroupBuilder<T, R>.() -> Unit
+    private val action: ReduceGroupBuilder<T, R>.() -> Unit
 ) : Action<T, R> {
 
     override fun invoke(node: DataNode<T>, meta: Meta): DataNode<R> {
-        node.checkType(inputType)
+        node.ensureType(inputType)
-        return DataNode.build(outputType) {
+        return DataNode.invoke(outputType) {
-            JoinGroupBuilder<T, R>(meta).apply(action).buildGroups(node).forEach { group ->
+            ReduceGroupBuilder<T, R>(meta).apply(action).buildGroups(node).forEach { group ->
 
-                val laminate = Laminate(group.meta, meta)
+                //val laminate = Laminate(group.meta, meta)
 
                 val dataMap = group.node.dataSequence().associate { it }
 
-                val groupName: String = group.name;
+                val groupName: String = group.name
 
-                val env = ActionEnv(groupName.toName(), laminate.builder())
+                val groupMeta = group.meta
 
-                val res: DynamicData<R> = dataMap.join(outputType, meta = laminate) { group.result.invoke(env, it) }
+                val env = ActionEnv(groupName.toName(), groupMeta, meta)
+
+                val res: DynamicData<R> = dataMap.reduce(
+                    outputType,
+                    meta = groupMeta
+                ) { group.result.invoke(env, it) }
 
                 set(env.name, res)
             }

dataforge-data/src/commonMain/kotlin/hep/dataforge/data/SplitAction.kt

@@ -33,15 +33,15 @@ class SplitBuilder<T : Any, R : Any>(val name: Name, val meta: Meta) {
 }
 
 class SplitAction<T : Any, R : Any>(
-    val inputType: KClass<T>,
+    val inputType: KClass<out T>,
-    val outputType: KClass<R>,
+    val outputType: KClass<out R>,
     private val action: SplitBuilder<T, R>.() -> Unit
 ) : Action<T, R> {
 
     override fun invoke(node: DataNode<T>, meta: Meta): DataNode<R> {
-        node.checkType(inputType)
+        node.ensureType(inputType)
 
-        return DataNode.build(outputType) {
+        return DataNode.invoke(outputType) {
             node.dataSequence().forEach { (name, data) ->
 
                 val laminate = Laminate(data.meta, meta)
@@ -55,7 +55,7 @@ class SplitAction<T : Any, R : Any>(
 
                     rule(env)
 
-                    val res = data.pipe(outputType, meta = env.meta) { env.result(it) }
+                    val res = data.map(outputType, meta = env.meta) { env.result(it) }
                     set(env.name, res)
                 }
             }

dataforge-data/src/commonMain/kotlin/hep/dataforge/data/dataCast.kt

@@ -0,0 +1,72 @@
+package hep.dataforge.data
+
+import hep.dataforge.meta.Meta
+import hep.dataforge.names.NameToken
+import kotlinx.coroutines.CoroutineScope
+import kotlinx.coroutines.Deferred
+import kotlin.reflect.KClass
+
+fun <R : Any, T : R> Data<T>.upcast(type: KClass<out R>): Data<R> {
+    return object : Data<R> by this {
+        override val type: KClass<out R> = type
+    }
+}
+
+/**
+ * Safe upcast a [Data] to a supertype
+ */
+inline fun <reified R : Any, T : R> Data<T>.upcast(): Data<R> = upcast(R::class)
+
+/**
+ * Check if node could be safely cast to given class
+ */
+expect fun <R : Any> DataNode<*>.canCast(type: KClass<out R>): Boolean
+
+/**
+ * Check if data could be safely cast to given class
+ */
+expect fun <R : Any> Data<*>.canCast(type: KClass<out R>): Boolean
+
+fun <R : Any> DataItem<*>.canCast(type: KClass<out R>): Boolean = when (this) {
+    is DataItem.Node -> value.canCast(type)
+    is DataItem.Leaf -> value.canCast(type)
+}
+
+/**
+ * Unsafe cast of data node
+ */
+@Suppress("UNCHECKED_CAST")
+fun <R : Any> Data<*>.cast(type: KClass<out R>): Data<R> {
+    return object : Data<R> {
+        override val meta: Meta get() = this@cast.meta
+        override val dependencies: Collection<Goal<*>> get() = this@cast.dependencies
+        override val result: Deferred<R>? get() = this@cast.result as Deferred<R>
+        override fun startAsync(scope: CoroutineScope): Deferred<R> = this@cast.startAsync(scope) as Deferred<R>
+        override fun reset() = this@cast.reset()
+        override val type: KClass<out R> = type
+    }
+}
+
+inline fun <reified R : Any> Data<*>.cast(): Data<R> = cast(R::class)
+
+@Suppress("UNCHECKED_CAST")
+fun <R : Any> DataNode<*>.cast(type: KClass<out R>): DataNode<R> {
+    return object : DataNode<R> {
+        override val type: KClass<out R> = type
+        override val items: Map<NameToken, DataItem<R>> get() = this@cast.items as Map<NameToken, DataItem<R>>
+    }
+}
+
+inline fun <reified R : Any> DataNode<*>.cast(): DataNode<R> = cast(R::class)
+
+/**
+ * Check that node is compatible with given type meaning that each element could be cast to the type
+ */
+fun <T : Any> DataNode<*>.ensureType(type: KClass<out T>) {
+    if (!canCast(type)) {
+        error("$type expected, but $type received")
+    }
+}
+
+
+//expect fun <T : Any, R : Any> DataNode<T>.cast(type: KClass<out R>): DataNode<R>

dataforge-data/src/commonTest/kotlin/hep/dataforge/data/DataTreeBuilderTest.kt

@@ -0,0 +1,32 @@
+package hep.dataforge.data
+
+import kotlin.test.Test
+import kotlin.test.assertTrue
+
+
+internal class DataTreeBuilderTest{
+    @Test
+    fun testDataUpdate(){
+        val updateData = DataNode<Any>{
+            "update" put {
+                "a" put Data.static("a")
+                "b" put Data.static("b")
+            }
+        }
+
+        val node = DataNode<Any>{
+            node("primary"){
+                static("a","a")
+                static("b","b")
+            }
+            static("root","root")
+            update(updateData)
+        }
+
+        println(node.toMeta())
+
+        assertTrue { node["update.a"] != null }
+        assertTrue { node["primary.a"] != null }
+
+    }
+}

dataforge-data/src/jsMain/kotlin/hep/dataforge/data/checkType.kt

@@ -1,10 +0,0 @@
-package hep.dataforge.data
-
-import kotlin.reflect.KClass
-
-/**
- * Check that node is compatible with given type meaning that each element could be cast to the type
- */
-actual fun DataNode<*>.checkType(type: KClass<*>) {
-    //Not supported in js yet
-}

dataforge-data/src/jsMain/kotlin/hep/dataforge/data/dataJS.kt

@@ -0,0 +1,16 @@
+package hep.dataforge.data
+
+import kotlin.reflect.KClass
+
+/**
+ * Check that node is compatible with given type meaning that each element could be cast to the type
+ */
+actual fun <R : Any> DataNode<*>.canCast(type: KClass<out R>): Boolean {
+    //Not supported in js yet
+    return true
+}
+
+actual fun <R : Any> Data<*>.canCast(type: KClass<out R>): Boolean {
+    //Not supported in js yet
+    return true
+}

dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/CastDataNode.kt

@@ -1,55 +0,0 @@
-package hep.dataforge.data
-
-import hep.dataforge.meta.Meta
-import hep.dataforge.names.NameToken
-import kotlinx.coroutines.CoroutineScope
-import kotlinx.coroutines.Deferred
-import kotlin.reflect.KClass
-import kotlin.reflect.full.isSubclassOf
-
-@Suppress("UNCHECKED_CAST")
-fun <T : Any, R : Any> Data<T>.safeCast(type: KClass<out R>): Data<R>? {
-    return if (this.type.isSubclassOf(type)) {
-        return object : Data<R> {
-            override val meta: Meta get() = this@safeCast.meta
-            override val dependencies: Collection<Goal<*>> get() = this@safeCast.dependencies
-            override val result: Deferred<R>? get() = this@safeCast.result as Deferred<R>
-            override fun startAsync(scope: CoroutineScope): Deferred<R>  = this@safeCast.startAsync(scope)  as Deferred<R>
-            override fun reset() = this@safeCast.reset()
-            override val type: KClass<out R> = type
-        }
-    } else {
-        null
-    }
-}
-
-/**
- * Filter a node by data and node type. Resulting node and its subnodes is guaranteed to have border type [type],
- * but could contain empty nodes
- */
-fun <T : Any, R : Any> DataNode<T>.cast(type: KClass<out R>): DataNode<R> {
-    return if (this is CastDataNode) {
-        origin.cast(type)
-    } else {
-        CastDataNode(this, type)
-    }
-}
-
-inline fun <T : Any, reified R : Any> DataNode<T>.cast(): DataNode<R> = cast(R::class)
-
-class CastDataNode<out T : Any>(val origin: DataNode<Any>, override val type: KClass<out T>) : DataNode<T> {
-    override val items: Map<NameToken, DataItem<T>>  by lazy {
-        origin.items.mapNotNull { (key, item) ->
-            when (item) {
-                is DataItem.Leaf -> {
-                    (item.value.safeCast(type))?.let {
-                        key to DataItem.Leaf(it)
-                    }
-                }
-                is DataItem.Node -> {
-                    key to DataItem.Node(item.value.cast(type))
-                }
-            }
-        }.associate { it }
-    }
-}

dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/TypeFilteredDataNode.kt

@@ -0,0 +1,25 @@
+package hep.dataforge.data
+
+import hep.dataforge.names.NameToken
+import kotlin.reflect.KClass
+
+
+/**
+ * A zero-copy data node wrapper that returns only children with appropriate type.
+ */
+class TypeFilteredDataNode<out T : Any>(val origin: DataNode<*>, override val type: KClass<out T>) : DataNode<T> {
+    override val items: Map<NameToken, DataItem<T>> by lazy {
+        origin.items.mapNotNull { (key, item) ->
+            when (item) {
+                is DataItem.Leaf -> {
+                    (item.value.filterIsInstance(type))?.let {
+                        key to DataItem.Leaf(it)
+                    }
+                }
+                is DataItem.Node -> {
+                    key to DataItem.Node(item.value.filterIsInstance(type))
+                }
+            }
+        }.associate { it }
+    }
+}

dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/dataJVM.kt

@@ -2,6 +2,7 @@ package hep.dataforge.data
 
 import kotlinx.coroutines.runBlocking
 import kotlin.reflect.KClass
+import kotlin.reflect.full.isSubclassOf
 import kotlin.reflect.full.isSuperclassOf
 
 /**
@@ -12,8 +13,39 @@ fun <T : Any> Data<T>.get(): T = runBlocking { await() }
 /**
  * Check that node is compatible with given type meaning that each element could be cast to the type
  */
-actual fun DataNode<*>.checkType(type: KClass<*>) {
+actual fun <R : Any> DataNode<*>.canCast(type: KClass<out R>): Boolean =
-    if (!type.isSuperclassOf(type)) {
+    type.isSuperclassOf(type)
-        error("$type expected, but $type received")
+
+actual fun <R : Any> Data<*>.canCast(type: KClass<out R>): Boolean =
+    this.type.isSubclassOf(type)
+
+/**
+ * Cast the node to given type if the cast is possible or return null
+ */
+fun <R : Any> Data<*>.filterIsInstance(type: KClass<out R>): Data<R>? =
+    if (canCast(type)) cast(type) else null
+
+/**
+ * Filter a node by data and node type. Resulting node and its subnodes is guaranteed to have border type [type],
+ * but could contain empty nodes
+ */
+fun <R : Any> DataNode<*>.filterIsInstance(type: KClass<out R>): DataNode<R> {
+    return if (canCast(type)) {
+        cast(type)
+    } else if (this is TypeFilteredDataNode) {
+        origin.filterIsInstance(type)
+    } else {
+        TypeFilteredDataNode(this, type)
     }
-}
+}
+
+/**
+ * Filter all elements of given data item that could be cast to given type. If no elements are available, return null.
+ */
+fun <R : Any> DataItem<*>?.filterIsInstance(type: KClass<out R>): DataItem<R>? = when (this) {
+    null -> null
+    is DataItem.Node -> DataItem.Node(this.value.filterIsInstance(type))
+    is DataItem.Leaf -> this.value.filterIsInstance(type)?.let { DataItem.Leaf(it) }
+}
+
+inline fun <reified R : Any> DataItem<*>?.filterIsInstance(): DataItem<R>? = this@filterIsInstance.filterIsInstance(R::class)

dataforge-io/build.gradle.kts

@@ -5,8 +5,8 @@ plugins {
 description = "IO module"
 
 scientifik{
-    serialization = true
+    withSerialization()
-    io = true
+    withIO()
 }
 
 
@@ -17,6 +17,11 @@ kotlin {
                 api(project(":dataforge-context"))
             }
         }
+        jvmMain{
+            dependencies {
+
+            }
+        }
         jsMain{
             dependencies{
                 api(npm("text-encoding"))

dataforge-io/dataforge-io-yaml/build.gradle.kts

@@ -0,0 +1,12 @@
+plugins {
+    id("scientifik.jvm")
+}
+
+description = "YAML meta IO"
+
+dependencies {
+    api(project(":dataforge-io"))
+    api("org.yaml:snakeyaml:1.25")
+    testImplementation(kotlin("test"))
+    testImplementation(kotlin("test-junit"))
+}

dataforge-io/dataforge-io-yaml/src/main/kotlin/hep/dataforge/io/yaml/FrontMatterEnvelopeFormat.kt

@@ -0,0 +1,88 @@
+package hep.dataforge.io.yaml
+
+import hep.dataforge.context.Context
+import hep.dataforge.io.*
+import hep.dataforge.meta.DFExperimental
+import hep.dataforge.meta.EmptyMeta
+import hep.dataforge.meta.Meta
+import kotlinx.io.core.*
+import kotlinx.serialization.toUtf8Bytes
+
+@DFExperimental
+class FrontMatterEnvelopeFormat(
+    val io: IOPlugin,
+    meta: Meta = EmptyMeta
+) : EnvelopeFormat {
+
+    override fun Input.readPartial(): PartialEnvelope {
+        var line: String = ""
+        var offset = 0u
+        do {
+            line = readUTF8Line() ?: error("Input does not contain front matter separator")
+            offset += line.toUtf8Bytes().size.toUInt()
+        } while (!line.startsWith(SEPARATOR))
+
+        val readMetaFormat =
+            metaTypeRegex.matchEntire(line)?.groupValues?.first()
+                ?.let { io.metaFormat(it) } ?: YamlMetaFormat.default
+
+        val metaBlock = buildPacket {
+            do {
+                line = readUTF8Line() ?: error("Input does not contain closing front matter separator")
+                appendln(line)
+                offset += line.toUtf8Bytes().size.toUInt()
+            } while (!line.startsWith(SEPARATOR))
+        }
+        val meta = readMetaFormat.fromBytes(metaBlock)
+        return PartialEnvelope(meta, offset, null)
+    }
+
+    override fun Input.readObject(): Envelope {
+        var line: String = ""
+        do {
+            line = readUTF8Line() ?: error("Input does not contain front matter separator")
+        } while (!line.startsWith(SEPARATOR))
+
+        val readMetaFormat =
+            metaTypeRegex.matchEntire(line)?.groupValues?.first()
+                ?.let { io.metaFormat(it) } ?: YamlMetaFormat.default
+
+        val metaBlock = buildPacket {
+            do {
+                appendln(readUTF8Line() ?: error("Input does not contain closing front matter separator"))
+            } while (!line.startsWith(SEPARATOR))
+        }
+        val meta = readMetaFormat.fromBytes(metaBlock)
+        val bytes = readBytes()
+        val data = bytes.asBinary()
+        return SimpleEnvelope(meta, data)
+    }
+
+    override fun Output.writeEnvelope(envelope: Envelope, metaFormatFactory: MetaFormatFactory, formatMeta: Meta) {
+        val metaFormat = metaFormatFactory(formatMeta, io.context)
+        writeText("$SEPARATOR\r\n")
+        metaFormat.run { writeObject(envelope.meta) }
+        writeText("$SEPARATOR\r\n")
+        envelope.data?.read { copyTo(this@writeEnvelope) }
+    }
+
+    companion object : EnvelopeFormatFactory {
+        const val SEPARATOR = "---"
+
+        private val metaTypeRegex = "---(\\w*)\\s*".toRegex()
+
+        override fun invoke(meta: Meta, context: Context): EnvelopeFormat {
+            return FrontMatterEnvelopeFormat(context.io,  meta)
+        }
+
+        override fun peekFormat(io: IOPlugin, input: Input): EnvelopeFormat? {
+            val line = input.readUTF8Line(3, 30)
+            return if (line != null && line.startsWith("---")) {
+                invoke()
+            } else {
+                null
+            }
+        }
+
+    }
+}

dataforge-io/dataforge-io-yaml/src/main/kotlin/hep/dataforge/io/yaml/YamlMetaFormat.kt

@@ -0,0 +1,56 @@
+package hep.dataforge.io.yaml
+
+import hep.dataforge.context.Context
+import hep.dataforge.descriptors.NodeDescriptor
+import hep.dataforge.io.MetaFormat
+import hep.dataforge.io.MetaFormatFactory
+import hep.dataforge.meta.DFExperimental
+import hep.dataforge.meta.Meta
+import hep.dataforge.meta.toMap
+import hep.dataforge.meta.toMeta
+import hep.dataforge.names.Name
+import hep.dataforge.names.plus
+import kotlinx.io.core.Input
+import kotlinx.io.core.Output
+import kotlinx.io.core.readUByte
+import kotlinx.io.core.writeText
+import org.yaml.snakeyaml.Yaml
+import java.io.InputStream
+
+private class InputAsStream(val input: Input) : InputStream() {
+    override fun read(): Int {
+        if (input.endOfInput) return -1
+        return input.readUByte().toInt()
+    }
+
+    override fun close() {
+        input.close()
+    }
+}
+
+private fun Input.asStream() = InputAsStream(this)
+
+@DFExperimental
+class YamlMetaFormat(val meta: Meta) : MetaFormat {
+    private val yaml = Yaml()
+
+    override fun Output.writeMeta(meta: Meta, descriptor: NodeDescriptor?) {
+        val string = yaml.dump(meta.toMap(descriptor))
+        writeText(string)
+    }
+
+    override fun Input.readMeta(descriptor: NodeDescriptor?): Meta {
+        val map: Map<String, Any?> = yaml.load(asStream())
+        return map.toMeta(descriptor)
+    }
+
+    companion object : MetaFormatFactory {
+        val default = YamlMetaFormat()
+
+        override fun invoke(meta: Meta, context: Context): MetaFormat = YamlMetaFormat(meta)
+
+        override val name: Name = super.name + "yaml"
+
+        override val key: Short = 0x594d //YM
+    }
+}

dataforge-io/dataforge-io-yaml/src/test/kotlin/hep/dataforge/io/yaml/YamlMetaFormatTest.kt

@@ -0,0 +1,43 @@
+package hep.dataforge.io.yaml
+
+import hep.dataforge.io.parse
+import hep.dataforge.io.toString
+import hep.dataforge.meta.Meta
+import hep.dataforge.meta.buildMeta
+import hep.dataforge.meta.get
+import hep.dataforge.meta.seal
+import org.junit.Test
+import kotlin.test.assertEquals
+
+
+class YamlMetaFormatTest{
+    @Test
+    fun testYamlMetaFormat(){
+        val meta = buildMeta {
+            "a" put 22
+            "node" put {
+                "b" put "DDD"
+                "c" put 11.1
+                "d" put {
+                    "d1" put {
+                        "d11" put "aaa"
+                        "d12" put "bbb"
+                    }
+                    "d2" put 2
+                }
+                "array" put doubleArrayOf(1.0, 2.0, 3.0)
+            }
+        }
+        val string = meta.toString(YamlMetaFormat)
+        println(string)
+        val result = YamlMetaFormat.parse(string)
+
+        assertEquals<Meta>(meta, meta.seal())
+
+        meta.items.keys.forEach {
+            if (meta[it] != result[it]) error("${meta[it]} != ${result[it]}")
+        }
+
+        assertEquals(meta, result)
+    }
+}

dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Binary.kt

@@ -1,9 +1,7 @@
 package hep.dataforge.io
 
-import kotlinx.io.core.ByteReadPacket
+import kotlinx.io.core.*
-import kotlinx.io.core.Input
+import kotlin.math.min
-import kotlinx.io.core.buildPacket
-import kotlinx.io.core.readBytes
 
 /**
  * A source of binary data
@@ -30,19 +28,21 @@ interface RandomAccessBinary : Binary {
     /**
      * Read at most [size] of bytes starting at [from] offset from the beginning of the binary.
      * This method could be called multiple times simultaneously.
+     *
+     * If size
      */
     fun <R> read(from: UInt, size: UInt = UInt.MAX_VALUE, block: Input.() -> R): R
 
     override fun <R> read(block: Input.() -> R): R = read(0.toUInt(), UInt.MAX_VALUE, block)
 }
 
-fun Binary.readAll(): ByteReadPacket = read {
+fun Binary.toBytes(): ByteArray = read {
-    ByteReadPacket(this.readBytes())
+    this.readBytes()
 }
 
 @ExperimentalUnsignedTypes
 fun RandomAccessBinary.readPacket(from: UInt, size: UInt): ByteReadPacket = read(from, size) {
-    ByteReadPacket(this.readBytes())
+    buildPacket { copyTo(this) }
 }
 
 @ExperimentalUnsignedTypes
@@ -53,33 +53,35 @@ object EmptyBinary : RandomAccessBinary {
     override fun <R> read(from: UInt, size: UInt, block: Input.() -> R): R {
         error("The binary is empty")
     }
+
 }
 
 @ExperimentalUnsignedTypes
-class ArrayBinary(val array: ByteArray) : RandomAccessBinary {
+inline class ArrayBinary(val array: ByteArray) : RandomAccessBinary {
     override val size: ULong get() = array.size.toULong()
 
     override fun <R> read(from: UInt, size: UInt, block: Input.() -> R): R {
-        return ByteReadPacket(array, from.toInt(), size.toInt()).block()
+        val theSize = min(size, array.size.toUInt() - from)
+        return buildPacket {
+            writeFully(array, from.toInt(), theSize.toInt())
+        }.block()
     }
 }
 
+fun ByteArray.asBinary() = ArrayBinary(this)
+
 /**
  * Read given binary as object using given format
  */
 fun <T : Any> Binary.readWith(format: IOFormat<T>): T = format.run {
     read {
-        readThis()
+        readObject()
     }
 }
 
-/**
+fun <T : Any> IOFormat<T>.writeBinary(obj: T): Binary {
- * Write this object to a binary
- * TODO make a lazy binary that does not use intermediate array
- */
-fun <T: Any> T.writeWith(format: IOFormat<T>): Binary = format.run{
     val packet = buildPacket {
-        writeThis(this@writeWith)
+        writeObject(obj)
     }
-    return@run ArrayBinary(packet.readBytes())
+    return ArrayBinary(packet.readBytes())
 }

dataforge-io/src/commonMain/kotlin/hep/dataforge/io/BinaryMetaFormat.kt

@@ -1,17 +1,22 @@
 package hep.dataforge.io
 
+import hep.dataforge.context.Context
 import hep.dataforge.descriptors.NodeDescriptor
 import hep.dataforge.meta.*
+import hep.dataforge.names.Name
+import hep.dataforge.names.plus
 import hep.dataforge.values.*
 import kotlinx.io.core.Input
 import kotlinx.io.core.Output
 import kotlinx.io.core.readText
 import kotlinx.io.core.writeText
 
-object BinaryMetaFormat : MetaFormat {
+object BinaryMetaFormat : MetaFormat, MetaFormatFactory {
-    override val name: String = "bin"
+    override val name: Name = super.name + "bin"
     override val key: Short = 0x4249//BI
 
+    override fun invoke(meta: Meta, context: Context): MetaFormat = this
+
     override fun Input.readMeta(descriptor: NodeDescriptor?): Meta {
         return (readMetaItem() as MetaItem.NodeItem).node
     }
@@ -23,7 +28,7 @@ object BinaryMetaFormat : MetaFormat {
         writeText(str)
     }
 
-    private fun Output.writeValue(value: Value) {
+    fun Output.writeValue(value: Value) {
         if (value.isList()) {
             writeChar('L')
             writeInt(value.list.size)
@@ -80,7 +85,7 @@ object BinaryMetaFormat : MetaFormat {
                     writeValue(item.value)
                 }
                 is MetaItem.NodeItem -> {
-                    writeThis(item.node)
+                    writeObject(item.node)
                 }
             }
         }
@@ -92,7 +97,7 @@ object BinaryMetaFormat : MetaFormat {
     }
 
     @Suppress("UNCHECKED_CAST")
-    private fun Input.readMetaItem(): MetaItem<MetaBuilder> {
+    fun Input.readMetaItem(): MetaItem<MetaBuilder> {
         return when (val keyChar = readByte().toChar()) {
             'S' -> MetaItem.ValueItem(StringValue(readString()))
             'N' -> MetaItem.ValueItem(Null)

dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Envelope.kt

@@ -1,9 +1,11 @@
 package hep.dataforge.io
 
-import hep.dataforge.meta.Laminate
+import hep.dataforge.meta.*
-import hep.dataforge.meta.Meta
+import hep.dataforge.names.asName
-import hep.dataforge.meta.get
+import hep.dataforge.names.plus
-import hep.dataforge.meta.string
+import kotlinx.io.core.Output
+import kotlinx.io.core.buildPacket
+import kotlinx.io.core.readBytes
 
 interface Envelope {
     val meta: Meta
@@ -14,12 +16,17 @@ interface Envelope {
         /**
          * meta keys
          */
-        const val ENVELOPE_NODE = "@envelope"
+        val ENVELOPE_NODE_KEY = "@envelope".asName()
-        const val ENVELOPE_TYPE_KEY = "$ENVELOPE_NODE.type"
+        val ENVELOPE_TYPE_KEY = ENVELOPE_NODE_KEY + "type"
-        const val ENVELOPE_DATA_TYPE_KEY = "$ENVELOPE_NODE.dataType"
+        val ENVELOPE_DATA_TYPE_KEY = ENVELOPE_NODE_KEY + "dataType"
-        const val ENVELOPE_DESCRIPTION_KEY = "$ENVELOPE_NODE.description"
+        val ENVELOPE_DATA_ID_KEY = ENVELOPE_NODE_KEY + "dataID"
+        val ENVELOPE_DESCRIPTION_KEY = ENVELOPE_NODE_KEY + "description"
         //const val ENVELOPE_TIME_KEY = "@envelope.time"
 
+        /**
+         * Build a static envelope using provided builder
+         */
+        operator fun invoke(block: EnvelopeBuilder.() -> Unit) = EnvelopeBuilder().apply(block).build()
     }
 }
 
@@ -28,24 +35,35 @@ class SimpleEnvelope(override val meta: Meta, override val data: Binary?) : Enve
 /**
  * The purpose of the envelope
  *
- * @return
  */
 val Envelope.type: String? get() = meta[Envelope.ENVELOPE_TYPE_KEY].string
 
 /**
  * The type of data encoding
  *
- * @return
  */
 val Envelope.dataType: String? get() = meta[Envelope.ENVELOPE_DATA_TYPE_KEY].string
 
 /**
  * Textual user friendly description
  *
- * @return
  */
 val Envelope.description: String? get() = meta[Envelope.ENVELOPE_DESCRIPTION_KEY].string
 
+/**
+ * An optional unique identifier that is used for data comparison. Data without identifier could not be compared to another data.
+ */
+val Envelope.dataID: String? get() = meta[Envelope.ENVELOPE_DATA_ID_KEY].string
+
+fun Envelope.metaEquals(other: Envelope): Boolean = this.meta == other.meta
+
+fun Envelope.dataEquals(other: Envelope): Boolean = this.dataID != null && this.dataID == other.dataID
+
+fun Envelope.contentEquals(other: Envelope): Boolean {
+    return (this === other || (metaEquals(other) && dataEquals(other)))
+}
+
+
 /**
  * An envelope, which wraps existing envelope and adds one or several additional layers of meta
  */
@@ -55,7 +73,7 @@ class ProxyEnvelope(val source: Envelope, vararg meta: Meta) : Envelope {
 }
 
 /**
- * Add few meta layers to existing envelope
+ * Add few meta layers to existing envelope (on top of existing meta)
  */
 fun Envelope.withMetaLayers(vararg layers: Meta): Envelope {
     return when {
@@ -63,4 +81,34 @@ fun Envelope.withMetaLayers(vararg layers: Meta): Envelope {
         this is ProxyEnvelope -> ProxyEnvelope(source, *layers, *this.meta.layers.toTypedArray())
         else -> ProxyEnvelope(this, *layers)
     }
+}
+
+class EnvelopeBuilder {
+    private val metaBuilder = MetaBuilder()
+    var data: Binary? = null
+
+    fun meta(block: MetaBuilder.() -> Unit) {
+        metaBuilder.apply(block)
+    }
+
+    fun meta(meta: Meta) {
+        metaBuilder.update(meta)
+    }
+
+    var type by metaBuilder.string(key = Envelope.ENVELOPE_TYPE_KEY)
+    var dataType by metaBuilder.string(key = Envelope.ENVELOPE_DATA_TYPE_KEY)
+    var dataID by metaBuilder.string(key = Envelope.ENVELOPE_DATA_ID_KEY)
+    var description by metaBuilder.string(key = Envelope.ENVELOPE_DESCRIPTION_KEY)
+
+    /**
+     * Construct a binary and transform it into byte-array based buffer
+     */
+    fun data(block: Output.() -> Unit) {
+        val bytes = buildPacket {
+            block()
+        }
+        data = ArrayBinary(bytes.readBytes())
+    }
+
+    internal fun build() = SimpleEnvelope(metaBuilder.seal(), data)
 }

dataforge-io/src/commonMain/kotlin/hep/dataforge/io/EnvelopeFormat.kt

@@ -1,11 +1,15 @@
 package hep.dataforge.io
 
-import hep.dataforge.context.Named
+import hep.dataforge.context.Context
-import hep.dataforge.io.EnvelopeFormat.Companion.ENVELOPE_FORMAT_TYPE
+import hep.dataforge.io.EnvelopeFormatFactory.Companion.ENVELOPE_FORMAT_TYPE
+import hep.dataforge.meta.EmptyMeta
 import hep.dataforge.meta.Meta
+import hep.dataforge.names.Name
+import hep.dataforge.names.asName
 import hep.dataforge.provider.Type
 import kotlinx.io.core.Input
 import kotlinx.io.core.Output
+import kotlin.reflect.KClass
 
 /**
  * A partially read envelope with meta, but without data
@@ -13,19 +17,33 @@ import kotlinx.io.core.Output
 @ExperimentalUnsignedTypes
 data class PartialEnvelope(val meta: Meta, val dataOffset: UInt, val dataSize: ULong?)
 
+
+interface EnvelopeFormat : IOFormat<Envelope> {
+    val defaultMetaFormat: MetaFormatFactory get() = JsonMetaFormat
+
+    fun Input.readPartial(): PartialEnvelope
+
+    fun Output.writeEnvelope(envelope: Envelope, metaFormatFactory: MetaFormatFactory, formatMeta: Meta = EmptyMeta)
+
+    override fun Input.readObject(): Envelope
+
+    override fun Output.writeObject(obj: Envelope): Unit = writeEnvelope(obj, defaultMetaFormat)
+}
+
 @Type(ENVELOPE_FORMAT_TYPE)
-interface EnvelopeFormat : IOFormat<Envelope>, Named {
+interface EnvelopeFormatFactory : IOFormatFactory<Envelope> {
-    fun Input.readPartial(formats: Collection<MetaFormat> = IOPlugin.defaultMetaFormats): PartialEnvelope
+    override val name: Name get() = "envelope".asName()
+    override val type: KClass<out Envelope> get() = Envelope::class
 
-    fun Input.readEnvelope(formats: Collection<MetaFormat> = IOPlugin.defaultMetaFormats): Envelope
+    override fun invoke(meta: Meta, context: Context): EnvelopeFormat
 
-    override fun Input.readThis(): Envelope = readEnvelope()
+    /**
-
+     * Try to infer specific format from input and return null if the attempt is failed.
-    fun Output.writeEnvelope(envelope: Envelope, format: MetaFormat = JsonMetaFormat)
+     * This method does **not** return Input into initial state.
-
+     */
-    override fun Output.writeThis(obj: Envelope) = writeEnvelope(obj)
+    fun peekFormat(io: IOPlugin, input: Input): EnvelopeFormat?
 
     companion object {
-        const val ENVELOPE_FORMAT_TYPE = "envelopeFormat"
+        const val ENVELOPE_FORMAT_TYPE = "io.format.envelope"
     }
 }

dataforge-io/src/commonMain/kotlin/hep/dataforge/io/FunctionServer.kt

@@ -1,38 +0,0 @@
-package hep.dataforge.io
-
-import kotlin.reflect.KClass
-
-/**
- * A descriptor for specific type of functions
- */
-interface FunctionSpec<T : Any, R : Any> {
-    val inputType: KClass<T>
-    val outputType: KClass<R>
-}
-
-/**
- * A server that could produce asynchronous function values
- */
-interface FunctionServer {
-    /**
-     * Call a function with given name and descriptor
-     */
-    suspend fun <T : Any, R : Any, D : FunctionSpec<T, R>> call(name: String, descriptor: D, arg: T): R
-
-    /**
-     * Resolve a function descriptor for given types
-     */
-    fun <T : Any, R : Any> resolveType(inputType: KClass<out T>, outputType: KClass<out R>): FunctionSpec<T, R>
-
-    /**
-     * Get a generic suspended function with given name and descriptor
-     */
-    operator fun <T : Any, R : Any, D : FunctionSpec<T, R>> get(name: String, descriptor: D): (suspend (T) -> R) =
-        { call(name, descriptor, it) }
-}
-
-suspend inline fun <reified T : Any, reified R : Any> FunctionServer.call(name: String, arg: T): R =
-    call(name, resolveType(T::class, R::class), arg)
-
-inline operator fun <reified T : Any, reified R : Any> FunctionServer.get(name: String): (suspend (T) -> R) =
-    get(name, resolveType(T::class, R::class))

dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOFormat.kt

@@ -1,14 +1,164 @@
 package hep.dataforge.io
 
+import hep.dataforge.context.Context
+import hep.dataforge.context.Factory
+import hep.dataforge.context.Named
+import hep.dataforge.io.IOFormatFactory.Companion.IO_FORMAT_TYPE
+import hep.dataforge.meta.Meta
+import hep.dataforge.meta.MetaItem
+import hep.dataforge.names.Name
+import hep.dataforge.names.asName
+import hep.dataforge.provider.Type
+import hep.dataforge.values.Value
 import kotlinx.io.core.*
+import kotlinx.io.pool.ObjectPool
+import kotlinx.serialization.ImplicitReflectionSerializer
+import kotlinx.serialization.KSerializer
+import kotlinx.serialization.cbor.Cbor
+import kotlinx.serialization.serializer
+import kotlin.math.min
+import kotlin.reflect.KClass
 
 /**
- * And interface for serialization facilities
+ * And interface for reading and writing objects into with IO streams
  */
 interface IOFormat<T : Any> {
-    fun Output.writeThis(obj: T)
+    fun Output.writeObject(obj: T)
-    fun Input.readThis(): T
+    fun Input.readObject(): T
 }
 
-fun <T : Any> IOFormat<T>.writePacket(obj: T): ByteReadPacket = buildPacket { writeThis(obj) }
+fun <T : Any> Input.readWith(format: IOFormat<T>): T = format.run { readObject() }
-fun <T : Any> IOFormat<T>.writeBytes(obj: T): ByteArray = buildPacket { writeThis(obj) }.readBytes()
+fun <T : Any> Output.writeWith(format: IOFormat<T>, obj: T) = format.run { writeObject(obj) }
+
+class ListIOFormat<T : Any>(val format: IOFormat<T>) : IOFormat<List<T>> {
+    override fun Output.writeObject(obj: List<T>) {
+        writeInt(obj.size)
+        format.run {
+            obj.forEach {
+                writeObject(it)
+            }
+        }
+    }
+
+    override fun Input.readObject(): List<T> {
+        val size = readInt()
+        return format.run {
+            List(size) { readObject() }
+        }
+    }
+}
+
+val <T : Any> IOFormat<T>.list get() = ListIOFormat(this)
+
+fun ObjectPool<IoBuffer>.fill(block: IoBuffer.() -> Unit): IoBuffer {
+    val buffer = borrow()
+    return try {
+        buffer.apply(block)
+    } catch (ex: Exception) {
+        //recycle(buffer)
+        throw ex
+    }
+}
+
+@Type(IO_FORMAT_TYPE)
+interface IOFormatFactory<T : Any> : Factory<IOFormat<T>>, Named {
+    /**
+     * Explicit type for dynamic type checks
+     */
+    val type: KClass<out T>
+
+    companion object {
+        const val IO_FORMAT_TYPE = "io.format"
+    }
+}
+
+@Deprecated("To be removed in io-2")
+inline fun buildPacketWithoutPool(headerSizeHint: Int = 0, block: BytePacketBuilder.() -> Unit): ByteReadPacket {
+    val builder = BytePacketBuilder(headerSizeHint, IoBuffer.NoPool)
+    block(builder)
+    return builder.build()
+}
+
+fun <T : Any> IOFormat<T>.writePacket(obj: T): ByteReadPacket = buildPacket { writeObject(obj) }
+fun <T : Any> IOFormat<T>.writeBytes(obj: T): ByteArray = buildPacket { writeObject(obj) }.readBytes()
+fun <T : Any> IOFormat<T>.readBytes(array: ByteArray): T {
+    //= ByteReadPacket(array).readThis()
+    val byteArrayInput: Input = object : AbstractInput(
+        IoBuffer.Pool.borrow(),
+        remaining = array.size.toLong(),
+        pool = IoBuffer.Pool
+    ) {
+        var written = 0
+        override fun closeSource() {
+            // do nothing
+        }
+
+        override fun fill(): IoBuffer? {
+            if (array.size - written <= 0) return null
+
+            return IoBuffer.Pool.fill {
+                reserveEndGap(IoBuffer.ReservedSize)
+                val toWrite = min(capacity, array.size - written)
+                writeFully(array, written, toWrite)
+                written += toWrite
+            }
+        }
+
+    }
+    return byteArrayInput.readObject()
+}
+
+object DoubleIOFormat : IOFormat<Double>, IOFormatFactory<Double> {
+    override fun invoke(meta: Meta, context: Context): IOFormat<Double> = this
+
+    override val name: Name = "double".asName()
+
+    override val type: KClass<out Double> get() = Double::class
+
+    override fun Output.writeObject(obj: Double) {
+        writeDouble(obj)
+    }
+
+    override fun Input.readObject(): Double = readDouble()
+}
+
+object ValueIOFormat : IOFormat<Value>, IOFormatFactory<Value> {
+    override fun invoke(meta: Meta, context: Context): IOFormat<Value> = this
+
+    override val name: Name = "value".asName()
+
+    override val type: KClass<out Value> get() = Value::class
+
+    override fun Output.writeObject(obj: Value) {
+        BinaryMetaFormat.run { writeValue(obj) }
+    }
+
+    override fun Input.readObject(): Value {
+        return (BinaryMetaFormat.run { readMetaItem() } as? MetaItem.ValueItem)?.value
+            ?: error("The item is not a value")
+    }
+}
+
+/**
+ * Experimental
+ */
+@ImplicitReflectionSerializer
+class SerializerIOFormat<T : Any>(
+    type: KClass<T>,
+    val serializer: KSerializer<T> = type.serializer()
+) : IOFormat<T> {
+
+    //override val name: Name = type.simpleName?.toName() ?: EmptyName
+
+
+    override fun Output.writeObject(obj: T) {
+        val bytes = Cbor.plain.dump(serializer, obj)
+        writeFully(bytes)
+    }
+
+    override fun Input.readObject(): T {
+        //FIXME reads the whole input
+        val bytes = readBytes()
+        return Cbor.plain.load(serializer, bytes)
+    }
+}

dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOPlugin.kt

@@ -1,37 +1,61 @@
 package hep.dataforge.io
 
-import hep.dataforge.context.AbstractPlugin
+import hep.dataforge.context.*
-import hep.dataforge.context.PluginFactory
+import hep.dataforge.io.EnvelopeFormatFactory.Companion.ENVELOPE_FORMAT_TYPE
-import hep.dataforge.context.PluginTag
+import hep.dataforge.io.IOFormatFactory.Companion.IO_FORMAT_TYPE
-import hep.dataforge.context.content
+import hep.dataforge.io.MetaFormatFactory.Companion.META_FORMAT_TYPE
-import hep.dataforge.meta.Meta
+import hep.dataforge.meta.*
 import hep.dataforge.names.Name
+import hep.dataforge.names.get
 import kotlin.reflect.KClass
 
 class IOPlugin(meta: Meta) : AbstractPlugin(meta) {
     override val tag: PluginTag get() = Companion.tag
 
-    val metaFormats by lazy {
+    val metaFormatFactories by lazy {
-        context.content<MetaFormat>(MetaFormat.META_FORMAT_TYPE).values
+        context.content<MetaFormatFactory>(META_FORMAT_TYPE).values
     }
 
-    fun metaFormat(key: Short): MetaFormat? = metaFormats.find { it.key == key }
+    fun metaFormat(key: Short, meta: Meta = EmptyMeta): MetaFormat? =
-    fun metaFormat(name: String): MetaFormat? = metaFormats.find { it.name == name }
+        metaFormatFactories.find { it.key == key }?.invoke(meta)
+
+    fun metaFormat(name: String, meta: Meta = EmptyMeta): MetaFormat? =
+        metaFormatFactories.find { it.name.toString() == name }?.invoke(meta)
+
+    val envelopeFormatFactories by lazy {
+        context.content<EnvelopeFormatFactory>(ENVELOPE_FORMAT_TYPE).values
+    }
 
     override fun provideTop(target: String): Map<Name, Any> {
         return when (target) {
-            MetaFormat.META_FORMAT_TYPE -> defaultMetaFormats.toMap()
+            META_FORMAT_TYPE -> defaultMetaFormats.toMap()
-            EnvelopeFormat.ENVELOPE_FORMAT_TYPE -> defaultEnvelopeFormats.toMap()
+            ENVELOPE_FORMAT_TYPE -> defaultEnvelopeFormats.toMap()
             else -> super.provideTop(target)
         }
     }
 
+    val ioFormats: Map<Name, IOFormatFactory<*>> by lazy {
+        context.content<IOFormatFactory<*>>(IO_FORMAT_TYPE)
+    }
+
+    fun <T : Any> resolveIOFormat(item: MetaItem<*>, type: KClass<out T>): IOFormat<T>? {
+        val key = item.string ?: item.node["name"]?.string ?: error("Format name not defined")
+        return ioFormats[key]?.let {
+            @Suppress("UNCHECKED_CAST")
+            if (it.type != type) error("Format type ${it.type} is not the same as requested type $type")
+            else it.invoke(item.node["meta"].node ?: EmptyMeta, context) as IOFormat<T>
+        }
+    }
+
     companion object : PluginFactory<IOPlugin> {
-        val defaultMetaFormats: List<MetaFormat> = listOf(JsonMetaFormat, BinaryMetaFormat)
+        val defaultMetaFormats: List<MetaFormatFactory> = listOf(JsonMetaFormat, BinaryMetaFormat)
         val defaultEnvelopeFormats = listOf(TaggedEnvelopeFormat)
 
         override val tag: PluginTag = PluginTag("io", group = PluginTag.DATAFORGE_GROUP)
+
         override val type: KClass<out IOPlugin> = IOPlugin::class
-        override fun invoke(meta: Meta): IOPlugin = IOPlugin(meta)
+        override fun invoke(meta: Meta, context: Context): IOPlugin = IOPlugin(meta)
     }
-}
+}
+
+val Context.io: IOPlugin get() = plugins.fetch(IOPlugin)

dataforge-io/src/commonMain/kotlin/hep/dataforge/io/JsonMetaFormat.kt

@@ -1,12 +1,17 @@
+@file:Suppress("UNUSED_PARAMETER")
+
 package hep.dataforge.io
 
+import hep.dataforge.context.Context
 import hep.dataforge.descriptors.ItemDescriptor
 import hep.dataforge.descriptors.NodeDescriptor
 import hep.dataforge.descriptors.ValueDescriptor
 import hep.dataforge.meta.Meta
 import hep.dataforge.meta.MetaBase
 import hep.dataforge.meta.MetaItem
+import hep.dataforge.names.Name
 import hep.dataforge.names.NameToken
+import hep.dataforge.names.plus
 import hep.dataforge.names.toName
 import hep.dataforge.values.*
 import kotlinx.io.core.Input
@@ -19,28 +24,32 @@ import kotlin.collections.component2
 import kotlin.collections.set
 
 
-object JsonMetaFormat : MetaFormat {
+class JsonMetaFormat(private val json: Json = Json.indented) : MetaFormat {
-
-    override val name: String = "json"
-    override val key: Short = 0x4a53//"JS"
 
     override fun Output.writeMeta(meta: Meta, descriptor: NodeDescriptor?) {
-        val json = meta.toJson(descriptor)
+        val jsonObject = meta.toJson(descriptor)
-        writeText(json.toString())
+        writeText(json.stringify(JsonObjectSerializer, jsonObject))
     }
 
     override fun Input.readMeta(descriptor: NodeDescriptor?): Meta {
         val str = readText()
-        val json = Json.plain.parseJson(str)
+        val jsonElement = json.parseJson(str)
+        return jsonElement.toMeta()
+    }
 
-        if (json is JsonObject) {
+    companion object : MetaFormatFactory {
-            return json.toMeta()
+        val default = JsonMetaFormat()
-        } else {
+
-            TODO("Non-object root not supported")
+        override fun invoke(meta: Meta, context: Context): MetaFormat = default
-        }
+
+        override val name: Name = super.name + "json"
+        override val key: Short = 0x4a53//"JS"
     }
 }
 
+/**
+ * @param descriptor reserved for custom serialization in future
+ */
 fun Value.toJson(descriptor: ValueDescriptor? = null): JsonElement {
     return if (isList()) {
         JsonArray(list.map { it.toJson() })
@@ -54,7 +63,7 @@ fun Value.toJson(descriptor: ValueDescriptor? = null): JsonElement {
     }
 }
 
-//Use theese methods to customize JSON key mapping
+//Use these methods to customize JSON key mapping
 private fun NameToken.toJsonKey(descriptor: ItemDescriptor?) = toString()
 
 private fun NodeDescriptor?.getDescriptor(key: String) = this?.items?.get(key)
@@ -78,7 +87,12 @@ fun Meta.toJson(descriptor: NodeDescriptor? = null): JsonObject {
     return JsonObject(map)
 }
 
-fun JsonObject.toMeta(descriptor: NodeDescriptor? = null): JsonMeta = JsonMeta(this, descriptor)
+fun JsonElement.toMeta(descriptor: NodeDescriptor? = null): Meta {
+    return when (val item = toMetaItem(descriptor)) {
+        is MetaItem.NodeItem<*> -> item.node
+        is MetaItem.ValueItem ->item.value.toMeta()
+    }
+}
 
 fun JsonPrimitive.toValue(descriptor: ValueDescriptor?): Value {
     return when (this) {
@@ -93,7 +107,7 @@ fun JsonElement.toMetaItem(descriptor: ItemDescriptor? = null): MetaItem<JsonMet
         MetaItem.ValueItem(value)
     }
     is JsonObject -> {
-        val meta = toMeta(descriptor as? NodeDescriptor)
+        val meta = JsonMeta(this, descriptor as? NodeDescriptor)
         MetaItem.NodeItem(meta)
     }
     is JsonArray -> {
@@ -129,7 +143,7 @@ class JsonMeta(val json: JsonObject, val descriptor: NodeDescriptor? = null) : M
                 this[name] = MetaItem.ValueItem(value.toValue(itemDescriptor as? ValueDescriptor)) as MetaItem<JsonMeta>
             }
             is JsonObject -> {
-                this[name] = MetaItem.NodeItem(value.toMeta(itemDescriptor as? NodeDescriptor))
+                this[name] = MetaItem.NodeItem(JsonMeta(value, itemDescriptor as? NodeDescriptor))
             }
             is JsonArray -> {
                 when {

dataforge-io/src/commonMain/kotlin/hep/dataforge/io/MetaFormat.kt

@@ -1,49 +1,64 @@
 package hep.dataforge.io
 
-import hep.dataforge.context.Named
+import hep.dataforge.context.Context
 import hep.dataforge.descriptors.NodeDescriptor
-import hep.dataforge.io.MetaFormat.Companion.META_FORMAT_TYPE
+import hep.dataforge.io.MetaFormatFactory.Companion.META_FORMAT_TYPE
 import hep.dataforge.meta.Meta
+import hep.dataforge.names.Name
+import hep.dataforge.names.asName
 import hep.dataforge.provider.Type
 import kotlinx.io.core.*
+import kotlin.reflect.KClass
 
 /**
  * A format for meta serialization
  */
-@Type(META_FORMAT_TYPE)
-interface MetaFormat : IOFormat<Meta>, Named {
-    override val name: String
-    val key: Short
 
-    override fun Output.writeThis(obj: Meta) {
+interface MetaFormat : IOFormat<Meta> {
+
+    override fun Output.writeObject(obj: Meta) {
         writeMeta(obj, null)
     }
 
-    override fun Input.readThis(): Meta = readMeta(null)
+    override fun Input.readObject(): Meta = readMeta()
 
     fun Output.writeMeta(meta: Meta, descriptor: NodeDescriptor? = null)
     fun Input.readMeta(descriptor: NodeDescriptor? = null): Meta
+}
 
-    companion object{
+@Type(META_FORMAT_TYPE)
-        const val META_FORMAT_TYPE = "metaFormat"
+interface MetaFormatFactory : IOFormatFactory<Meta> {
+    override val name: Name get() = "meta".asName()
+
+    override val type: KClass<out Meta> get() = Meta::class
+
+    val key: Short
+
+    override operator fun invoke(meta: Meta, context: Context): MetaFormat
+
+    companion object {
+        const val META_FORMAT_TYPE = "io.format.meta"
     }
 }
 
 fun Meta.toString(format: MetaFormat): String = buildPacket {
-    format.run { writeThis(this@toString) }
+    format.run { writeObject(this@toString) }
 }.readText()
 
-fun Meta.toBytes(format: MetaFormat = JsonMetaFormat): ByteReadPacket = buildPacket {
+fun Meta.toString(formatFactory: MetaFormatFactory): String = toString(formatFactory())
-    format.run { writeThis(this@toBytes) }
-}
 
+fun Meta.toBytes(format: MetaFormat = JsonMetaFormat.default): ByteReadPacket = buildPacket {
+    format.run { writeObject(this@toBytes) }
+}
 
 fun MetaFormat.parse(str: String): Meta {
-    return ByteReadPacket(str.toByteArray()).readThis()
+    return buildPacket { writeText(str) }.readObject()
 }
 
+fun MetaFormatFactory.parse(str: String): Meta = invoke().parse(str)
+
 fun MetaFormat.fromBytes(packet: ByteReadPacket): Meta {
-    return packet.readThis()
+    return packet.readObject()
 }
 
 

dataforge-io/src/commonMain/kotlin/hep/dataforge/io/MetaSerializer.kt

@@ -1,53 +0,0 @@
-package hep.dataforge.io
-
-import hep.dataforge.meta.Config
-import hep.dataforge.meta.Meta
-import hep.dataforge.meta.toConfig
-import hep.dataforge.names.Name
-import hep.dataforge.names.toName
-import kotlinx.serialization.*
-import kotlinx.serialization.internal.StringDescriptor
-import kotlinx.serialization.json.JsonObjectSerializer
-
-@Serializer(Name::class)
-object NameSerializer : KSerializer<Name> {
-    override val descriptor: SerialDescriptor = StringDescriptor
-
-    override fun deserialize(decoder: Decoder): Name {
-        return decoder.decodeString().toName()
-    }
-
-    override fun serialize(encoder: Encoder, obj: Name) {
-        encoder.encodeString(obj.toString())
-    }
-}
-
-/**
- * Serialized for meta
- */
-@Serializer(Meta::class)
-object MetaSerializer : KSerializer<Meta> {
-    override val descriptor: SerialDescriptor = JsonObjectSerializer.descriptor
-
-    override fun deserialize(decoder: Decoder): Meta {
-        //currently just delegates serialization to json serializer
-        return JsonObjectSerializer.deserialize(decoder).toMeta()
-    }
-
-    override fun serialize(encoder: Encoder, obj: Meta) {
-        JsonObjectSerializer.serialize(encoder, obj.toJson())
-    }
-}
-
-@Serializer(Config::class)
-object ConfigSerializer : KSerializer<Config> {
-    override val descriptor: SerialDescriptor = JsonObjectSerializer.descriptor
-
-    override fun deserialize(decoder: Decoder): Config {
-        return JsonObjectSerializer.deserialize(decoder).toMeta().toConfig()
-    }
-
-    override fun serialize(encoder: Encoder, obj: Config) {
-        JsonObjectSerializer.serialize(encoder, obj.toJson())
-    }
-}

dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Responder.kt

@@ -0,0 +1,8 @@
+package hep.dataforge.io
+
+interface Responder {
+    /**
+     * Send a request and wait for response for this specific request
+     */
+    suspend fun respond(request: Envelope): Envelope
+}

dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt

@@ -1,43 +1,50 @@
 package hep.dataforge.io
 
+import hep.dataforge.context.Context
+import hep.dataforge.meta.Meta
+import hep.dataforge.meta.get
+import hep.dataforge.meta.string
+import hep.dataforge.names.Name
+import hep.dataforge.names.plus
+import hep.dataforge.names.toName
+import kotlinx.io.charsets.Charsets
 import kotlinx.io.core.*
 
-
 @ExperimentalUnsignedTypes
-object TaggedEnvelopeFormat : EnvelopeFormat {
+class TaggedEnvelopeFormat(
-    const val VERSION = "DF03"
+    val io: IOPlugin,
-    private const val START_SEQUENCE = "#~"
+    val version: VERSION = TaggedEnvelopeFormat.VERSION.DF02
-    private const val END_SEQUENCE = "~#\r\n"
+) : EnvelopeFormat {
-    private const val TAG_SIZE = 26u
+
+//    private val metaFormat = io.metaFormat(metaFormatKey)
+//        ?: error("Meta format with key $metaFormatKey could not be resolved in $io")
 
-    override val name: String get() = VERSION
 
     private fun Tag.toBytes(): ByteReadPacket = buildPacket(24) {
         writeText(START_SEQUENCE)
-        writeText(VERSION)
+        writeText(version.name)
         writeShort(metaFormatKey)
         writeUInt(metaSize)
-        writeULong(dataSize)
+        when (version) {
+            TaggedEnvelopeFormat.VERSION.DF02 -> {
+                writeUInt(dataSize.toUInt())
+            }
+            TaggedEnvelopeFormat.VERSION.DF03 -> {
+                writeULong(dataSize)
+            }
+        }
         writeText(END_SEQUENCE)
     }
 
-    private fun Input.readTag(): Tag {
+    override fun Output.writeEnvelope(envelope: Envelope, metaFormatFactory: MetaFormatFactory, formatMeta: Meta) {
-        val start = readTextExactBytes(2)
+        val metaFormat = metaFormatFactory.invoke(formatMeta, io.context)
-        if (start != START_SEQUENCE) error("The input is not an envelope")
+        val metaBytes = metaFormat.writeBytes(envelope.meta)
-        val version = readTextExactBytes(4)
+        val tag = Tag(metaFormatFactory.key, metaBytes.size.toUInt() + 2u, envelope.data?.size ?: 0.toULong())
-        if (version != VERSION) error("Wrong version of DataForge: expected $VERSION but found $version")
-        val metaFormatKey = readShort()
-        val metaLength = readUInt()
-        val dataLength = readULong()
-        return Tag(metaFormatKey, metaLength, dataLength)
-    }
-
-    override fun Output.writeEnvelope(envelope: Envelope, format: MetaFormat) {
-        val metaBytes = format.writeBytes(envelope.meta)
-        val tag = Tag(format.key, metaBytes.size.toUInt(), envelope.data?.size ?: 0.toULong())
         writePacket(tag.toBytes())
         writeFully(metaBytes)
+        writeText("\r\n")
         envelope.data?.read { copyTo(this@writeEnvelope) }
+        flush()
     }
 
     /**
@@ -46,30 +53,29 @@ object TaggedEnvelopeFormat : EnvelopeFormat {
      * @param input an input to read from
      * @param formats a collection of meta formats to resolve
      */
-    override fun Input.readEnvelope(formats: Collection<MetaFormat>): Envelope {
+    override fun Input.readObject(): Envelope {
-        val tag = readTag()
+        val tag = readTag(version)
 
-        val metaFormat = formats.find { it.key == tag.metaFormatKey }
+        val metaFormat = io.metaFormat(tag.metaFormatKey)
             ?: error("Meta format with key ${tag.metaFormatKey} not found")
 
         val metaPacket = ByteReadPacket(readBytes(tag.metaSize.toInt()))
-        val meta = metaFormat.run { metaPacket.readThis() }
-
         val dataBytes = readBytes(tag.dataSize.toInt())
 
+        val meta = metaFormat.run { metaPacket.readObject() }
         return SimpleEnvelope(meta, ArrayBinary(dataBytes))
     }
 
-    override fun Input.readPartial(formats: Collection<MetaFormat>): PartialEnvelope {
+    override fun Input.readPartial(): PartialEnvelope {
-        val tag = readTag()
+        val tag = readTag(version)
 
-        val metaFormat = formats.find { it.key == tag.metaFormatKey }
+        val metaFormat = io.metaFormat(tag.metaFormatKey)
             ?: error("Meta format with key ${tag.metaFormatKey} not found")
 
         val metaPacket = ByteReadPacket(readBytes(tag.metaSize.toInt()))
-        val meta = metaFormat.run { metaPacket.readThis() }
+        val meta = metaFormat.run { metaPacket.readObject() }
 
-        return PartialEnvelope(meta, TAG_SIZE + tag.metaSize, tag.dataSize)
+        return PartialEnvelope(meta, version.tagSize + tag.metaSize, tag.dataSize)
     }
 
     private data class Tag(
@@ -78,4 +84,57 @@ object TaggedEnvelopeFormat : EnvelopeFormat {
         val dataSize: ULong
     )
 
+    enum class VERSION(val tagSize: UInt) {
+        DF02(20u),
+        DF03(24u)
+    }
+
+    companion object : EnvelopeFormatFactory {
+        private const val START_SEQUENCE = "#~"
+        private const val END_SEQUENCE = "~#\r\n"
+
+        override val name: Name = super.name + "tagged"
+
+        override fun invoke(meta: Meta, context: Context): EnvelopeFormat {
+            val io = context.io
+
+            val metaFormatName = meta["name"].string?.toName() ?: JsonMetaFormat.name
+            val metaFormatFactory = io.metaFormatFactories.find { it.name == metaFormatName }
+                ?: error("Meta format could not be resolved")
+
+            return TaggedEnvelopeFormat(io)
+        }
+
+        private fun Input.readTag(version: VERSION): Tag {
+            val start = readTextExactBytes(2, charset = Charsets.ISO_8859_1)
+            if (start != START_SEQUENCE) error("The input is not an envelope")
+            val versionString = readTextExactBytes(4, charset = Charsets.ISO_8859_1)
+            if (version.name != versionString) error("Wrong version of DataForge: expected $version but found $versionString")
+            val metaFormatKey = readShort()
+            val metaLength = readUInt()
+            val dataLength: ULong = when (version) {
+                VERSION.DF02 -> readUInt().toULong()
+                VERSION.DF03 -> readULong()
+            }
+            val end = readTextExactBytes(4, charset = Charsets.ISO_8859_1)
+            if (end != END_SEQUENCE) error("The input is not an envelope")
+            return Tag(metaFormatKey, metaLength, dataLength)
+        }
+
+        override fun peekFormat(io: IOPlugin, input: Input): EnvelopeFormat? {
+            return try {
+                val header = input.readTextExactBytes(6)
+                when (header.substring(2..5)) {
+                    VERSION.DF02.name -> TaggedEnvelopeFormat(io, VERSION.DF02)
+                    VERSION.DF03.name -> TaggedEnvelopeFormat(io, VERSION.DF03)
+                    else -> null
+                }
+            } catch (ex: Exception) {
+                null
+            }
+        }
+
+        val default by lazy { invoke() }
+    }
+
 }

dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaglessEnvelopeFormat.kt

@@ -0,0 +1,189 @@
+package hep.dataforge.io
+
+import hep.dataforge.context.Context
+import hep.dataforge.meta.*
+import hep.dataforge.names.asName
+import kotlinx.io.core.*
+import kotlinx.serialization.toUtf8Bytes
+
+class TaglessEnvelopeFormat(
+    val io: IOPlugin,
+    meta: Meta = EmptyMeta
+) : EnvelopeFormat {
+
+    private val metaStart = meta[META_START_PROPERTY].string ?: DEFAULT_META_START
+    private val dataStart = meta[DATA_START_PROPERTY].string ?: DEFAULT_DATA_START
+
+    private fun Output.writeProperty(key: String, value: Any) {
+        writeText("#? $key: $value;\r\n")
+    }
+
+    override fun Output.writeEnvelope(envelope: Envelope, metaFormatFactory: MetaFormatFactory, formatMeta: Meta) {
+        val metaFormat = metaFormatFactory(formatMeta, io.context)
+
+        //printing header
+        writeText(TAGLESS_ENVELOPE_HEADER + "\r\n")
+
+        //printing all properties
+        writeProperty(META_TYPE_PROPERTY, metaFormatFactory.type)
+        //TODO add optional metaFormat properties
+        writeProperty(DATA_LENGTH_PROPERTY, envelope.data?.size ?: 0)
+
+        //Printing meta
+        if (!envelope.meta.isEmpty()) {
+            val metaBytes = metaFormat.writeBytes(envelope.meta)
+            writeProperty(META_LENGTH_PROPERTY, metaBytes.size)
+            writeText(metaStart + "\r\n")
+            writeFully(metaBytes)
+            writeText("\r\n")
+        }
+
+        //Printing data
+        envelope.data?.let { data ->
+            writeText(dataStart + "\r\n")
+            writeFully(data.toBytes())
+        }
+    }
+
+    override fun Input.readObject(): Envelope {
+        var line: String = ""
+        do {
+            line = readUTF8Line() ?: error("Input does not contain tagless envelope header")
+        } while (!line.startsWith(TAGLESS_ENVELOPE_HEADER))
+        val properties = HashMap<String, String>()
+
+        line = ""
+        while (line.isBlank() || line.startsWith("#?")) {
+            if (line.startsWith("#?")) {
+                val match = propertyPattern.find(line)
+                    ?: error("Line $line does not match property declaration pattern")
+                val (key, value) = match.destructured
+                properties[key] = value
+            }
+            line = readUTF8Line() ?: return SimpleEnvelope(Meta.empty, null)
+        }
+
+        var meta: Meta = EmptyMeta
+
+        if (line.startsWith(metaStart)) {
+            val metaFormat = properties[META_TYPE_PROPERTY]?.let { io.metaFormat(it) } ?: JsonMetaFormat.default
+            val metaSize = properties.get(META_LENGTH_PROPERTY)?.toInt()
+            meta = if (metaSize != null) {
+                val metaPacket = buildPacket {
+                    writeFully(readBytes(metaSize))
+                }
+                metaFormat.run { metaPacket.readObject() }
+            } else {
+                metaFormat.run {
+                    readObject()
+                }
+            }
+        }
+
+        do {
+            line = readUTF8Line() ?: return SimpleEnvelope(meta, null)
+            //returning an Envelope without data if end of input is reached
+        } while (!line.startsWith(dataStart))
+
+        val data: Binary? = if (properties.containsKey(DATA_LENGTH_PROPERTY)) {
+            val bytes = ByteArray(properties[DATA_LENGTH_PROPERTY]!!.toInt())
+            readFully(bytes)
+            bytes.asBinary()
+        } else {
+            val bytes = readBytes()
+            bytes.asBinary()
+        }
+
+        return SimpleEnvelope(meta, data)
+    }
+
+    override fun Input.readPartial(): PartialEnvelope {
+        var offset = 0u
+        var line: String = ""
+        do {
+            line = readUTF8Line() ?: error("Input does not contain tagless envelope header")
+            offset += line.toUtf8Bytes().size.toUInt()
+        } while (!line.startsWith(TAGLESS_ENVELOPE_HEADER))
+        val properties = HashMap<String, String>()
+
+        line = ""
+        while (line.isBlank() || line.startsWith("#?")) {
+            if (line.startsWith("#?")) {
+                val match = propertyPattern.find(line)
+                    ?: error("Line $line does not match property declaration pattern")
+                val (key, value) = match.destructured
+                properties[key] = value
+            }
+            line = readUTF8Line() ?: return PartialEnvelope(Meta.empty, offset.toUInt(), 0.toULong())
+            offset += line.toUtf8Bytes().size.toUInt()
+        }
+
+        var meta: Meta = EmptyMeta
+
+        if (line.startsWith(metaStart)) {
+            val metaFormat = properties[META_TYPE_PROPERTY]?.let { io.metaFormat(it) } ?: JsonMetaFormat.default
+
+            val metaSize = properties.get(META_LENGTH_PROPERTY)?.toInt()
+            meta = if (metaSize != null) {
+                val metaPacket = buildPacket {
+                    writeFully(readBytes(metaSize))
+                }
+                offset += metaSize.toUInt()
+                metaFormat.run { metaPacket.readObject() }
+            } else {
+                error("Can't partially read an envelope with undefined meta size")
+            }
+        }
+
+        do {
+            line = readUTF8Line() ?: return PartialEnvelope(Meta.empty, offset.toUInt(), 0.toULong())
+            offset += line.toUtf8Bytes().size.toUInt()
+            //returning an Envelope without data if end of input is reached
+        } while (!line.startsWith(dataStart))
+
+        val dataSize = properties[DATA_LENGTH_PROPERTY]?.toULong()
+        return PartialEnvelope(meta, offset, dataSize)
+    }
+
+    companion object : EnvelopeFormatFactory {
+
+        private val propertyPattern = "#\\?\\s*(?<key>[\\w.]*)\\s*:\\s*(?<value>[^;]*);?".toRegex()
+
+        const val META_TYPE_PROPERTY = "metaType"
+        const val META_LENGTH_PROPERTY = "metaLength"
+        const val DATA_LENGTH_PROPERTY = "dataLength"
+
+
+        const val TAGLESS_ENVELOPE_TYPE = "tagless"
+
+        const val TAGLESS_ENVELOPE_HEADER = "#~DFTL~#"
+        const val META_START_PROPERTY = "metaSeparator"
+        const val DEFAULT_META_START = "#~META~#"
+        const val DATA_START_PROPERTY = "dataSeparator"
+        const val DEFAULT_DATA_START = "#~DATA~#"
+
+        const val code: Int = 0x4446544c //DFTL
+
+        override val name = TAGLESS_ENVELOPE_TYPE.asName()
+
+        override fun invoke(meta: Meta, context: Context): EnvelopeFormat {
+            return TaglessEnvelopeFormat(context.io, meta)
+        }
+
+        val default by lazy { invoke() }
+
+        override fun peekFormat(io: IOPlugin, input: Input): EnvelopeFormat? {
+            return try {
+                val buffer = ByteArray(TAGLESS_ENVELOPE_HEADER.length)
+                input.readFully(buffer)
+                return if (buffer.toString() == TAGLESS_ENVELOPE_HEADER) {
+                    TaglessEnvelopeFormat(io)
+                } else {
+                    null
+                }
+            } catch (ex: Exception) {
+                null
+            }
+        }
+    }
+}

dataforge-io/src/commonMain/kotlin/hep/dataforge/io/functions/FunctionServer.kt

@@ -0,0 +1,75 @@
+package hep.dataforge.io.functions
+
+import hep.dataforge.context.ContextAware
+import hep.dataforge.io.IOFormat
+import hep.dataforge.io.IOPlugin
+import hep.dataforge.meta.Meta
+import hep.dataforge.meta.get
+import hep.dataforge.names.asName
+import hep.dataforge.names.plus
+import kotlin.reflect.KClass
+
+
+/**
+ * A server that could produce asynchronous function values
+ */
+interface FunctionServer : ContextAware {
+    /**
+     * Call a function with given name and descriptor
+     */
+    suspend fun <T : Any, R : Any> call(meta: Meta, arg: T, inputType: KClass<out T>, outputType: KClass<out R>): R
+
+    suspend fun <T : Any, R : Any> callMany(
+        meta: Meta,
+        arg: List<T>,
+        inputType: KClass<out T>,
+        outputType: KClass<out R>
+    ): List<R> = List(arg.size) {
+        call<T, R>(meta, arg[it], inputType, outputType)
+    }
+
+    /**
+     * Get a generic suspended function with given name and descriptor
+     */
+    fun <T : Any, R : Any> function(
+        meta: Meta,
+        inputType: KClass<out T>,
+        outputType: KClass<out R>
+    ): (suspend (T) -> R) = { call(meta, it, inputType, outputType) }
+
+    companion object {
+        const val FUNCTION_NAME_KEY = "function"
+        val FORMAT_KEY = "format".asName()
+        val INPUT_FORMAT_KEY = FORMAT_KEY + "input"
+        val OUTPUT_FORMAT_KEY = FORMAT_KEY + "output"
+    }
+}
+
+suspend inline fun <reified T : Any, reified R : Any> FunctionServer.call(meta: Meta, arg: T) =
+    call(meta, arg, T::class, R::class)
+
+suspend inline fun <reified T : Any, reified R : Any> FunctionServer.callMany(meta: Meta, arg: List<T>) =
+    callMany(meta, arg, T::class, R::class)
+
+inline fun <reified T : Any, reified R : Any> FunctionServer.function(meta: Meta) =
+    function(meta, T::class, R::class)
+
+fun <T : Any> IOPlugin.getInputFormat(meta: Meta, type: KClass<out T>): IOFormat<T> {
+    return meta[FunctionServer.INPUT_FORMAT_KEY]?.let {
+        resolveIOFormat<T>(it, type)
+    } ?: error("Input format not resolved")
+}
+
+fun <R : Any> IOPlugin.getOutputFormat(meta: Meta, type: KClass<out R>): IOFormat<R> {
+    return meta[FunctionServer.OUTPUT_FORMAT_KEY]?.let {
+        resolveIOFormat<R>(it, type)
+    } ?: error("Input format not resolved")
+}
+
+inline fun <reified T : Any> IOPlugin.getInputFormat(meta: Meta): IOFormat<T> =
+    getInputFormat(meta, T::class)
+
+inline fun <reified R : Any> IOPlugin.getOutputFormat(meta: Meta): IOFormat<R> =
+    getOutputFormat(meta, R::class)
+
+

dataforge-io/src/commonMain/kotlin/hep/dataforge/io/functions/RemoteFunctionClient.kt

@@ -0,0 +1,98 @@
+package hep.dataforge.io.functions
+
+import hep.dataforge.context.Context
+import hep.dataforge.context.ContextAware
+import hep.dataforge.io.*
+import hep.dataforge.meta.Meta
+import hep.dataforge.meta.get
+import hep.dataforge.meta.int
+import kotlin.reflect.KClass
+
+class RemoteFunctionClient(override val context: Context, val responder: Responder) : FunctionServer, ContextAware {
+
+    private fun <T : Any> IOPlugin.encodeOne(
+        meta: Meta,
+        value: T,
+        valueType: KClass<out T> = value::class
+    ): Envelope = Envelope.invoke {
+        meta(meta)
+        type = REQUEST_TYPE
+        data {
+            val inputFormat: IOFormat<T> = getInputFormat(meta, valueType)
+            inputFormat.run {
+                writeObject(value)
+            }
+        }
+    }
+
+    private fun <T : Any> IOPlugin.encodeMany(
+        meta: Meta,
+        values: List<T>,
+        valueType: KClass<out T>
+    ): Envelope = Envelope.invoke {
+        meta(meta)
+        type = REQUEST_TYPE
+        meta {
+            SIZE_KEY put values.size
+        }
+        data {
+            val inputFormat: IOFormat<T> = getInputFormat(meta, valueType)
+            inputFormat.run {
+                values.forEach {
+                    writeObject(it)
+                }
+            }
+        }
+    }
+
+    private fun <R : Any> IOPlugin.decode(envelope: Envelope, valueType: KClass<out R>): List<R> {
+        require(envelope.type == RESPONSE_TYPE) { "Unexpected message type: ${envelope.type}" }
+        val size = envelope.meta[SIZE_KEY].int ?: 1
+
+        return if (size == 0) {
+            emptyList()
+        } else {
+            val outputFormat: IOFormat<R> = getOutputFormat(envelope.meta, valueType)
+            envelope.data?.read {
+                List<R>(size) {
+                    outputFormat.run {
+                        readObject()
+                    }
+                }
+            } ?: error("Message does not contain data")
+        }
+    }
+
+    private val plugin by lazy {
+        context.plugins.load(IOPlugin)
+    }
+
+    override suspend fun <T : Any, R : Any> call(
+        meta: Meta,
+        arg: T,
+        inputType: KClass<out T>,
+        outputType: KClass<out R>
+    ): R = plugin.run {
+        val request = encodeOne(meta, arg)
+        val response = responder.respond(request)
+        return decode<R>(response, outputType).first()
+    }
+
+    override suspend fun <T : Any, R : Any> callMany(
+        meta: Meta,
+        arg: List<T>,
+        inputType: KClass<out T>,
+        outputType: KClass<out R>
+    ): List<R> = plugin.run {
+        val request = encodeMany(meta, arg, inputType)
+        val response = responder.respond(request)
+        return decode<R>(response, outputType)
+    }
+
+    companion object {
+        const val REQUEST_TYPE = "function.request"
+        const val RESPONSE_TYPE = "function.response"
+
+        const val SIZE_KEY = "size"
+    }
+}

dataforge-io/src/commonMain/kotlin/hep/dataforge/io/functions/RemoteFunctionServer.kt

@@ -0,0 +1,58 @@
+package hep.dataforge.io.functions
+
+import hep.dataforge.context.Context
+import hep.dataforge.context.ContextAware
+import hep.dataforge.io.Envelope
+import hep.dataforge.io.IOPlugin
+import hep.dataforge.io.Responder
+import hep.dataforge.io.type
+import hep.dataforge.meta.get
+import hep.dataforge.meta.int
+
+class RemoteFunctionServer(
+    override val context: Context,
+    val functionServer: FunctionServer
+) : ContextAware, Responder {
+
+    private val plugin by lazy {
+        context.plugins.load(IOPlugin)
+    }
+
+
+    override suspend fun respond(request: Envelope): Envelope {
+        require(request.type == RemoteFunctionClient.REQUEST_TYPE) { "Unexpected message type: ${request.type}" }
+
+        val inputFormat = plugin.getInputFormat<Any>(request.meta)
+        val outputFormat = plugin.getOutputFormat<Any>(request.meta)
+
+        val size = request.meta[RemoteFunctionClient.SIZE_KEY].int ?: 1
+
+        val input = request.data?.read {
+            inputFormat.run {
+                List(size) {
+                    readObject()
+                }
+            }
+        } ?: error("Input is empty")
+
+        val output = functionServer.callMany<Any, Any>(
+            request.meta,
+            input
+        )
+
+        return Envelope.invoke {
+            meta {
+                meta(request.meta)
+            }
+            type = RemoteFunctionClient.RESPONSE_TYPE
+            data {
+                outputFormat.run {
+                    output.forEach {
+                        writeObject(it)
+                    }
+                }
+            }
+
+        }
+    }
+}

dataforge-io/src/commonMain/kotlin/hep/dataforge/io/serialization/MetaSerializer.kt

@@ -0,0 +1,141 @@
+package hep.dataforge.io.serialization
+
+import hep.dataforge.io.toJson
+import hep.dataforge.io.toMeta
+import hep.dataforge.meta.*
+import hep.dataforge.names.NameToken
+import hep.dataforge.values.*
+import kotlinx.serialization.*
+import kotlinx.serialization.internal.*
+import kotlinx.serialization.json.JsonInput
+import kotlinx.serialization.json.JsonObjectSerializer
+import kotlinx.serialization.json.JsonOutput
+
+
+@Serializer(Value::class)
+object ValueSerializer : KSerializer<Value> {
+    private val valueTypeSerializer = EnumSerializer(ValueType::class)
+    private val listSerializer by lazy { ArrayListSerializer(ValueSerializer) }
+
+    override val descriptor: SerialDescriptor = descriptor("hep.dataforge.values.Value") {
+        boolean("isList")
+        enum<ValueType>("valueType")
+        element("value", null)
+    }
+
+    private fun Decoder.decodeValue(): Value {
+        return when (decode(valueTypeSerializer)) {
+            ValueType.NULL -> Null
+            ValueType.NUMBER -> decodeDouble().asValue() //TODO differentiate?
+            ValueType.BOOLEAN -> decodeBoolean().asValue()
+            ValueType.STRING -> decodeString().asValue()
+            else -> decodeString().parseValue()
+        }
+    }
+
+
+    override fun deserialize(decoder: Decoder): Value {
+        val isList = decoder.decodeBoolean()
+        return if (isList) {
+            listSerializer.deserialize(decoder).asValue()
+        } else {
+            decoder.decodeValue()
+        }
+    }
+
+    private fun Encoder.encodeValue(value: Value) {
+        encode(valueTypeSerializer, value.type)
+        when (value.type) {
+            ValueType.NULL -> {
+                // do nothing
+            }
+            ValueType.NUMBER -> encodeDouble(value.double)
+            ValueType.BOOLEAN -> encodeBoolean(value.boolean)
+            ValueType.STRING -> encodeString(value.string)
+            else -> encodeString(value.string)
+        }
+    }
+
+    override fun serialize(encoder: Encoder, obj: Value) {
+        encoder.encodeBoolean(obj.isList())
+        if (obj.isList()) {
+            listSerializer.serialize(encoder, obj.list)
+        } else {
+            encoder.encodeValue(obj)
+        }
+    }
+}
+
+@Serializer(MetaItem::class)
+object MetaItemSerializer : KSerializer<MetaItem<*>> {
+    override val descriptor: SerialDescriptor = descriptor("MetaItem") {
+        boolean("isNode")
+        element("value", null)
+    }
+
+
+    override fun deserialize(decoder: Decoder): MetaItem<*> {
+        val isNode = decoder.decodeBoolean()
+        return if (isNode) {
+            MetaItem.NodeItem(decoder.decode(MetaSerializer))
+        } else {
+            MetaItem.ValueItem(decoder.decode(ValueSerializer))
+        }
+    }
+
+    override fun serialize(encoder: Encoder, obj: MetaItem<*>) {
+        encoder.encodeBoolean(obj is MetaItem.NodeItem)
+        when (obj) {
+            is MetaItem.NodeItem -> MetaSerializer.serialize(encoder, obj.node)
+            is MetaItem.ValueItem -> ValueSerializer.serialize(encoder, obj.value)
+        }
+    }
+}
+
+private class DeserializedMeta(override val items: Map<NameToken, MetaItem<*>>) : MetaBase()
+
+/**
+ * Serialized for meta
+ */
+@Serializer(Meta::class)
+object MetaSerializer : KSerializer<Meta> {
+    private val mapSerializer = HashMapSerializer(
+        StringSerializer,
+        MetaItemSerializer
+    )
+
+    override val descriptor: SerialDescriptor = NamedMapClassDescriptor(
+        "hep.dataforge.meta.Meta",
+        StringSerializer.descriptor,
+        MetaItemSerializer.descriptor
+    )
+
+    override fun deserialize(decoder: Decoder): Meta {
+        return if (decoder is JsonInput) {
+            JsonObjectSerializer.deserialize(decoder).toMeta()
+        } else {
+            DeserializedMeta(mapSerializer.deserialize(decoder).mapKeys { NameToken(it.key) })
+        }
+    }
+
+    override fun serialize(encoder: Encoder, obj: Meta) {
+        if (encoder is JsonOutput) {
+            JsonObjectSerializer.serialize(encoder, obj.toJson())
+        } else {
+            mapSerializer.serialize(encoder, obj.items.mapKeys { it.key.toString() })
+        }
+    }
+}
+
+@Serializer(Config::class)
+object ConfigSerializer : KSerializer<Config> {
+    override val descriptor: SerialDescriptor = MetaSerializer.descriptor
+
+    override fun deserialize(decoder: Decoder): Config {
+        return MetaSerializer.deserialize(decoder).toConfig()
+    }
+
+    override fun serialize(encoder: Encoder, obj: Config) {
+        MetaSerializer.serialize(encoder, obj)
+    }
+}

dataforge-io/src/commonMain/kotlin/hep/dataforge/io/serialization/nameSerializers.kt

@@ -0,0 +1,33 @@
+package hep.dataforge.io.serialization
+
+import hep.dataforge.names.Name
+import hep.dataforge.names.NameToken
+import hep.dataforge.names.toName
+import kotlinx.serialization.*
+import kotlinx.serialization.internal.StringDescriptor
+
+@Serializer(Name::class)
+object NameSerializer : KSerializer<Name> {
+    override val descriptor: SerialDescriptor = StringDescriptor.withName("Name")
+
+    override fun deserialize(decoder: Decoder): Name {
+        return decoder.decodeString().toName()
+    }
+
+    override fun serialize(encoder: Encoder, obj: Name) {
+        encoder.encodeString(obj.toString())
+    }
+}
+
+@Serializer(NameToken::class)
+object NameTokenSerializer : KSerializer<NameToken> {
+    override val descriptor: SerialDescriptor = StringDescriptor.withName("NameToken")
+
+    override fun deserialize(decoder: Decoder): NameToken {
+        return decoder.decodeString().toName().first()!!
+    }
+
+    override fun serialize(encoder: Encoder, obj: NameToken) {
+        encoder.encodeString(obj.toString())
+    }
+}

dataforge-io/src/commonMain/kotlin/hep/dataforge/io/serialization/serializationUtils.kt

@@ -0,0 +1,80 @@
+package hep.dataforge.io.serialization
+
+import kotlinx.serialization.CompositeDecoder
+import kotlinx.serialization.Decoder
+import kotlinx.serialization.KSerializer
+import kotlinx.serialization.SerialDescriptor
+import kotlinx.serialization.internal.*
+
+/**
+ * A convenience builder for serial descriptors
+ */
+inline class SerialDescriptorBuilder(private val impl: SerialClassDescImpl) {
+    fun element(
+        name: String,
+        descriptor: SerialDescriptor?,
+        isOptional: Boolean = false,
+        vararg annotations: Annotation
+    ) {
+        impl.addElement(name, isOptional)
+        descriptor?.let { impl.pushDescriptor(descriptor) }
+        annotations.forEach {
+            impl.pushAnnotation(it)
+        }
+    }
+
+    fun element(
+        name: String,
+        isOptional: Boolean = false,
+        vararg annotations: Annotation,
+        block: SerialDescriptorBuilder.() -> Unit
+    ) {
+        impl.addElement(name, isOptional)
+        impl.pushDescriptor(SerialDescriptorBuilder(SerialClassDescImpl(name)).apply(block).build())
+        annotations.forEach {
+            impl.pushAnnotation(it)
+        }
+    }
+
+    fun boolean(name: String, isOptional: Boolean = false, vararg annotations: Annotation) =
+        element(name, BooleanDescriptor, isOptional, *annotations)
+
+    fun string(name: String, isOptional: Boolean = false, vararg annotations: Annotation) =
+        element(name, StringDescriptor, isOptional, *annotations)
+
+    fun int(name: String, isOptional: Boolean = false, vararg annotations: Annotation) =
+        element(name, IntDescriptor, isOptional, *annotations)
+
+    fun double(name: String, isOptional: Boolean = false, vararg annotations: Annotation) =
+        element(name, DoubleDescriptor, isOptional, *annotations)
+
+    fun float(name: String, isOptional: Boolean = false, vararg annotations: Annotation) =
+        element(name, FloatDescriptor, isOptional, *annotations)
+
+    fun long(name: String, isOptional: Boolean = false, vararg annotations: Annotation) =
+        element(name, LongDescriptor, isOptional, *annotations)
+
+    fun doubleArray(name: String, isOptional: Boolean = false, vararg annotations: Annotation) =
+        element(name, DoubleArraySerializer.descriptor, isOptional, *annotations)
+
+    inline fun <reified E : Enum<E>> enum(name: String, isOptional: Boolean = false, vararg annotations: Annotation) =
+        element(name, EnumSerializer(E::class).descriptor, isOptional, *annotations)
+
+    fun classAnnotation(a: Annotation) = impl.pushClassAnnotation(a)
+
+    fun build(): SerialDescriptor = impl
+}
+
+inline fun <reified T : Any> KSerializer<T>.descriptor(
+    name: String,
+    block: SerialDescriptorBuilder.() -> Unit
+): SerialDescriptor =
+    SerialDescriptorBuilder(SerialClassDescImpl(name)).apply(block).build()
+
+fun Decoder.decodeStructure(
+    desc: SerialDescriptor,
+    vararg typeParams: KSerializer<*> = emptyArray(),
+    block: CompositeDecoder.() -> Unit
+) {
+    beginStructure(desc, *typeParams).apply(block).endStructure(desc)
+}

dataforge-io/src/commonTest/kotlin/hep/dataforge/io/EnvelopeFormatTest.kt

@@ -0,0 +1,46 @@
+package hep.dataforge.io
+
+import kotlin.test.Test
+import kotlin.test.assertEquals
+
+
+class EnvelopeFormatTest {
+    val envelope = Envelope.invoke {
+        type = "test.format"
+        meta{
+            "d" put 22.2
+        }
+        data{
+            writeDouble(22.2)
+        }
+    }
+
+    @ExperimentalStdlibApi
+    @Test
+    fun testTaggedFormat(){
+        TaggedEnvelopeFormat.default.run {
+            val bytes = writeBytes(envelope)
+            println(bytes.decodeToString())
+            val res = readBytes(bytes)
+            assertEquals(envelope.meta,res.meta)
+            val double = res.data?.read {
+                readDouble()
+            }
+            assertEquals(22.2, double)
+        }
+    }
+
+    @Test
+    fun testTaglessFormat(){
+        TaglessEnvelopeFormat.default.run {
+            val bytes = writeBytes(envelope)
+            println(bytes.decodeToString())
+            val res = readBytes(bytes)
+            assertEquals(envelope.meta,res.meta)
+            val double = res.data?.read {
+                readDouble()
+            }
+            assertEquals(22.2, double)
+        }
+    }
+}

dataforge-io/src/commonTest/kotlin/hep/dataforge/io/MetaFormatTest.kt

@@ -11,11 +11,11 @@ class MetaFormatTest {
     @Test
     fun testBinaryMetaFormat() {
         val meta = buildMeta {
-            "a" to 22
+            "a" put 22
-            "node" to {
+            "node" put {
-                "b" to "DDD"
+                "b" put "DDD"
-                "c" to 11.1
+                "c" put 11.1
-                "array" to doubleArrayOf(1.0, 2.0, 3.0)
+                "array" put doubleArrayOf(1.0, 2.0, 3.0)
             }
         }
         val bytes = meta.toBytes(BinaryMetaFormat)
@@ -26,11 +26,11 @@ class MetaFormatTest {
     @Test
     fun testJsonMetaFormat() {
         val meta = buildMeta {
-            "a" to 22
+            "a" put 22
-            "node" to {
+            "node" put {
-                "b" to "DDD"
+                "b" put "DDD"
-                "c" to 11.1
+                "c" put 11.1
-                "array" to doubleArrayOf(1.0, 2.0, 3.0)
+                "array" put doubleArrayOf(1.0, 2.0, 3.0)
             }
         }
         val string = meta.toString(JsonMetaFormat)

dataforge-io/src/commonTest/kotlin/hep/dataforge/io/MetaSerializerTest.kt

@@ -1,7 +1,13 @@
 package hep.dataforge.io
 
+import hep.dataforge.io.serialization.MetaItemSerializer
+import hep.dataforge.io.serialization.MetaSerializer
+import hep.dataforge.io.serialization.NameSerializer
 import hep.dataforge.meta.buildMeta
 import hep.dataforge.names.toName
+import kotlinx.io.charsets.Charsets
+import kotlinx.io.core.String
+import kotlinx.serialization.cbor.Cbor
 import kotlinx.serialization.json.Json
 import kotlin.test.Test
 import kotlin.test.assertEquals
@@ -10,11 +16,11 @@ class MetaSerializerTest {
     @Test
     fun testMetaSerialization() {
         val meta = buildMeta {
-            "a" to 22
+            "a" put 22
-            "node" to {
+            "node" put {
-                "b" to "DDD"
+                "b" put "DDD"
-                "c" to 11.1
+                "c" put 11.1
-                "array" to doubleArrayOf(1.0, 2.0, 3.0)
+                "array" put doubleArrayOf(1.0, 2.0, 3.0)
             }
         }
 
@@ -23,6 +29,23 @@ class MetaSerializerTest {
         assertEquals(restored, meta)
     }
 
+    @Test
+    fun testCborSerialization() {
+        val meta = buildMeta {
+            "a" put 22
+            "node" put {
+                "b" put "DDD"
+                "c" put 11.1
+                "array" put doubleArrayOf(1.0, 2.0, 3.0)
+            }
+        }
+
+        val bytes = Cbor.dump(MetaSerializer, meta)
+        println(String(bytes, charset = Charsets.ISO_8859_1))
+        val restored = Cbor.load(MetaSerializer, bytes)
+        assertEquals(restored, meta)
+    }
+
     @Test
     fun testNameSerialization() {
         val name = "a.b.c".toName()
@@ -30,4 +53,9 @@ class MetaSerializerTest {
         val restored = Json.plain.parse(NameSerializer, string)
         assertEquals(restored, name)
     }
+
+    @Test
+    fun testMetaItemDescriptor(){
+        val descriptor = MetaItemSerializer.descriptor.getElementDescriptor(0)
+    }
 }

dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileBinary.kt

@@ -1,21 +1,31 @@
 package hep.dataforge.io
 
-import kotlinx.io.core.ByteReadPacket
 import kotlinx.io.core.Input
+import kotlinx.io.core.buildPacket
 import java.nio.channels.FileChannel
 import java.nio.file.Files
 import java.nio.file.Path
 import java.nio.file.StandardOpenOption
+import kotlin.math.min
 
 @ExperimentalUnsignedTypes
 class FileBinary(val path: Path, private val offset: UInt = 0u, size: ULong? = null) : RandomAccessBinary {
 
     override val size: ULong = size ?: (Files.size(path).toULong() - offset).toULong()
 
-    override fun <R> read(from: UInt, size: UInt, block: Input.() -> R): R {
+    init {
-        FileChannel.open(path, StandardOpenOption.READ).use {
+        if( size != null && Files.size(path) < offset.toLong() + size.toLong()){
-            val buffer = it.map(FileChannel.MapMode.READ_ONLY, (from + offset).toLong(), size.toLong())
+            error("Can't read binary from file. File is to short.")
-            return ByteReadPacket(buffer).block()
         }
     }
-}
+
+    override fun <R> read(from: UInt, size: UInt, block: Input.() -> R): R {
+        FileChannel.open(path, StandardOpenOption.READ).use {
+            val theSize: UInt = min(size, Files.size(path).toUInt() - offset)
+            val buffer = it.map(FileChannel.MapMode.READ_ONLY, (from + offset).toLong(), theSize.toLong())
+            return buildPacket { writeFully(buffer) }.block()
+        }
+    }
+}
+
+fun Path.asBinary(offset: UInt = 0u, size: ULong? = null): FileBinary = FileBinary(this, offset, size)

dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileEnvelope.kt

@@ -1,5 +1,6 @@
 package hep.dataforge.io
 
+import hep.dataforge.meta.EmptyMeta
 import hep.dataforge.meta.Meta
 import kotlinx.io.nio.asInput
 import kotlinx.io.nio.asOutput
@@ -14,7 +15,7 @@ class FileEnvelope internal constructor(val path: Path, val format: EnvelopeForm
 
     init {
         val input = Files.newByteChannel(path, StandardOpenOption.READ).asInput()
-        partialEnvelope = format.run { input.readPartial() }
+        partialEnvelope = format.run { input.use { it.readPartial()} }
     }
 
     override val meta: Meta get() = partialEnvelope.meta
@@ -22,22 +23,30 @@ class FileEnvelope internal constructor(val path: Path, val format: EnvelopeForm
     override val data: Binary? = FileBinary(path, partialEnvelope.dataOffset, partialEnvelope.dataSize)
 }
 
-fun Path.readEnvelope(format: EnvelopeFormat) = FileEnvelope(this, format)
+fun IOPlugin.readEnvelopeFile(
+    path: Path,
+    formatFactory: EnvelopeFormatFactory = TaggedEnvelopeFormat,
+    formatMeta: Meta = EmptyMeta
+): FileEnvelope {
+    val format = formatFactory(formatMeta, context)
+    return FileEnvelope(path, format)
+}
 
-fun Path.writeEnvelope(
+fun IOPlugin.writeEnvelopeFile(
+    path: Path,
     envelope: Envelope,
-    format: EnvelopeFormat = TaggedEnvelopeFormat,
+    formatFactory: EnvelopeFormatFactory = TaggedEnvelopeFormat,
-    metaFormat: MetaFormat = JsonMetaFormat
+    formatMeta: Meta = EmptyMeta
 ) {
     val output = Files.newByteChannel(
-        this,
+        path,
         StandardOpenOption.WRITE,
         StandardOpenOption.CREATE,
         StandardOpenOption.TRUNCATE_EXISTING
     ).asOutput()
 
-    with(format) {
+    with(formatFactory(formatMeta, context)) {
-        output.writeEnvelope(envelope, metaFormat)
+        output.writeObject(envelope)
     }
 }
 

dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/ioFormatsJVM.kt

@@ -0,0 +1,51 @@
+package hep.dataforge.io
+
+import hep.dataforge.descriptors.NodeDescriptor
+import hep.dataforge.io.functions.FunctionServer
+import hep.dataforge.io.functions.FunctionServer.Companion.FUNCTION_NAME_KEY
+import hep.dataforge.io.functions.FunctionServer.Companion.INPUT_FORMAT_KEY
+import hep.dataforge.io.functions.FunctionServer.Companion.OUTPUT_FORMAT_KEY
+import hep.dataforge.io.functions.function
+import hep.dataforge.meta.Meta
+import hep.dataforge.meta.buildMeta
+import hep.dataforge.names.Name
+import kotlinx.io.nio.asOutput
+import java.nio.file.Files
+import java.nio.file.Path
+import java.nio.file.StandardOpenOption
+import kotlin.reflect.KClass
+import kotlin.reflect.full.isSuperclassOf
+
+inline fun <reified T : Any> IOPlugin.resolveIOFormat(): IOFormat<T>? {
+    return ioFormats.values.find { it.type.isSuperclassOf(T::class) } as IOFormat<T>?
+}
+
+fun IOPlugin.resolveIOFormatName(type: KClass<*>): Name {
+    return ioFormats.entries.find { it.value.type.isSuperclassOf(type) }?.key
+        ?: error("Can't resolve IOFormat for type $type")
+}
+
+inline fun <reified T : Any, reified R : Any> IOPlugin.generateFunctionMeta(functionName: String): Meta = buildMeta {
+    FUNCTION_NAME_KEY put functionName
+    INPUT_FORMAT_KEY put resolveIOFormatName(T::class).toString()
+    OUTPUT_FORMAT_KEY put resolveIOFormatName(R::class).toString()
+}
+
+inline fun <reified T : Any, reified R : Any> FunctionServer.function(
+    functionName: String
+): (suspend (T) -> R) {
+    val plugin = context.plugins.get<IOPlugin>() ?: error("IO plugin not loaded")
+    val meta = plugin.generateFunctionMeta<T, R>(functionName)
+    return function(meta)
+}
+
+/**
+ * Write meta to file in a given [format]
+ */
+fun Meta.write(path: Path, format: MetaFormat, descriptor: NodeDescriptor? = null) {
+    format.run {
+        Files.newByteChannel(path, StandardOpenOption.WRITE, StandardOpenOption.CREATE_NEW)
+            .asOutput()
+            .writeMeta(this@write, descriptor)
+    }
+}

dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/tcp/EnvelopeClient.kt

@@ -0,0 +1,67 @@
+package hep.dataforge.io.tcp
+
+import hep.dataforge.context.Context
+import hep.dataforge.context.ContextAware
+import hep.dataforge.io.*
+import hep.dataforge.meta.EmptyMeta
+import hep.dataforge.meta.Meta
+import kotlinx.coroutines.asCoroutineDispatcher
+import kotlinx.coroutines.withContext
+import kotlinx.io.streams.writePacket
+import java.net.Socket
+import java.util.concurrent.Executors
+import kotlin.time.ExperimentalTime
+
+@ExperimentalTime
+class EnvelopeClient(
+    override val context: Context,
+    val host: String,
+    val port: Int,
+    formatFactory: EnvelopeFormatFactory = TaggedEnvelopeFormat,
+    formatMeta: Meta = EmptyMeta
+) : Responder, ContextAware {
+
+    private val dispatcher = Executors.newSingleThreadExecutor().asCoroutineDispatcher()
+
+    private val format = formatFactory(formatMeta, context = context)
+
+//    private var socket: SocketChannel? = null
+//
+//    private fun getSocket(): Socket {
+//        val socket = socket ?: Socket(host, port).also { this.socket = it }
+//        return if (socket.isConnected) {
+//            socket
+//        } else {
+//            Socket(host, port).also { this.socket = it }
+//        }
+//    }
+
+    suspend fun close() {
+        try {
+            respond(
+                Envelope.invoke {
+                    type = EnvelopeServer.SHUTDOWN_ENVELOPE_TYPE
+                }
+            )
+        } catch (ex: Exception) {
+            logger.error { ex }
+        }
+    }
+
+
+    override suspend fun respond(request: Envelope): Envelope = withContext(dispatcher) {
+        //val address = InetSocketAddress(host,port)
+        val socket = Socket(host, port)
+        val input = socket.getInputStream().asInput()
+        val output = socket.getOutputStream()
+        format.run {
+            output.writePacket {
+                writeObject(request)
+            }
+            logger.debug { "Sent request with type ${request.type} to ${socket.remoteSocketAddress}" }
+            val res = input.readObject()
+            logger.debug { "Received response with type ${res.type} from ${socket.remoteSocketAddress}" }
+            return@withContext res
+        }
+    }
+}

dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/tcp/EnvelopeServer.kt

@@ -0,0 +1,102 @@
+package hep.dataforge.io.tcp
+
+import hep.dataforge.context.Context
+import hep.dataforge.context.ContextAware
+import hep.dataforge.io.EnvelopeFormatFactory
+import hep.dataforge.io.Responder
+import hep.dataforge.io.TaggedEnvelopeFormat
+import hep.dataforge.io.type
+import hep.dataforge.meta.EmptyMeta
+import hep.dataforge.meta.Meta
+import kotlinx.coroutines.*
+import kotlinx.io.streams.writePacket
+import java.net.ServerSocket
+import java.net.Socket
+import kotlin.concurrent.thread
+
+class EnvelopeServer(
+    override val context: Context,
+    val port: Int,
+    val responder: Responder,
+    val scope: CoroutineScope,
+    formatFactory: EnvelopeFormatFactory = TaggedEnvelopeFormat,
+    formatMeta: Meta = EmptyMeta
+) : ContextAware {
+
+    private var job: Job? = null
+
+    private val format = formatFactory(formatMeta, context = context)
+
+    fun start() {
+        if (job == null) {
+            logger.info { "Starting envelope server on port $port" }
+            val job = scope.launch(Dispatchers.IO) {
+                val serverSocket = ServerSocket(port)
+                //TODO add handshake and format negotiation
+                while (isActive && !serverSocket.isClosed) {
+                    val socket = serverSocket.accept()
+                    logger.info { "Accepted connection from ${socket.remoteSocketAddress}" }
+                    readSocket(socket)
+                }
+            }
+        }
+    }
+
+    fun stop() {
+        logger.info { "Stopping envelope server on port $port" }
+        job?.cancel()
+        job = null
+    }
+
+//    private fun CoroutineScope.readSocket(socket: Socket) {
+//        launch(Dispatchers.IO) {
+//            val input = socket.getInputStream().asInput()
+//            val output = socket.getOutputStream().asOutput()
+//            format.run {
+//                while (isActive && socket.isConnected) {
+//                    val request = input.readThis()
+//                    logger.debug { "Accepted request with type ${request.type} from ${socket.remoteSocketAddress}" }
+//                    if (request.type == SHUTDOWN_ENVELOPE_TYPE) {
+//                        //Echo shutdown command
+//                        logger.info { "Accepted graceful shutdown signal from ${socket.inetAddress}" }
+//                        socket.close()
+//                        cancel("Graceful connection shutdown requested by client")
+//                    }
+//                    val response = responder.respond(request)
+//                    output.writeThis(response)
+//                }
+//            }
+//        }
+//    }
+
+    private fun readSocket(socket: Socket) {
+        thread {
+            val input = socket.getInputStream().asInput()
+            val outputStream = socket.getOutputStream()
+            format.run {
+                while (socket.isConnected) {
+                    val request = input.readObject()
+                    logger.debug { "Accepted request with type ${request.type} from ${socket.remoteSocketAddress}" }
+                    if (request.type == SHUTDOWN_ENVELOPE_TYPE) {
+                        //Echo shutdown command
+                        logger.info { "Accepted graceful shutdown signal from ${socket.inetAddress}" }
+                        socket.close()
+                        return@thread
+//                        cancel("Graceful connection shutdown requested by client")
+                    }
+                    runBlocking {
+                        val response = responder.respond(request)
+                        outputStream.writePacket {
+                            writeObject(response)
+                        }
+                        logger.debug { "Sent response with type ${response.type} to ${socket.remoteSocketAddress}" }
+                    }
+                }
+            }
+        }
+    }
+
+    companion object {
+        const val SHUTDOWN_ENVELOPE_TYPE = "@shutdown"
+    }
+}

dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/tcp/InputStreamAsInput.kt

@@ -0,0 +1,33 @@
+package hep.dataforge.io.tcp
+
+import kotlinx.io.core.AbstractInput
+import kotlinx.io.core.Input
+import kotlinx.io.core.IoBuffer
+import kotlinx.io.core.IoBuffer.Companion.NoPool
+import kotlinx.io.core.writePacket
+import kotlinx.io.streams.readPacketAtMost
+import java.io.InputStream
+
+/**
+ * Modified version of InputStream to Input converter that supports waiting for input
+ */
+internal class InputStreamAsInput(
+    private val stream: InputStream
+) : AbstractInput(pool = NoPool) {
+
+
+    override fun fill(): IoBuffer? {
+        val packet = stream.readPacketAtMost(4096)
+        return pool.borrow().apply {
+            resetForWrite(4096)
+            writePacket(packet)
+        }
+    }
+
+    override fun closeSource() {
+        stream.close()
+    }
+}
+
+fun InputStream.asInput(): Input =
+    InputStreamAsInput(this)

dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/FileBinaryTest.kt

@@ -0,0 +1,56 @@
+package hep.dataforge.io
+
+import hep.dataforge.context.Global
+import java.nio.file.Files
+import kotlin.test.Test
+import kotlin.test.assertEquals
+
+class FileBinaryTest {
+    val envelope = Envelope {
+        meta {
+            "a" put "AAA"
+            "b" put 22.2
+        }
+        dataType = "hep.dataforge.test"
+        dataID = "myData" // добавил только что
+        data {
+            writeDouble(16.7)
+        }
+    }
+
+    @Test
+    fun testSize() {
+        val binary = envelope.data
+        assertEquals(binary?.size?.toInt(), binary?.toBytes()?.size)
+    }
+
+    @Test
+    fun testFileData(){
+        val dataFile = Files.createTempFile("dataforge_test_bin", ".bin")
+        dataFile.toFile().writeText("This is my binary")
+        val envelopeFromFile = Envelope {
+            meta {
+                "a" put "AAA"
+                "b" put 22.2
+            }
+            dataType = "hep.dataforge.satellite"
+            dataID = "cellDepositTest" // добавил только что
+            data = dataFile.asBinary()
+        }
+        val binary = envelopeFromFile.data!!
+        println(binary.toBytes().size)
+        assertEquals(binary.size.toInt(), binary.toBytes().size)
+
+    }
+
+    @Test
+    fun testFileDataSizeRewriting() {
+        println(System.getProperty("user.dir"))
+        val tmpPath = Files.createTempFile("dataforge_test", ".df")
+        Global.io.writeEnvelopeFile(tmpPath, envelope)
+
+        val binary = Global.io.readEnvelopeFile(tmpPath).data!!
+        assertEquals(binary.size.toInt(), binary.toBytes().size)
+    }
+
+}

dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/FileEnvelopeTest.kt

@@ -0,0 +1,31 @@
+package hep.dataforge.io
+
+import hep.dataforge.context.Global
+import java.nio.file.Files
+import kotlin.test.Test
+import kotlin.test.assertTrue
+
+
+class FileEnvelopeTest {
+    val envelope = Envelope {
+        meta {
+            "a" put "AAA"
+            "b" put 22.2
+        }
+        dataType = "hep.dataforge.test"
+        dataID = "myData" // добавил только что
+        data {
+            writeDouble(16.7)
+
+        }
+    }
+
+    @Test
+    fun testFileWriteRead() {
+        val tmpPath = Files.createTempFile("dataforge_test", ".df")
+        Global.io.writeEnvelopeFile(tmpPath,envelope)
+        println(tmpPath.toUri())
+        val restored: Envelope = Global.io.readEnvelopeFile(tmpPath)
+        assertTrue { envelope.contentEquals(restored) }
+    }
+}

dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/tcp/EnvelopeServerTest.kt

@@ -0,0 +1,67 @@
+package hep.dataforge.io.tcp
+
+import hep.dataforge.context.Global
+import hep.dataforge.io.Envelope
+import hep.dataforge.io.Responder
+import hep.dataforge.io.TaggedEnvelopeFormat
+import hep.dataforge.io.writeBytes
+import kotlinx.coroutines.GlobalScope
+import kotlinx.coroutines.runBlocking
+import org.junit.AfterClass
+import org.junit.BeforeClass
+import org.junit.Test
+import kotlin.test.assertEquals
+import kotlin.time.ExperimentalTime
+
+@ExperimentalStdlibApi
+object EchoResponder : Responder {
+    override suspend fun respond(request: Envelope): Envelope {
+        val string = TaggedEnvelopeFormat().run { writeBytes(request).decodeToString() }
+        println("ECHO:")
+        println(string)
+        return request
+    }
+}
+
+@ExperimentalTime
+@ExperimentalStdlibApi
+class EnvelopeServerTest {
+    companion object {
+        @JvmStatic
+        val echoEnvelopeServer = EnvelopeServer(Global, 7778, EchoResponder, GlobalScope)
+
+        @BeforeClass
+        @JvmStatic
+        fun start() {
+            echoEnvelopeServer.start()
+        }
+
+        @AfterClass
+        @JvmStatic
+        fun close() {
+            echoEnvelopeServer.stop()
+        }
+    }
+
+    @Test
+    fun doEchoTest() {
+        val request = Envelope.invoke {
+            type = "test.echo"
+            meta {
+                "test.value" put 22
+            }
+            data {
+                writeDouble(22.7)
+            }
+        }
+        val client = EnvelopeClient(Global, host = "localhost", port = 7778)
+        runBlocking {
+            val response = client.respond(request)
+
+
+            assertEquals(request.meta, response.meta)
+//            assertEquals(request.data?.toBytes()?.decodeToString(), response.data?.toBytes()?.decodeToString())
+            client.close()
+        }
+    }
+}

dataforge-meta/src/commonMain/kotlin/hep/dataforge/descriptors/ItemDescriptor.kt

@@ -32,13 +32,11 @@ sealed class ItemDescriptor(override val config: Config) : Specific {
     var info: String? by string()
 
     /**
-     * A list of tags for this item. Tags used to customize item usage
+     * Additional attributes of an item. For example validation and widget parameters
      *
      * @return
      */
-    var tags: List<String> by value { value ->
+    var attributes by node()
-        value?.list?.map { it.string } ?: emptyList()
-    }
 
     /**
      * True if the item is required
@@ -54,7 +52,7 @@ sealed class ItemDescriptor(override val config: Config) : Specific {
  *
  * @author Alexander Nozik
  */
-class NodeDescriptor(config: Config) : ItemDescriptor(config){
+class NodeDescriptor(config: Config) : ItemDescriptor(config) {
 
     /**
      * True if the node is required
@@ -68,18 +66,18 @@ class NodeDescriptor(config: Config) : ItemDescriptor(config){
      *
      * @return
      */
-    var default: Meta? by node()
+    var default: Config? by node()
 
     /**
      * The list of value descriptors
      */
     val values: Map<String, ValueDescriptor>
-        get() = config.getAll(VALUE_KEY.toName()).entries.associate { (name, node) ->
+        get() = config.getIndexed(VALUE_KEY.toName()).entries.associate { (name, node) ->
             name to ValueDescriptor.wrap(node.node ?: error("Value descriptor must be a node"))
         }
 
     fun value(name: String, descriptor: ValueDescriptor) {
-        if(items.keys.contains(name)) error("The key $name already exists in descriptor")
+        if (items.keys.contains(name)) error("The key $name already exists in descriptor")
         val token = NameToken(VALUE_KEY, name)
         config[token] = descriptor.config
     }
@@ -95,13 +93,13 @@ class NodeDescriptor(config: Config) : ItemDescriptor(config){
      * The map of children node descriptors
      */
     val nodes: Map<String, NodeDescriptor>
-        get() = config.getAll(NODE_KEY.toName()).entries.associate { (name, node) ->
+        get() = config.getIndexed(NODE_KEY.toName()).entries.associate { (name, node) ->
             name to wrap(node.node ?: error("Node descriptor must be a node"))
         }
 
 
     fun node(name: String, descriptor: NodeDescriptor) {
-        if(items.keys.contains(name)) error("The key $name already exists in descriptor")
+        if (items.keys.contains(name)) error("The key $name already exists in descriptor")
         val token = NameToken(NODE_KEY, name)
         config[token] = descriptor.config
     }
@@ -117,7 +115,7 @@ class NodeDescriptor(config: Config) : ItemDescriptor(config){
 
     companion object : Specification<NodeDescriptor> {
 
-//        const val ITEM_KEY = "item"
+        //        const val ITEM_KEY = "item"
         const val NODE_KEY = "node"
         const val VALUE_KEY = "value"
 
@@ -135,7 +133,7 @@ class NodeDescriptor(config: Config) : ItemDescriptor(config){
  *
  * @author Alexander Nozik
  */
-class ValueDescriptor(config: Config) : ItemDescriptor(config){
+class ValueDescriptor(config: Config) : ItemDescriptor(config) {
 
 
     /**

dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Laminate.kt

@@ -4,8 +4,6 @@ import hep.dataforge.names.NameToken
 
 /**
  * A meta laminate consisting of multiple immutable meta layers. For mutable front layer, use [Styled].
- *
- *
  */
 class Laminate(layers: List<Meta>) : MetaBase() {
 
@@ -63,7 +61,7 @@ class Laminate(layers: List<Meta>) : MetaBase() {
                 }
                 else -> map {
                     when (it) {
-                        is MetaItem.ValueItem -> MetaItem.NodeItem(buildMeta { Meta.VALUE_KEY to it.value })
+                        is MetaItem.ValueItem -> MetaItem.NodeItem(buildMeta { Meta.VALUE_KEY put it.value })
                         is MetaItem.NodeItem -> it
                     }
                 }.merge()

dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Meta.kt

@@ -59,6 +59,8 @@ interface Meta : MetaRepr {
          * A key for single value node
          */
         const val VALUE_KEY = "@value"
+
+        val empty: EmptyMeta = EmptyMeta
     }
 }
 
@@ -78,27 +80,6 @@ operator fun Meta?.get(name: Name): MetaItem<*>? {
 operator fun Meta?.get(token: NameToken): MetaItem<*>? = this?.items?.get(token)
 operator fun Meta?.get(key: String): MetaItem<*>? = get(key.toName())
 
-/**
- * Get all items matching given name.
- */
-fun Meta.getAll(name: Name): Map<String, MetaItem<*>> {
-    val root = when (name.length) {
-        0 -> error("Can't use empty name for that")
-        1 -> this
-        else -> (this[name.cutLast()] as? NodeItem<*>)?.node
-    }
-
-    val (body, index) = name.last()!!
-    val regex = index.toRegex()
-
-    return root?.items
-        ?.filter { it.key.body == body && (index.isEmpty() || regex.matches(it.key.index)) }
-        ?.mapKeys { it.key.index }
-        ?: emptyMap()
-}
-
-fun Meta.getAll(name: String): Map<String, MetaItem<*>> = getAll(name.toName())
-
 /**
  * Get a sequence of [Name]-[Value] pairs
  */
@@ -136,27 +117,6 @@ interface MetaNode<M : MetaNode<M>> : Meta {
     override val items: Map<NameToken, MetaItem<M>>
 }
 
-/**
- * Get all items matching given name.
- */
-fun <M : MetaNode<M>> M.getAll(name: Name): Map<String, MetaItem<M>> {
-    val root: MetaNode<M>? = when (name.length) {
-        0 -> error("Can't use empty name for that")
-        1 -> this
-        else -> (this[name.cutLast()] as? NodeItem<M>)?.node
-    }
-
-    val (body, index) = name.last()!!
-    val regex = index.toRegex()
-
-    return root?.items
-        ?.filter { it.key.body == body && (index.isEmpty() || regex.matches(it.key.index)) }
-        ?.mapKeys { it.key.index }
-        ?: emptyMap()
-}
-
-fun <M : MetaNode<M>> M.getAll(name: String): Map<String, MetaItem<M>> = getAll(name.toName())
-
 operator fun <M : MetaNode<M>> MetaNode<M>?.get(name: Name): MetaItem<M>? {
     if (this == null) return null
     return name.first()?.let { token ->
@@ -183,20 +143,15 @@ operator fun <M : MetaNode<M>> MetaNode<M>?.get(key: NameToken): MetaItem<M>? =
 /**
  * Equals, hashcode and to string for any meta
  */
-abstract class MetaBase: Meta{
+abstract class MetaBase : Meta {
 
-    override fun equals(other: Any?): Boolean  = if(other is Meta) {
+    override fun equals(other: Any?): Boolean = if (other is Meta) {
         this.items == other.items
-//        val items = items
-//        val otherItems = other.items
-//        (items.keys == otherItems.keys) && items.keys.all {
-//            items[it] == otherItems[it]
-//        }
     } else {
         false
     }
 
-    override fun hashCode(): Int  = items.hashCode()
+    override fun hashCode(): Int = items.hashCode()
 
     override fun toString(): String = items.toString()
 }
@@ -232,8 +187,7 @@ object EmptyMeta : MetaBase() {
 /**
  * Unsafe methods to access values and nodes directly from [MetaItem]
  */
-
+val MetaItem<*>?.value: Value?
-val MetaItem<*>?.value
     get() = (this as? ValueItem)?.value
         ?: (this?.node?.get(VALUE_KEY) as? ValueItem)?.value
 
@@ -247,7 +201,7 @@ val MetaItem<*>?.long get() = number?.toLong()
 val MetaItem<*>?.short get() = number?.toShort()
 
 inline fun <reified E : Enum<E>> MetaItem<*>?.enum() = if (this is ValueItem && this.value is EnumValue<*>) {
-    this.value as E
+    this.value.value as E
 } else {
     string?.let { enumValueOf<E>(it) }
 }
@@ -257,17 +211,8 @@ val MetaItem<*>?.stringList get() = value?.list?.map { it.string } ?: emptyList(
 val <M : Meta> MetaItem<M>?.node: M?
     get() = when (this) {
         null -> null
-        is ValueItem -> error("Trying to interpret value meta item as node item")
+        is ValueItem -> null//error("Trying to interpret value meta item as node item")
         is NodeItem -> node
     }
 
-/**
+fun Meta.isEmpty() = this === EmptyMeta || this.items.isEmpty()
- * Generic meta-holder object
- */
-interface Metoid {
-    val meta: Meta
-}
-
-fun Value.toMeta() = buildMeta { Meta.VALUE_KEY to this }
-
-fun Meta.isEmpty() = this === EmptyMeta || this.items.isEmpty()

dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MetaBuilder.kt

@@ -2,50 +2,118 @@ package hep.dataforge.meta
 
 import hep.dataforge.names.Name
 import hep.dataforge.names.asName
+import hep.dataforge.values.EnumValue
 import hep.dataforge.values.Value
+import hep.dataforge.values.asValue
+import kotlin.jvm.JvmName
 
 /**
  * DSL builder for meta. Is not intended to store mutable state
  */
+@DFBuilder
 class MetaBuilder : AbstractMutableMeta<MetaBuilder>() {
     override fun wrapNode(meta: Meta): MetaBuilder = if (meta is MetaBuilder) meta else meta.builder()
     override fun empty(): MetaBuilder = MetaBuilder()
 
-    infix fun String.to(value: Any) {
+    infix fun String.put(value: Value?) {
-        if (value is Meta) {
+        set(this, value)
-            this@MetaBuilder[this] = value
-        }
-        this@MetaBuilder[this] = Value.of(value)
     }
 
-    infix fun String.to(meta: Meta) {
+    infix fun String.put(string: String?) {
+        set(this, string?.asValue())
+    }
+
+    infix fun String.put(number: Number?) {
+        set(this, number?.asValue())
+    }
+
+    infix fun String.put(boolean: Boolean?) {
+        set(this, boolean?.asValue())
+    }
+
+    infix fun String.put(enum: Enum<*>) {
+        set(this, EnumValue(enum))
+    }
+
+    @JvmName("putValues")
+    infix fun String.put(iterable: Iterable<Value>) {
+        set(this, iterable.asValue())
+    }
+
+    @JvmName("putNumbers")
+    infix fun String.put(iterable: Iterable<Number>) {
+        set(this, iterable.map { it.asValue() }.asValue())
+    }
+
+    @JvmName("putStrings")
+    infix fun String.put(iterable: Iterable<String>) {
+        set(this, iterable.map { it.asValue() }.asValue())
+    }
+
+    infix fun String.put(array: DoubleArray) {
+        set(this, array.asValue())
+    }
+
+    infix fun String.putValue(any: Any?) {
+        set(this, Value.of(any))
+    }
+
+    infix fun String.put(meta: Meta?) {
         this@MetaBuilder[this] = meta
     }
 
-    infix fun String.to(value: Iterable<Meta>) {
+    infix fun String.put(repr: MetaRepr?) {
+        set(this, repr?.toMeta())
+    }
+
+    @JvmName("putMetas")
+    infix fun String.put(value: Iterable<Meta>) {
         this@MetaBuilder[this] = value.toList()
     }
 
-    infix fun String.to(metaBuilder: MetaBuilder.() -> Unit) {
+    infix fun String.put(metaBuilder: MetaBuilder.() -> Unit) {
         this@MetaBuilder[this] = MetaBuilder().apply(metaBuilder)
     }
 
-    infix fun Name.to(value: Any) {
+    infix fun Name.put(value: Value?) {
-        if (value is Meta) {
+        set(this, value)
-            this@MetaBuilder[this] = value
-        }
-        this@MetaBuilder[this] = Value.of(value)
     }
 
-    infix fun Name.to(meta: Meta) {
+    infix fun Name.put(string: String?) {
+        set(this, string?.asValue())
+    }
+
+    infix fun Name.put(number: Number?) {
+        set(this, number?.asValue())
+    }
+
+    infix fun Name.put(boolean: Boolean?) {
+        set(this, boolean?.asValue())
+    }
+
+    infix fun Name.put(enum: Enum<*>) {
+        set(this, EnumValue(enum))
+    }
+
+    @JvmName("putValues")
+    infix fun Name.put(iterable: Iterable<Value>) {
+        set(this, iterable.asValue())
+    }
+
+    infix fun Name.put(meta: Meta?) {
         this@MetaBuilder[this] = meta
     }
 
-    infix fun Name.to(value: Iterable<Meta>) {
+    infix fun Name.put(repr: MetaRepr?) {
+        set(this, repr?.toMeta())
+    }
+
+    @JvmName("putMetas")
+    infix fun Name.put(value: Iterable<Meta>) {
         this@MetaBuilder[this] = value.toList()
     }
 
-    infix fun Name.to(metaBuilder: MetaBuilder.() -> Unit) {
+    infix fun Name.put(metaBuilder: MetaBuilder.() -> Unit) {
         this@MetaBuilder[this] = MetaBuilder().apply(metaBuilder)
     }
 }
@@ -65,6 +133,11 @@ fun Meta.builder(): MetaBuilder {
     }
 }
 
+/**
+ * Create a deep copy of this meta and apply builder to it
+ */
+fun Meta.edit(builder: MetaBuilder.() -> Unit): MetaBuilder = builder().apply(builder)
+
 /**
  * Build a [MetaBuilder] using given transformation
  */

dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MutableMeta.kt

@@ -164,7 +164,7 @@ fun MutableMeta<*>.append(name: Name, value: Any?) {
     if (newIndex.isNotEmpty()) {
         set(name, value)
     } else {
-        val index = (getAll(name).keys.mapNotNull { it.toIntOrNull() }.max() ?: -1) + 1
+        val index = (getIndexed(name).keys.mapNotNull { it.toIntOrNull() }.max() ?: -1) + 1
         set(name.withIndex(index.toString()), value)
     }
 }

dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Specific.kt

@@ -1,5 +1,8 @@
 package hep.dataforge.meta
 
+import hep.dataforge.names.Name
+import kotlin.jvm.JvmName
+
 /**
  * Marker interface for classes with specifications
  */
@@ -61,5 +64,10 @@ fun <C : Specific, S : Specification<C>> S.createStyle(action: C.() -> Unit): Me
 
 fun <C : Specific> Specific.spec(
     spec: Specification<C>,
-    key: String? = null
+    key: Name? = null
-): MutableMorphDelegate<Config, C> = MutableMorphDelegate(config, key) { spec.wrap(it) }
+): MutableMorphDelegate<Config, C> = MutableMorphDelegate(config, key) { spec.wrap(it) }
+
+fun <T: Specific> MetaItem<*>.spec(spec: Specification<T>): T? = node?.let { spec.wrap(it)}
+
+@JvmName("configSpec")
+fun <T: Specific> MetaItem<Config>.spec(spec: Specification<T>): T? = node?.let { spec.wrap(it)}

dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/annotations.kt

@@ -0,0 +1,11 @@
+package hep.dataforge.meta
+
+/**
+ * General marker for dataforge builders
+ */
+@DslMarker
+annotation class DFBuilder
+
+@Experimental(level = Experimental.Level.WARNING)
+@Retention(AnnotationRetention.BINARY)
+annotation class DFExperimental

dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/configDelegates.kt

@@ -1,5 +1,6 @@
 package hep.dataforge.meta
 
+import hep.dataforge.names.Name
 import hep.dataforge.values.DoubleArrayValue
 import hep.dataforge.values.Null
 import hep.dataforge.values.Value
@@ -11,126 +12,126 @@ import kotlin.jvm.JvmName
 /**
  * A property delegate that uses custom key
  */
-fun Configurable.value(default: Any = Null, key: String? = null): MutableValueDelegate<Config> =
+fun Configurable.value(default: Any = Null, key: Name? = null): MutableValueDelegate<Config> =
     MutableValueDelegate(config, key, Value.of(default))
 
 fun <T> Configurable.value(
     default: T? = null,
-    key: String? = null,
+    key: Name? = null,
     writer: (T) -> Value = { Value.of(it) },
     reader: (Value?) -> T
 ): ReadWriteDelegateWrapper<Value?, T> =
     MutableValueDelegate(config, key, default?.let { Value.of(it) }).transform(reader = reader, writer = writer)
 
-fun Configurable.string(default: String? = null, key: String? = null): MutableStringDelegate<Config> =
+fun Configurable.string(default: String? = null, key: Name? = null): MutableStringDelegate<Config> =
     MutableStringDelegate(config, key, default)
 
-fun Configurable.boolean(default: Boolean? = null, key: String? = null): MutableBooleanDelegate<Config> =
+fun Configurable.boolean(default: Boolean? = null, key: Name? = null): MutableBooleanDelegate<Config> =
     MutableBooleanDelegate(config, key, default)
 
-fun Configurable.number(default: Number? = null, key: String? = null): MutableNumberDelegate<Config> =
+fun Configurable.number(default: Number? = null, key: Name? = null): MutableNumberDelegate<Config> =
     MutableNumberDelegate(config, key, default)
 
 /* Number delegates*/
 
-fun Configurable.int(default: Int? = null, key: String? = null) =
+fun Configurable.int(default: Int? = null, key: Name? = null) =
     number(default, key).int
 
-fun Configurable.double(default: Double? = null, key: String? = null) =
+fun Configurable.double(default: Double? = null, key: Name? = null) =
     number(default, key).double
 
-fun Configurable.long(default: Long? = null, key: String? = null) =
+fun Configurable.long(default: Long? = null, key: Name? = null) =
     number(default, key).long
 
-fun Configurable.short(default: Short? = null, key: String? = null) =
+fun Configurable.short(default: Short? = null, key: Name? = null) =
     number(default, key).short
 
-fun Configurable.float(default: Float? = null, key: String? = null) =
+fun Configurable.float(default: Float? = null, key: Name? = null) =
     number(default, key).float
 
 
 @JvmName("safeString")
-fun Configurable.string(default: String, key: String? = null) =
+fun Configurable.string(default: String, key: Name? = null) =
     MutableSafeStringDelegate(config, key) { default }
 
 @JvmName("safeBoolean")
-fun Configurable.boolean(default: Boolean, key: String? = null) =
+fun Configurable.boolean(default: Boolean, key: Name? = null) =
     MutableSafeBooleanDelegate(config, key) { default }
 
 @JvmName("safeNumber")
-fun Configurable.number(default: Number, key: String? = null) =
+fun Configurable.number(default: Number, key: Name? = null) =
     MutableSafeNumberDelegate(config, key) { default }
 
 @JvmName("safeString")
-fun Configurable.string(key: String? = null, default: () -> String) =
+fun Configurable.string(key: Name? = null, default: () -> String) =
     MutableSafeStringDelegate(config, key, default)
 
 @JvmName("safeBoolean")
-fun Configurable.boolean(key: String? = null, default: () -> Boolean) =
+fun Configurable.boolean(key: Name? = null, default: () -> Boolean) =
     MutableSafeBooleanDelegate(config, key, default)
 
 @JvmName("safeNumber")
-fun Configurable.number(key: String? = null, default: () -> Number) =
+fun Configurable.number(key: Name? = null, default: () -> Number) =
     MutableSafeNumberDelegate(config, key, default)
 
 
 /* Safe number delegates*/
 
 @JvmName("safeInt")
-fun Configurable.int(default: Int, key: String? = null) =
+fun Configurable.int(default: Int, key: Name? = null) =
     number(default, key).int
 
 @JvmName("safeDouble")
-fun Configurable.double(default: Double, key: String? = null) =
+fun Configurable.double(default: Double, key: Name? = null) =
     number(default, key).double
 
 @JvmName("safeLong")
-fun Configurable.long(default: Long, key: String? = null) =
+fun Configurable.long(default: Long, key: Name? = null) =
     number(default, key).long
 
 @JvmName("safeShort")
-fun Configurable.short(default: Short, key: String? = null) =
+fun Configurable.short(default: Short, key: Name? = null) =
     number(default, key).short
 
 @JvmName("safeFloat")
-fun Configurable.float(default: Float, key: String? = null) =
+fun Configurable.float(default: Float, key: Name? = null) =
     number(default, key).float
 
 /**
  * Enum delegate
  */
-inline fun <reified E : Enum<E>> Configurable.enum(default: E, key: String? = null) =
+inline fun <reified E : Enum<E>> Configurable.enum(default: E, key: Name? = null) =
     MutableSafeEnumvDelegate(config, key, default) { enumValueOf(it) }
 
 /* Node delegates */
 
-fun Configurable.node(key: String? = null) = MutableNodeDelegate(config, key)
+fun Configurable.node(key: Name? = null): MutableNodeDelegate<Config> = MutableNodeDelegate(config, key)
 
-fun <T : Specific> Configurable.spec(spec: Specification<T>, key: String? = null) =
+fun <T : Specific> Configurable.spec(spec: Specification<T>, key: Name? = null) =
     MutableMorphDelegate(config, key) { spec.wrap(it) }
 
-fun <T : Specific> Configurable.spec(builder: (Config) -> T, key: String? = null) =
+fun <T : Specific> Configurable.spec(builder: (Config) -> T, key: Name? = null) =
     MutableMorphDelegate(config, key) { specification(builder).wrap(it) }
 
 /*
  * Extra delegates for special cases
  */
 
-fun Configurable.stringList(key: String? = null): ReadWriteDelegateWrapper<Value?, List<String>> =
+fun Configurable.stringList(vararg strings: String, key: Name? = null): ReadWriteDelegateWrapper<Value?, List<String>> =
-    value(emptyList(), key) { it?.list?.map { value -> value.string } ?: emptyList() }
+    value(strings.asList(), key) { it?.list?.map { value -> value.string } ?: emptyList() }
 
-fun Configurable.numberList(key: String? = null): ReadWriteDelegateWrapper<Value?, List<Number>> =
+fun Configurable.numberList(vararg numbers: Number, key: Name? = null): ReadWriteDelegateWrapper<Value?, List<Number>> =
-    value(emptyList(), key) { it?.list?.map { value -> value.number } ?: emptyList() }
+    value(numbers.asList(), key) { it?.list?.map { value -> value.number } ?: emptyList() }
 
 /**
  * A special delegate for double arrays
  */
-fun Configurable.doubleArray(key: String? = null): ReadWriteDelegateWrapper<Value?, DoubleArray> =
+fun Configurable.doubleArray(key: Name? = null): ReadWriteDelegateWrapper<Value?, DoubleArray> =
     value(doubleArrayOf(), key) {
         (it as? DoubleArrayValue)?.value
             ?: it?.list?.map { value -> value.number.toDouble() }?.toDoubleArray()
             ?: doubleArrayOf()
     }
 
-fun <T : Configurable> Configurable.child(key: String? = null, converter: (Meta) -> T) =
+fun <T : Configurable> Configurable.node(key: Name? = null, converter: (Meta) -> T) =
     MutableMorphDelegate(config, key, converter)

dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/mapMeta.kt

@@ -0,0 +1,39 @@
+package hep.dataforge.meta
+
+import hep.dataforge.descriptors.NodeDescriptor
+import hep.dataforge.values.Value
+
+///**
+// * Find all elements with given body
+// */
+//private fun Meta.byBody(body: String): Map<String, MetaItem<*>> =
+//    items.filter { it.key.body == body }.mapKeys { it.key.index }
+//
+//private fun Meta.distinctNames() = items.keys.map { it.body }.distinct()
+
+/**
+ * Convert meta to map of maps
+ */
+@DFExperimental
+fun Meta.toMap(descriptor: NodeDescriptor? = null): Map<String, Any?> {
+    return items.entries.associate { (token, item) ->
+        token.toString() to when (item) {
+            is MetaItem.NodeItem -> item.node.toMap()
+            is MetaItem.ValueItem -> item.value.value
+        }
+    }
+}
+
+/**
+ * Convert map of maps to meta
+ */
+@DFExperimental
+fun Map<String, Any?>.toMeta(descriptor: NodeDescriptor? = null): Meta = buildMeta {
+    entries.forEach { (key, value) ->
+        @Suppress("UNCHECKED_CAST")
+        when (value) {
+            is Map<*, *> -> setNode(key, (value as Map<String, Any?>).toMeta())
+            else -> setValue(key, Value.of(value))
+        }
+    }
+}

dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/metaDelegates.kt

@@ -1,5 +1,7 @@
 package hep.dataforge.meta
 
+import hep.dataforge.names.Name
+import hep.dataforge.names.asName
 import hep.dataforge.values.Null
 import hep.dataforge.values.Value
 import hep.dataforge.values.asValue
@@ -26,15 +28,21 @@ class StringDelegate(val meta: Meta, private val key: String? = null, private va
     }
 }
 
-class BooleanDelegate(val meta: Meta, private val key: String? = null, private val default: Boolean? = null) :
+class BooleanDelegate(
-    ReadOnlyProperty<Metoid, Boolean?> {
+    val meta: Meta,
-    override fun getValue(thisRef: Metoid, property: KProperty<*>): Boolean? {
+    private val key: String? = null,
+    private val default: Boolean? = null
+) : ReadOnlyProperty<Any?, Boolean?> {
+    override fun getValue(thisRef: Any?, property: KProperty<*>): Boolean? {
         return meta[key ?: property.name]?.boolean ?: default
     }
 }
 
-class NumberDelegate(val meta: Meta, private val key: String? = null, private val default: Number? = null) :
+class NumberDelegate(
-    ReadOnlyProperty<Any?, Number?> {
+    val meta: Meta,
+    private val key: String? = null,
+    private val default: Number? = null
+) : ReadOnlyProperty<Any?, Number?> {
     override fun getValue(thisRef: Any?, property: KProperty<*>): Number? {
         return meta[key ?: property.name]?.number ?: default
     }
@@ -136,7 +144,7 @@ fun Meta.boolean(default: Boolean? = null, key: String? = null) = BooleanDelegat
 
 fun Meta.number(default: Number? = null, key: String? = null) = NumberDelegate(this, key, default)
 
-fun Meta.child(key: String? = null) = ChildDelegate(this, key) { it }
+fun Meta.node(key: String? = null) = ChildDelegate(this, key) { it }
 
 @JvmName("safeString")
 fun Meta.string(default: String, key: String? = null) =
@@ -166,22 +174,19 @@ fun Meta.number(key: String? = null, default: () -> Number) =
 inline fun <reified E : Enum<E>> Meta.enum(default: E, key: String? = null) =
     SafeEnumDelegate(this, key, default) { enumValueOf(it) }
 
-
-fun <T : Metoid> Metoid.child(key: String? = null, converter: (Meta) -> T) = ChildDelegate(meta, key, converter)
-
 /* Read-write delegates */
 
 class MutableValueDelegate<M : MutableMeta<M>>(
     val meta: M,
-    private val key: String? = null,
+    private val key: Name? = null,
     private val default: Value? = null
 ) : ReadWriteProperty<Any?, Value?> {
     override fun getValue(thisRef: Any?, property: KProperty<*>): Value? {
-        return meta[key ?: property.name]?.value ?: default
+        return meta[key ?: property.name.asName()]?.value ?: default
     }
 
     override fun setValue(thisRef: Any?, property: KProperty<*>, value: Value?) {
-        val name = key ?: property.name
+        val name = key ?: property.name.asName()
         if (value == null) {
             meta.remove(name)
         } else {
@@ -195,15 +200,15 @@ class MutableValueDelegate<M : MutableMeta<M>>(
 
 class MutableStringDelegate<M : MutableMeta<M>>(
     val meta: M,
-    private val key: String? = null,
+    private val key: Name? = null,
     private val default: String? = null
 ) : ReadWriteProperty<Any?, String?> {
     override fun getValue(thisRef: Any?, property: KProperty<*>): String? {
-        return meta[key ?: property.name]?.string ?: default
+        return meta[key ?: property.name.asName()]?.string ?: default
     }
 
     override fun setValue(thisRef: Any?, property: KProperty<*>, value: String?) {
-        val name = key ?: property.name
+        val name = key ?: property.name.asName()
         if (value == null) {
             meta.remove(name)
         } else {
@@ -214,15 +219,15 @@ class MutableStringDelegate<M : MutableMeta<M>>(
 
 class MutableBooleanDelegate<M : MutableMeta<M>>(
     val meta: M,
-    private val key: String? = null,
+    private val key: Name? = null,
     private val default: Boolean? = null
 ) : ReadWriteProperty<Any?, Boolean?> {
     override fun getValue(thisRef: Any?, property: KProperty<*>): Boolean? {
-        return meta[key ?: property.name]?.boolean ?: default
+        return meta[key ?: property.name.asName()]?.boolean ?: default
     }
 
     override fun setValue(thisRef: Any?, property: KProperty<*>, value: Boolean?) {
-        val name = key ?: property.name
+        val name = key ?: property.name.asName()
         if (value == null) {
             meta.remove(name)
         } else {
@@ -233,15 +238,15 @@ class MutableBooleanDelegate<M : MutableMeta<M>>(
 
 class MutableNumberDelegate<M : MutableMeta<M>>(
     val meta: M,
-    private val key: String? = null,
+    private val key: Name? = null,
     private val default: Number? = null
 ) : ReadWriteProperty<Any?, Number?> {
     override fun getValue(thisRef: Any?, property: KProperty<*>): Number? {
-        return meta[key ?: property.name]?.number ?: default
+        return meta[key ?: property.name.asName()]?.number ?: default
     }
 
     override fun setValue(thisRef: Any?, property: KProperty<*>, value: Number?) {
-        val name = key ?: property.name
+        val name = key ?: property.name.asName()
         if (value == null) {
             meta.remove(name)
         } else {
@@ -260,52 +265,52 @@ class MutableNumberDelegate<M : MutableMeta<M>>(
 
 class MutableSafeStringDelegate<M : MutableMeta<M>>(
     val meta: M,
-    private val key: String? = null,
+    private val key: Name? = null,
     default: () -> String
 ) : ReadWriteProperty<Any?, String> {
 
     private val default: String by lazy(default)
 
     override fun getValue(thisRef: Any?, property: KProperty<*>): String {
-        return meta[key ?: property.name]?.string ?: default
+        return meta[key ?: property.name.asName()]?.string ?: default
     }
 
     override fun setValue(thisRef: Any?, property: KProperty<*>, value: String) {
-        meta.setValue(key ?: property.name, value.asValue())
+        meta.setValue(key ?: property.name.asName(), value.asValue())
     }
 }
 
 class MutableSafeBooleanDelegate<M : MutableMeta<M>>(
     val meta: M,
-    private val key: String? = null,
+    private val key: Name? = null,
     default: () -> Boolean
 ) : ReadWriteProperty<Any?, Boolean> {
 
     private val default: Boolean by lazy(default)
 
     override fun getValue(thisRef: Any?, property: KProperty<*>): Boolean {
-        return meta[key ?: property.name]?.boolean ?: default
+        return meta[key ?: property.name.asName()]?.boolean ?: default
     }
 
     override fun setValue(thisRef: Any?, property: KProperty<*>, value: Boolean) {
-        meta.setValue(key ?: property.name, value.asValue())
+        meta.setValue(key ?: property.name.asName(), value.asValue())
     }
 }
 
 class MutableSafeNumberDelegate<M : MutableMeta<M>>(
     val meta: M,
-    private val key: String? = null,
+    private val key: Name? = null,
     default: () -> Number
 ) : ReadWriteProperty<Any?, Number> {
 
     private val default: Number by lazy(default)
 
     override fun getValue(thisRef: Any?, property: KProperty<*>): Number {
-        return meta[key ?: property.name]?.number ?: default
+        return meta[key ?: property.name.asName()]?.number ?: default
     }
 
     override fun setValue(thisRef: Any?, property: KProperty<*>, value: Number) {
-        meta.setValue(key ?: property.name, value.asValue())
+        meta.setValue(key ?: property.name.asName(), value.asValue())
     }
 
     val double get() = ReadWriteDelegateWrapper(this, reader = { it.toDouble() }, writer = { it })
@@ -317,16 +322,16 @@ class MutableSafeNumberDelegate<M : MutableMeta<M>>(
 
 class MutableSafeEnumvDelegate<M : MutableMeta<M>, E : Enum<E>>(
     val meta: M,
-    private val key: String? = null,
+    private val key: Name? = null,
     private val default: E,
     private val resolver: (String) -> E
 ) : ReadWriteProperty<Any?, E> {
     override fun getValue(thisRef: Any?, property: KProperty<*>): E {
-        return (meta[key ?: property.name]?.string)?.let { resolver(it) } ?: default
+        return (meta[key ?: property.name.asName()]?.string)?.let { resolver(it) } ?: default
     }
 
     override fun setValue(thisRef: Any?, property: KProperty<*>, value: E) {
-        meta.setValue(key ?: property.name, value.name.asValue())
+        meta.setValue(key ?: property.name.asName(), value.name.asValue())
     }
 }
 
@@ -334,31 +339,31 @@ class MutableSafeEnumvDelegate<M : MutableMeta<M>, E : Enum<E>>(
 
 class MutableNodeDelegate<M : MutableMeta<M>>(
     val meta: M,
-    private val key: String? = null
+    private val key: Name? = null
-) : ReadWriteProperty<Any?, Meta?> {
+) : ReadWriteProperty<Any?, M?> {
-    override fun getValue(thisRef: Any?, property: KProperty<*>): Meta? {
+    override fun getValue(thisRef: Any?, property: KProperty<*>): M? {
-        return meta[key ?: property.name]?.node
+        return meta[key ?: property.name.asName()]?.node
     }
 
-    override fun setValue(thisRef: Any?, property: KProperty<*>, value: Meta?) {
+    override fun setValue(thisRef: Any?, property: KProperty<*>, value: M?) {
-        meta[key ?: property.name] = value
+        meta[key ?: property.name.asName()] = value
     }
 }
 
 class MutableMorphDelegate<M : MutableMeta<M>, T : Configurable>(
     val meta: M,
-    private val key: String? = null,
+    private val key: Name? = null,
     private val converter: (Meta) -> T
 ) : ReadWriteProperty<Any?, T?> {
     override fun getValue(thisRef: Any?, property: KProperty<*>): T? {
-        return meta[key ?: property.name]?.node?.let(converter)
+        return meta[key ?: property.name.asName()]?.node?.let(converter)
     }
 
     override fun setValue(thisRef: Any?, property: KProperty<*>, value: T?) {
         if (value == null) {
-            meta.remove(key ?: property.name)
+            meta.remove(key ?: property.name.asName())
         } else {
-            meta[key ?: property.name] = value.config
+            meta[key ?: property.name.asName()] = value.config
         }
     }
 }
@@ -383,44 +388,45 @@ class ReadWriteDelegateWrapper<T, R>(
 /**
  * A property delegate that uses custom key
  */
-fun <M : MutableMeta<M>> M.value(default: Value = Null, key: String? = null) =
+fun <M : MutableMeta<M>> M.value(default: Value = Null, key: Name? = null) =
     MutableValueDelegate(this, key, default)
 
-fun <M : MutableMeta<M>> M.string(default: String? = null, key: String? = null) =
+fun <M : MutableMeta<M>> M.string(default: String? = null, key: Name? = null) =
     MutableStringDelegate(this, key, default)
 
-fun <M : MutableMeta<M>> M.boolean(default: Boolean? = null, key: String? = null) =
+fun <M : MutableMeta<M>> M.boolean(default: Boolean? = null, key: Name? = null) =
     MutableBooleanDelegate(this, key, default)
 
-fun <M : MutableMeta<M>> M.number(default: Number? = null, key: String? = null) =
+fun <M : MutableMeta<M>> M.number(default: Number? = null, key: Name? = null) =
     MutableNumberDelegate(this, key, default)
 
-fun <M : MutableMeta<M>> M.node(key: String? = null) = MutableNodeDelegate(this, key)
+fun <M : MutableMeta<M>> M.node(key: Name? = null) =
+    MutableNodeDelegate(this, key)
 
 @JvmName("safeString")
-fun <M : MutableMeta<M>> M.string(default: String, key: String? = null) =
+fun <M : MutableMeta<M>> M.string(default: String, key: Name? = null) =
     MutableSafeStringDelegate(this, key) { default }
 
 @JvmName("safeBoolean")
-fun <M : MutableMeta<M>> M.boolean(default: Boolean, key: String? = null) =
+fun <M : MutableMeta<M>> M.boolean(default: Boolean, key: Name? = null) =
     MutableSafeBooleanDelegate(this, key) { default }
 
 @JvmName("safeNumber")
-fun <M : MutableMeta<M>> M.number(default: Number, key: String? = null) =
+fun <M : MutableMeta<M>> M.number(default: Number, key: Name? = null) =
     MutableSafeNumberDelegate(this, key) { default }
 
 @JvmName("safeString")
-fun <M : MutableMeta<M>> M.string(key: String? = null, default: () -> String) =
+fun <M : MutableMeta<M>> M.string(key: Name? = null, default: () -> String) =
     MutableSafeStringDelegate(this, key, default)
 
 @JvmName("safeBoolean")
-fun <M : MutableMeta<M>> M.boolean(key: String? = null, default: () -> Boolean) =
+fun <M : MutableMeta<M>> M.boolean(key: Name? = null, default: () -> Boolean) =
     MutableSafeBooleanDelegate(this, key, default)
 
 @JvmName("safeNumber")
-fun <M : MutableMeta<M>> M.number(key: String? = null, default: () -> Number) =
+fun <M : MutableMeta<M>> M.number(key: Name? = null, default: () -> Number) =
     MutableSafeNumberDelegate(this, key, default)
 
 
-inline fun <M : MutableMeta<M>, reified E : Enum<E>> M.enum(default: E, key: String? = null) =
+inline fun <M : MutableMeta<M>, reified E : Enum<E>> M.enum(default: E, key: Name? = null) =
     MutableSafeEnumvDelegate(this, key, default) { enumValueOf(it) }

dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/metaMatcher.kt

@@ -0,0 +1,51 @@
+package hep.dataforge.meta
+
+import hep.dataforge.names.Name
+import hep.dataforge.names.toName
+
+/**
+ * Get all items matching given name.
+ */
+@DFExperimental
+fun Meta.getIndexed(name: Name): Map<String, MetaItem<*>> {
+    val root = when (name.length) {
+        0 -> error("Can't use empty name for that")
+        1 -> this
+        else -> (this[name.cutLast()] as? MetaItem.NodeItem<*>)?.node
+    }
+
+    val (body, index) = name.last()!!
+    val regex = index.toRegex()
+
+    return root?.items
+        ?.filter { it.key.body == body && (index.isEmpty() || regex.matches(it.key.index)) }
+        ?.mapKeys { it.key.index }
+        ?: emptyMap()
+}
+
+@DFExperimental
+fun Meta.getIndexed(name: String): Map<String, MetaItem<*>> = this@getIndexed.getIndexed(name.toName())
+
+
+/**
+ * Get all items matching given name.
+ */
+@DFExperimental
+fun <M : MetaNode<M>> M.getIndexed(name: Name): Map<String, MetaItem<M>> {
+    val root: MetaNode<M>? = when (name.length) {
+        0 -> error("Can't use empty name for that")
+        1 -> this
+        else -> (this[name.cutLast()] as? MetaItem.NodeItem<M>)?.node
+    }
+
+    val (body, index) = name.last()!!
+    val regex = index.toRegex()
+
+    return root?.items
+        ?.filter { it.key.body == body && (index.isEmpty() || regex.matches(it.key.index)) }
+        ?.mapKeys { it.key.index }
+        ?: emptyMap()
+}
+
+@DFExperimental
+fun <M : MetaNode<M>> M.getIndexed(name: String): Map<String, MetaItem<M>> = getIndexed(name.toName())

dataforge-meta/src/commonMain/kotlin/hep/dataforge/names/Name.kt

@@ -139,9 +139,7 @@ fun String.toName(): Name {
  * Convert the [String] to a [Name] by simply wrapping it in a single name token without parsing.
  * The input string could contain dots and braces, but they are just escaped, not parsed.
  */
-fun String.asName(): Name {
+fun String.asName(): Name = if (isBlank()) EmptyName else NameToken(this).asName()
-    return NameToken(this).asName()
-}
 
 operator fun NameToken.plus(other: Name): Name = Name(listOf(this) + other.tokens)
 
@@ -149,6 +147,8 @@ operator fun Name.plus(other: Name): Name = Name(this.tokens + other.tokens)
 
 operator fun Name.plus(other: String): Name = this + other.toName()
 
+operator fun Name.plus(other: NameToken): Name = Name(tokens + other)
+
 fun Name.appendLeft(other: String): Name = NameToken(other) + this
 
 fun NameToken.asName() = Name(listOf(this))

dataforge-meta/src/commonMain/kotlin/hep/dataforge/values/Value.kt

@@ -41,10 +41,12 @@ interface Value {
      * get this value represented as List
      */
     val list: List<Value>
-        get() = listOf(this)
+        get() = if(this == Null) emptyList() else listOf(this)
 
     override fun equals(other: Any?): Boolean
 
+    override fun hashCode(): Int
+
     companion object {
         const val TYPE = "value"
 
@@ -58,7 +60,14 @@ interface Value {
                 true -> True
                 false -> False
                 is Number -> value.asValue()
-                is Iterable<*> -> ListValue(value.map { of(it) })
+                is Iterable<*> -> {
+                    val list = value.map { of(it) }
+                    if (list.isEmpty()) {
+                        Null
+                    } else {
+                        ListValue(list)
+                    }
+                }
                 is DoubleArray -> value.asValue()
                 is IntArray -> value.asValue()
                 is FloatArray -> value.asValue()
@@ -86,14 +95,9 @@ object Null : Value {
     override fun toString(): String = value.toString()
 
     override fun equals(other: Any?): Boolean = other === Null
+    override fun hashCode(): Int = 0
 }
 
-/**
- * Check if value is null
- */
-fun Value.isNull(): Boolean = this == Null
-
-
 /**
  * Singleton true value
  */
@@ -106,7 +110,7 @@ object True : Value {
     override fun toString(): String = value.toString()
 
     override fun equals(other: Any?): Boolean = other === True
-
+    override fun hashCode(): Int = 1
 }
 
 /**
@@ -121,10 +125,9 @@ object False : Value {
     override fun toString(): String = True.value.toString()
 
     override fun equals(other: Any?): Boolean = other === False
+    override fun hashCode(): Int = -1
 }
 
-val Value.boolean get() = this == True || this.list.firstOrNull() == True || (type == ValueType.STRING && string.toBoolean())
-
 class NumberValue(override val number: Number) : Value {
     override val value: Any? get() = number
     override val type: ValueType get() = ValueType.NUMBER
@@ -178,23 +181,21 @@ class EnumValue<E : Enum<*>>(override val value: E) : Value {
 
 class ListValue(override val list: List<Value>) : Value {
     init {
-        if (list.isEmpty()) {
+        require(list.isNotEmpty()) { "Can't create list value from empty list" }
-            throw IllegalArgumentException("Can't create list value from empty list")
-        }
     }
 
-    override val value: Any? get() = list
+    override val value: List<Value> get() = list
     override val type: ValueType get() = list.first().type
     override val number: Number get() = list.first().number
     override val string: String get() = list.first().string
 
-    override fun toString(): String = list.joinToString (prefix = "[", postfix = "]")
+    override fun toString(): String = list.joinToString(prefix = "[", postfix = "]")
 
     override fun equals(other: Any?): Boolean {
         if (this === other) return true
         if (other !is Value) return false
-        if( other is DoubleArrayValue){
+        if (other is DoubleArrayValue) {
-
+            return DoubleArray(list.size) { list[it].number.toDouble() }.contentEquals(other.value)
         }
         return list == other.list
     }
@@ -206,29 +207,26 @@ class ListValue(override val list: List<Value>) : Value {
 
 }
 
-/**
- * Check if value is list
- */
-fun Value.isList(): Boolean = this.list.size > 1
-
-
 fun Number.asValue(): Value = NumberValue(this)
 
 fun Boolean.asValue(): Value = if (this) True else False
 
 fun String.asValue(): Value = StringValue(this)
 
-fun Iterable<Value>.asValue(): Value = ListValue(this.toList())
+fun Iterable<Value>.asValue(): Value {
+    val list = toList()
+    return if (list.isEmpty()) Null else ListValue(this.toList())
+}
 
-fun IntArray.asValue(): Value = ListValue(map{NumberValue(it)})
+fun IntArray.asValue(): Value = if (isEmpty()) Null else ListValue(map { NumberValue(it) })
 
-fun LongArray.asValue(): Value = ListValue(map{NumberValue(it)})
+fun LongArray.asValue(): Value = if (isEmpty()) Null else ListValue(map { NumberValue(it) })
 
-fun ShortArray.asValue(): Value = ListValue(map{NumberValue(it)})
+fun ShortArray.asValue(): Value = if (isEmpty()) Null else ListValue(map { NumberValue(it) })
 
-fun FloatArray.asValue(): Value = ListValue(map{NumberValue(it)})
+fun FloatArray.asValue(): Value = if (isEmpty()) Null else ListValue(map { NumberValue(it) })
 
-fun ByteArray.asValue(): Value = ListValue(map{NumberValue(it)})
+fun ByteArray.asValue(): Value = if (isEmpty()) Null else ListValue(map { NumberValue(it) })
 
 
 /**

dataforge-meta/src/commonMain/kotlin/hep/dataforge/values/exoticValues.kt

@@ -14,6 +14,8 @@ class LazyParsedValue(override val string: String) : Value {
     override fun toString(): String = string
 
     override fun equals(other: Any?): Boolean = other is Value && this.parsedValue == other
+
+    override fun hashCode(): Int  = string.hashCode()
 }
 
 fun String.lazyParseValue(): LazyParsedValue = LazyParsedValue(this)
@@ -44,4 +46,4 @@ class DoubleArrayValue(override val value: DoubleArray) : Value {
     override fun toString(): String = list.joinToString (prefix = "[", postfix = "]")
 }
 
-fun DoubleArray.asValue(): DoubleArrayValue = DoubleArrayValue(this)
+fun DoubleArray.asValue(): Value = if(isEmpty()) Null else DoubleArrayValue(this)

dataforge-meta/src/commonMain/kotlin/hep/dataforge/values/valueExtensions.kt

@@ -0,0 +1,37 @@
+package hep.dataforge.values
+
+import hep.dataforge.meta.Meta
+import hep.dataforge.meta.buildMeta
+
+/**
+ * Check if value is null
+ */
+fun Value.isNull(): Boolean = this == Null
+
+/**
+ * Check if value is list
+ */
+fun Value.isList(): Boolean = this.list.size > 1
+
+val Value.boolean
+    get() = this == True
+            || this.list.firstOrNull() == True
+            || (type == ValueType.STRING && string.toBoolean())
+
+
+val Value.int get() = number.toInt()
+val Value.double get() = number.toDouble()
+val Value.float get() = number.toFloat()
+val Value.long get() = number.toLong()
+
+val Value.stringList: List<String> get() = list.map { it.string }
+
+val Value.doubleArray: DoubleArray
+    get() = if (this is DoubleArrayValue) {
+        value
+    } else {
+        DoubleArray(list.size) { list[it].double }
+    }
+
+
+fun Value.toMeta() = buildMeta { Meta.VALUE_KEY put this }

dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/MetaBuilderTest.kt

@@ -9,13 +9,13 @@ class MetaBuilderTest {
     @Test
     fun testBuilder() {
         val meta = buildMeta {
-            "a" to 22
+            "a" put 22
-            "b" to listOf(1, 2, 3)
+            "b" put listOf(1, 2, 3)
             this["c"] = "myValue".asValue()
-            "node" to {
+            "node" put {
-                "e" to 12.2
+                "e" put 12.2
-                "childNode" to {
+                "childNode" put {
-                    "f" to true
+                    "f" put true
                 }
             }
         }
@@ -27,12 +27,12 @@ class MetaBuilderTest {
     fun testSNS(){
         val meta = buildMeta {
             repeat(10){
-                "b.a[$it]" to it
+                "b.a[$it]" put it
             }
         }.seal()
         assertEquals(10, meta.values().count())
 
-        val nodes = meta.getAll("b.a")
+        val nodes = meta.getIndexed("b.a")
 
         assertEquals(3, nodes["3"]?.int)
     }

dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/MetaExtensionTest.kt

@@ -0,0 +1,23 @@
+package hep.dataforge.meta
+
+import kotlin.test.Test
+
+
+class MetaExtensionTest {
+
+    enum class TestEnum{
+        test
+    }
+
+    @Test
+    fun testEnum(){
+        val meta = buildMeta{"enum" put TestEnum.test}
+        meta["enum"].enum<TestEnum>()
+    }
+    @Test
+    fun testEnumByString(){
+        val meta = buildMeta{"enum" put TestEnum.test.name}
+        println(meta["enum"].enum<TestEnum>())
+    }
+
+}

dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/MetaTest.kt

@@ -17,17 +17,36 @@ class MetaTest {
     @Test
     fun metaEqualityTest() {
         val meta1 = buildMeta {
-            "a" to 22
+            "a" put 22
-            "b" to {
+            "b" put {
-                "c" to "ddd"
+                "c" put "ddd"
             }
         }
         val meta2 = buildMeta {
-            "b" to {
+            "b" put {
-                "c" to "ddd"
+                "c" put "ddd"
             }
-            "a" to 22
+            "a" put 22
         }.seal()
         assertEquals<Meta>(meta1, meta2)
     }
+
+    @Test
+    fun metaToMap(){
+        val meta = buildMeta {
+            "a" put 22
+            "b" put {
+                "c" put "ddd"
+            }
+            "list" put (0..4).map {
+                buildMeta {
+                    "value" put it
+                }
+            }
+        }
+        val map = meta.toMap()
+        val reconstructed = map.toMeta()
+
+        assertEquals(meta,reconstructed)
+    }
 }

dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/MutableMetaTest.kt

@@ -7,12 +7,12 @@ class MutableMetaTest{
     @Test
     fun testRemove(){
         val meta = buildMeta {
-            "aNode" to {
+            "aNode" put {
-                "innerNode" to {
+                "innerNode" put {
-                    "innerValue" to true
+                    "innerValue" put true
                 }
-                "b" to 22
+                "b" put 22
-                "c" to "StringValue"
+                "c" put "StringValue"
             }
         }.toConfig()
 

dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/SpecificationTest.kt

@@ -0,0 +1,23 @@
+package hep.dataforge.meta
+
+import kotlin.test.Test
+import kotlin.test.assertEquals
+
+class SpecificationTest {
+    class TestSpecific(override val config: Config) : Specific {
+        var list by numberList(1, 2, 3)
+
+        companion object : Specification<TestSpecific> {
+            override fun wrap(config: Config): TestSpecific = TestSpecific(config)
+        }
+    }
+
+
+    @Test
+    fun testSpecific(){
+        val testObject = TestSpecific.build {
+            list = emptyList()
+        }
+        assertEquals(emptyList(), testObject.list)
+    }
+}

dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/StyledTest.kt

@@ -9,8 +9,8 @@ class StyledTest{
     fun testSNS(){
         val meta = buildMeta {
             repeat(10){
-                "b.a[$it]" to {
+                "b.a[$it]" put {
-                    "d" to it
+                    "d" put it
                 }
             }
         }.seal().withStyle()
@@ -18,9 +18,9 @@ class StyledTest{
 
         val bNode = meta["b"].node
 
-        val aNodes = bNode?.getAll("a")
+        val aNodes = bNode?.getIndexed("a")
 
-        val allNodes = meta.getAll("b.a")
+        val allNodes = meta.getIndexed("b.a")
 
         assertEquals(3, aNodes?.get("3").node["d"].int)
         assertEquals(3, allNodes["3"].node["d"].int)

dataforge-meta/src/jsMain/kotlin/hep/dataforge/meta/DynamicMeta.kt

@@ -3,18 +3,27 @@ package hep.dataforge.meta
 import hep.dataforge.names.NameToken
 import hep.dataforge.values.Null
 import hep.dataforge.values.Value
+import hep.dataforge.values.isList
 
 
 //TODO add Meta wrapper for dynamic
 
+fun Value.toDynamic(): dynamic {
+    return if (isList()) {
+        list.map { it.toDynamic() }.toTypedArray().asDynamic()
+    } else {
+        value.asDynamic()
+    }
+}
+
 /**
  * Represent or copy this [Meta] to dynamic object to be passed to JS libraries
  */
 fun Meta.toDynamic(): dynamic {
-    if(this is DynamicMeta) return this.obj
+    if (this is DynamicMeta) return this.obj
 
     fun MetaItem<*>.toDynamic(): dynamic = when (this) {
-        is MetaItem.ValueItem -> this.value.value.asDynamic()
+        is MetaItem.ValueItem -> this.value.toDynamic()
         is MetaItem.NodeItem -> this.node.toDynamic()
     }
 
@@ -35,15 +44,21 @@ class DynamicMeta(val obj: dynamic) : MetaBase() {
     private fun isArray(@Suppress("UNUSED_PARAMETER") obj: dynamic): Boolean =
         js("Array.isArray(obj)") as Boolean
 
+    private fun isPrimitive(obj: dynamic): Boolean =
+        (jsTypeOf(obj) != "object")
+
     @Suppress("UNCHECKED_CAST")
     private fun asItem(obj: dynamic): MetaItem<DynamicMeta>? {
-        if (obj == null) return MetaItem.ValueItem(Null)
+        return when {
-        return when (jsTypeOf(obj as? Any)) {
+            obj == null -> MetaItem.ValueItem(Null)
-            "boolean" -> MetaItem.ValueItem(Value.of(obj as Boolean))
+            isArray(obj) && (obj as Array<Any?>).all { isPrimitive(it) } -> MetaItem.ValueItem(Value.of(obj as Array<Any?>))
-            "number" -> MetaItem.ValueItem(Value.of(obj as Number))
+            else -> when (jsTypeOf(obj)) {
-            "string" -> MetaItem.ValueItem(Value.of(obj as String))
+                "boolean" -> MetaItem.ValueItem(Value.of(obj as Boolean))
-            "object" -> MetaItem.NodeItem(DynamicMeta(obj))
+                "number" -> MetaItem.ValueItem(Value.of(obj as Number))
-            else -> null
+                "string" -> MetaItem.ValueItem(Value.of(obj as String))
+                "object" -> MetaItem.NodeItem(DynamicMeta(obj))
+                else -> null
+            }
         }
     }
 
@@ -51,11 +66,15 @@ class DynamicMeta(val obj: dynamic) : MetaBase() {
         get() = keys().flatMap<String, Pair<NameToken, MetaItem<DynamicMeta>>> { key ->
             val value = obj[key] ?: return@flatMap emptyList()
             if (isArray(value)) {
-                return@flatMap (value as Array<dynamic>)
+                val array = value as Array<Any?>
-                    .mapIndexedNotNull() { index, it ->
+                return@flatMap if (array.all { isPrimitive(it) }) {
+                    listOf(NameToken(key) to MetaItem.ValueItem(Value.of(array)))
+                } else {
+                    array.mapIndexedNotNull { index, it ->
                         val item = asItem(it) ?: return@mapIndexedNotNull null
                         NameToken(key, index.toString()) to item
                     }
+                }
             } else {
                 val item = asItem(value) ?: return@flatMap emptyList()
                 listOf(NameToken(key) to item)

dataforge-meta/src/jsTest/kotlin/hep/dataforge/meta/DynamicMetaTest.kt

@@ -1,7 +1,9 @@
 package hep.dataforge.meta
 
+import hep.dataforge.values.int
 import kotlin.test.Test
 import kotlin.test.assertEquals
+import kotlin.test.assertTrue
 
 
 class DynamicMetaTest {
@@ -10,7 +12,7 @@ class DynamicMetaTest {
     fun testDynamicMeta() {
         val d = js("{}")
         d.a = 22
-        d.array = arrayOf(1,2,3)
+        d.array = arrayOf(1, 2, 3)
         d.b = "myString"
         d.ob = js("{}")
         d.ob.childNode = 18
@@ -18,7 +20,35 @@ class DynamicMetaTest {
 
         val meta = DynamicMeta(d)
         assertEquals(true, meta["ob.booleanNode"].boolean)
-        assertEquals(2,meta["array[1]"].int)
+        assertEquals(2, meta["array"].value?.list?.get(1)?.int)
+        assertEquals(4, meta.items.size)
+    }
+
+    @Test
+    fun testMetaToDynamic(){
+        val meta = buildMeta {
+            "a" put 22
+            "array" put listOf(1, 2, 3)
+            "b" put "myString"
+            "ob" put {
+                "childNode" put 18
+                "booleanNode" put true
+            }
+        }
+
+        val dynamic = meta.toDynamic()
+
+        assertEquals(2,dynamic.array[1])
+
+        assertEquals(22, dynamic.a)
+
+        val keys = js("Object.keys(dynamic)") as Array<String>
+
+        assertTrue { keys.contains("ob") }
+
+        assertEquals(18, dynamic.ob.childNode)
+
+        assertEquals<Meta>(meta, DynamicMeta(dynamic))
     }
 
 }

dataforge-output/src/commonMain/kotlin/hep/dataforge/output/OutputManager.kt

@@ -70,7 +70,7 @@ class ConsoleOutputManager : AbstractPlugin(), OutputManager {
 
         override val type = ConsoleOutputManager::class
 
-        override fun invoke(meta:Meta) = ConsoleOutputManager()
+        override fun invoke(meta: Meta, context: Context) = ConsoleOutputManager()
     }
 }
 

dataforge-scripting/src/jvmTest/kotlin/hep/dataforge/scripting/BuildersKtTest.kt

@@ -3,7 +3,9 @@ package hep.dataforge.scripting
 import hep.dataforge.context.Global
 import hep.dataforge.meta.get
 import hep.dataforge.meta.int
-import hep.dataforge.workspace.*
+import hep.dataforge.workspace.SimpleWorkspaceBuilder
+import hep.dataforge.workspace.context
+import hep.dataforge.workspace.target
 import org.junit.Test
 import kotlin.test.assertEquals
 
@@ -17,7 +19,7 @@ class BuildersKtTest {
             context("test")
 
             target("testTarget"){
-                "a" to 12
+                "a" put 12
             }
         }
     }
@@ -30,7 +32,7 @@ class BuildersKtTest {
             context("test")
 
             target("testTarget"){
-                "a" to 12
+                "a" put 12
             }
         """.trimIndent()
         val workspace = Builders.buildWorkspace(script)

dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/Dependency.kt

@@ -6,10 +6,7 @@ import hep.dataforge.data.filter
 import hep.dataforge.meta.Meta
 import hep.dataforge.meta.MetaRepr
 import hep.dataforge.meta.buildMeta
-import hep.dataforge.names.EmptyName
+import hep.dataforge.names.*
-import hep.dataforge.names.Name
-import hep.dataforge.names.get
-import hep.dataforge.names.isEmpty
 
 /**
  * A dependency of the task which allows to lazily create a data tree for single dependency
@@ -24,13 +21,13 @@ class DataDependency(val filter: DataFilter, val placement: Name = EmptyName) :
         return if (placement.isEmpty()) {
             result
         } else {
-            DataNode.build(Any::class) { this[placement] = result }
+            DataNode.invoke(Any::class) { this[placement] = result }
         }
     }
 
     override fun toMeta(): Meta = buildMeta {
-        "data" to filter.config
+        "data" put filter.config
-        "to" to placement
+        "to" put placement.toString()
     }
 }
 
@@ -38,31 +35,63 @@ class AllDataDependency(val placement: Name = EmptyName) : Dependency() {
     override fun apply(workspace: Workspace): DataNode<Any> = if (placement.isEmpty()) {
         workspace.data
     } else {
-        DataNode.build(Any::class) { this[placement] = workspace.data }
+        DataNode.invoke(Any::class) { this[placement] = workspace.data }
     }
 
     override fun toMeta() = buildMeta {
-        "data" to "*"
+        "data" put "@all"
-        "to" to placement
+        "to" put placement.toString()
     }
-
 }
 
-class TaskModelDependency(val name: String, val meta: Meta, val placement: Name = EmptyName) : Dependency() {
+abstract class TaskDependency<out T : Any>(
-    override fun apply(workspace: Workspace): DataNode<Any> {
+    val meta: Meta,
-        val task = workspace.tasks[name] ?: error("Task with name $name is not found in the workspace")
+    val placement: Name = EmptyName
+) : Dependency() {
+    abstract fun resolveTask(workspace: Workspace): Task<T>
+
+    /**
+     * A name of the dependency for logging and serialization
+     */
+    abstract val name: Name
+
+    override fun apply(workspace: Workspace): DataNode<T> {
+        val task = resolveTask(workspace)
         if (task.isTerminal) TODO("Support terminal task")
         val result = workspace.run(task, meta)
         return if (placement.isEmpty()) {
             result
         } else {
-            DataNode.build(Any::class) { this[placement] = result }
+            DataNode(task.type) { this[placement] = result }
         }
     }
 
     override fun toMeta(): Meta = buildMeta {
-        "task" to name
+        "task" put name.toString()
-        "meta" to meta
+        "meta" put meta
-        "to" to placement
+        "to" put placement.toString()
     }
+}
+
+class DirectTaskDependency<T : Any>(
+    val task: Task<T>,
+    meta: Meta,
+    placement: Name
+) : TaskDependency<T>(meta, placement) {
+    override fun resolveTask(workspace: Workspace): Task<T> = task
+
+    override val name: Name get() = DIRECT_TASK_NAME + task.name
+
+    companion object {
+        val DIRECT_TASK_NAME = "@direct".asName()
+    }
+}
+
+class WorkspaceTaskDependency(
+    override val name: Name,
+    meta: Meta,
+    placement: Name
+) : TaskDependency<Any>(meta, placement) {
+    override fun resolveTask(workspace: Workspace): Task<*> =
+        workspace.tasks[name] ?: error("Task with name $name is not found in the workspace")
 }

dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/GenericTask.kt

@@ -1,37 +1,38 @@
 package hep.dataforge.workspace
 
-import hep.dataforge.data.*
+import hep.dataforge.data.DataNode
 import hep.dataforge.descriptors.NodeDescriptor
 import hep.dataforge.meta.Meta
 import hep.dataforge.meta.get
 import hep.dataforge.meta.node
+import hep.dataforge.names.Name
 import kotlin.reflect.KClass
 
 //data class TaskEnv(val workspace: Workspace, val model: TaskModel)
 
 
 class GenericTask<R : Any>(
-    override val name: String,
+    override val name: Name,
     override val type: KClass<out R>,
     override val descriptor: NodeDescriptor,
     private val modelTransform: TaskModelBuilder.(Meta) -> Unit,
     private val dataTransform: Workspace.() -> TaskModel.(DataNode<Any>) -> DataNode<R>
 ) : Task<R> {
 
-    private fun gather(workspace: Workspace, model: TaskModel): DataNode<Any> {
+//    private fun gather(workspace: Workspace, model: TaskModel): DataNode<Any> {
-        return DataNode.build(Any::class) {
+//        return DataNode.invoke(Any::class) {
-            model.dependencies.forEach { dep ->
+//            model.dependencies.forEach { dep ->
-                update(dep.apply(workspace))
+//                update(dep.apply(workspace))
-            }
+//            }
-        }
+//        }
-    }
+//    }
 
     override fun run(workspace: Workspace, model: TaskModel): DataNode<R> {
         //validate model
         validate(model)
 
         // gather data
-        val input = gather(workspace, model)
+        val input = model.buildInput(workspace)// gather(workspace, model)
 
         //execute
         workspace.context.logger.info{"Starting task '$name' on ${model.target} with meta: \n${model.meta}"}
@@ -54,7 +55,7 @@ class GenericTask<R : Any>(
     override fun build(workspace: Workspace, taskConfig: Meta): TaskModel {
         val taskMeta = taskConfig[name]?.node ?: taskConfig
         val builder = TaskModelBuilder(name, taskMeta)
-        modelTransform.invoke(builder, taskMeta)
+        builder.modelTransform(taskMeta)
         return builder.build()
     }
     //TODO add validation

dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/SimpleWorkspace.kt

@@ -1,12 +1,11 @@
 package hep.dataforge.workspace
 
 import hep.dataforge.context.Context
-import hep.dataforge.context.Global
 import hep.dataforge.context.content
+import hep.dataforge.context.toMap
 import hep.dataforge.data.DataNode
 import hep.dataforge.meta.Meta
 import hep.dataforge.names.Name
-import hep.dataforge.names.toName
 
 
 /**
@@ -20,11 +19,10 @@ class SimpleWorkspace(
 ) : Workspace {
 
     override val tasks: Map<Name, Task<*>> by lazy {
-        context.content<Task<*>>(Task.TYPE) + tasks.associate { it.name.toName() to it }
+        context.content<Task<*>>(Task.TYPE) + tasks.toMap()
     }
 
     companion object {
-        fun build(parent: Context = Global, block: SimpleWorkspaceBuilder.() -> Unit): SimpleWorkspace =
+
-            SimpleWorkspaceBuilder(parent).apply(block).build()
     }
 }

dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskBuilder.kt

@@ -6,54 +6,78 @@ import hep.dataforge.descriptors.NodeDescriptor
 import hep.dataforge.meta.Meta
 import hep.dataforge.meta.get
 import hep.dataforge.meta.string
+import hep.dataforge.names.EmptyName
 import hep.dataforge.names.Name
+import hep.dataforge.names.isEmpty
 import hep.dataforge.names.toName
+import kotlin.jvm.JvmName
 import kotlin.reflect.KClass
 
 @TaskBuildScope
-class TaskBuilder(val name: String) {
+class TaskBuilder<R : Any>(val name: Name, val type: KClass<out R>) {
-    private var modelTransform: TaskModelBuilder.(Meta) -> Unit = { data("*") }
+    private var modelTransform: TaskModelBuilder.(Meta) -> Unit = { allData() }
+//    private val additionalDependencies = HashSet<Dependency>()
     var descriptor: NodeDescriptor? = null
+    private val dataTransforms: MutableList<DataTransformation> = ArrayList()
 
     /**
      * TODO will look better as extension class
      */
-    private class DataTransformation(
+    private inner class DataTransformation(
         val from: String = "",
         val to: String = "",
-        val transform: (Context, TaskModel, DataNode<Any>) -> DataNode<Any>
+        val transform: (Context, TaskModel, DataNode<Any>) -> DataNode<R>
     ) {
-        operator fun invoke(workspace: Workspace, model: TaskModel, node: DataNode<Any>): DataNode<Any>? {
+        operator fun invoke(workspace: Workspace, model: TaskModel, node: DataNode<Any>): DataNode<R>? {
             val localData = if (from.isEmpty()) {
                 node
             } else {
-                node[from.toName()].node ?: return null
+                node[from].node ?: return null
             }
             return transform(workspace.context, model, localData)
         }
     }
 
-    private val dataTransforms: MutableList<DataTransformation> = ArrayList();
+//    override fun add(dependency: Dependency) {
+//        additionalDependencies.add(dependency)
+//    }
 
     fun model(modelTransform: TaskModelBuilder.(Meta) -> Unit) {
         this.modelTransform = modelTransform
     }
 
-    fun <T : Any> transform(
+    /**
-        inputType: KClass<T>,
+     * Add a transformation on untyped data
+     */
+    @JvmName("rawTransform")
+    fun transform(
         from: String = "",
         to: String = "",
-        block: TaskModel.(Context, DataNode<T>) -> DataNode<Any>
+        block: TaskEnv.(DataNode<*>) -> DataNode<R>
     ) {
         dataTransforms += DataTransformation(from, to) { context, model, data ->
-            block(model, context, data.cast(inputType))
+            val env = TaskEnv(EmptyName, model.meta, context, data)
+            env.block(data)
+        }
+    }
+
+    fun <T : Any> transform(
+        inputType: KClass<out T>,
+        from: String = "",
+        to: String = "",
+        block: TaskEnv.(DataNode<T>) -> DataNode<R>
+    ) {
+        dataTransforms += DataTransformation(from, to) { context, model, data ->
+            data.ensureType(inputType)
+            val env = TaskEnv(EmptyName, model.meta, context, data)
+            env.block(data.cast(inputType))
         }
     }
 
     inline fun <reified T : Any> transform(
         from: String = "",
         to: String = "",
-        noinline block: TaskModel.(Context, DataNode<T>) -> DataNode<Any>
+        noinline block: TaskEnv.(DataNode<T>) -> DataNode<R>
     ) {
         transform(T::class, from, to, block)
     }
@@ -61,52 +85,57 @@ class TaskBuilder(val name: String) {
     /**
      * Perform given action on data elements in `from` node in input and put the result to `to` node
      */
-    inline fun <reified T : Any, reified R : Any> action(
+    inline fun <reified T : Any> action(
         from: String = "",
         to: String = "",
-        crossinline block: Context.() -> Action<T, R>
+        crossinline block: TaskEnv.() -> Action<T, R>
     ) {
-        transform(from, to) { context, data: DataNode<T> ->
+        transform(from, to) { data: DataNode<T> ->
-            block(context).invoke(data, meta)
+            block().invoke(data, meta)
         }
     }
 
-    class TaskEnv(val name: Name, val meta: Meta, val context: Context)
+    class TaskEnv(val name: Name, val meta: Meta, val context: Context, val data: DataNode<Any>) {
+        operator fun <T : Any> DirectTaskDependency<T>.invoke(): DataNode<T> = if(placement.isEmpty()){
+            data.cast(task.type)
+        } else {
+            data[placement].node?.cast(task.type)
+                ?: error("Could not find results of direct task dependency $this at \"$placement\"")
+        }
+    }
 
     /**
-     * A customized pipe action with ability to change meta and name
+     * A customized map action with ability to change meta and name
      */
-    inline fun <reified T : Any, reified R : Any> customPipe(
+    inline fun <reified T : Any> mapAction(
         from: String = "",
         to: String = "",
-        crossinline block: PipeBuilder<T, R>.(Context) -> Unit
+        crossinline block: MapActionBuilder<T, R>.(TaskEnv) -> Unit
     ) {
         action(from, to) {
-            val context = this
+            MapAction(
-            PipeAction(
                 inputType = T::class,
-                outputType = R::class
+                outputType = type
-            ) { block(context) }
+            ) { block(this@action) }
         }
     }
 
     /**
-     * A simple pipe action without changing meta or name
+     * A simple map action without changing meta or name
      */
-    inline fun <reified T : Any, reified R : Any> pipe(
+    inline fun <reified T : Any> map(
         from: String = "",
         to: String = "",
         crossinline block: suspend TaskEnv.(T) -> R
     ) {
         action(from, to) {
-            val context = this
+            MapAction(
-            PipeAction(
                 inputType = T::class,
-                outputType = R::class
+                outputType = type
             ) {
                 //TODO automatically append task meta
                 result = { data ->
-                    TaskEnv(name, meta, context).block(data)
+                    block(data)
                 }
             }
         }
@@ -115,15 +144,15 @@ class TaskBuilder(val name: String) {
     /**
      * Join elements in gathered data by multiple groups
      */
-    inline fun <reified T : Any, reified R : Any> joinByGroup(
+    inline fun <reified T : Any> reduceByGroup(
         from: String = "",
         to: String = "",
-        crossinline block: JoinGroupBuilder<T, R>.(Context) -> Unit
+        crossinline block: ReduceGroupBuilder<T, R>.(TaskEnv) -> Unit        //TODO needs KEEP-176
     ) {
         action(from, to) {
-            JoinAction(
+            ReduceAction(
                 inputType = T::class,
-                outputType = R::class
+                outputType = type
             ) { block(this@action) }
         }
     }
@@ -131,21 +160,20 @@ class TaskBuilder(val name: String) {
     /**
      * Join all elemlents in gathered data matching input type
      */
-    inline fun <reified T : Any, reified R : Any> join(
+    inline fun <reified T : Any> reduce(
         from: String = "",
         to: String = "",
         crossinline block: suspend TaskEnv.(Map<Name, T>) -> R
     ) {
         action(from, to) {
-            val context = this
+            ReduceAction(
-            JoinAction(
                 inputType = T::class,
-                outputType = R::class,
+                outputType = type,
                 action = {
                     result(
                         actionMeta[TaskModel.MODEL_TARGET_KEY]?.string ?: "@anonymous"
                     ) { data ->
-                        TaskEnv(name, meta, context).block(data)
+                        block(data)
                     }
                 }
             )
@@ -155,15 +183,15 @@ class TaskBuilder(val name: String) {
     /**
      * Split each element in gathered data into fixed number of fragments
      */
-    inline fun <reified T : Any, reified R : Any> split(
+    inline fun <reified T : Any> split(
         from: String = "",
         to: String = "",
-        crossinline block: SplitBuilder<T, R>.(Context) -> Unit
+        crossinline block: SplitBuilder<T, R>.(TaskEnv) -> Unit  //TODO needs KEEP-176
     ) {
         action(from, to) {
             SplitAction(
                 inputType = T::class,
-                outputType = R::class
+                outputType = type
             ) { block(this@action) }
         }
     }
@@ -175,22 +203,28 @@ class TaskBuilder(val name: String) {
         this.descriptor = NodeDescriptor.build(transform)
     }
 
-    internal fun build(): GenericTask<Any> =
+    internal fun build(): GenericTask<R> {
-        GenericTask(
+//        val actualTransform: TaskModelBuilder.(Meta) -> Unit = {
+//            modelTransform
+//            dependencies.addAll(additionalDependencies)
+//        }
+
+        return GenericTask(
             name,
-            Any::class,
+            type,
             descriptor ?: NodeDescriptor.empty(),
             modelTransform
         ) {
             val workspace = this
-            { data ->
+            return@GenericTask { data ->
                 val model = this
                 if (dataTransforms.isEmpty()) {
                     //return data node as is
-                    logger.warn("No transformation present, returning input data")
+                    logger.warn { "No transformation present, returning input data" }
-                    data
+                    data.ensureType(type)
+                    data.cast(type)
                 } else {
-                    val builder = DataTreeBuilder(Any::class)
+                    val builder = DataTreeBuilder(type)
                     dataTransforms.forEach { transformation ->
                         val res = transformation(workspace, model, data)
                         if (res != null) {
@@ -205,14 +239,6 @@ class TaskBuilder(val name: String) {
                 }
             }
         }
+    }
 }
 
-fun Workspace.Companion.task(name: String, builder: TaskBuilder.() -> Unit): GenericTask<Any> {
-    return TaskBuilder(name).apply(builder).build()
-}
-
-fun WorkspaceBuilder.task(name: String, builder: TaskBuilder.() -> Unit) {
-    task(TaskBuilder(name).apply(builder).build())
-}
-
-//TODO add delegates to build gradle-like tasks

dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskModel.kt

@@ -8,10 +8,10 @@ package hep.dataforge.workspace
 import hep.dataforge.data.DataFilter
 import hep.dataforge.data.DataTree
 import hep.dataforge.data.DataTreeBuilder
-import hep.dataforge.data.dataSequence
 import hep.dataforge.meta.*
 import hep.dataforge.names.EmptyName
 import hep.dataforge.names.Name
+import hep.dataforge.names.asName
 import hep.dataforge.names.toName
 import hep.dataforge.workspace.TaskModel.Companion.MODEL_TARGET_KEY
 
@@ -23,7 +23,7 @@ import hep.dataforge.workspace.TaskModel.Companion.MODEL_TARGET_KEY
  * @param dependencies a list of direct dependencies for this task
  */
 data class TaskModel(
-    val name: String,
+    val name: Name,
     val meta: Meta,
     val dependencies: Collection<Dependency>
 ) : MetaRepr {
@@ -31,19 +31,19 @@ data class TaskModel(
     //TODO add pre-run check of task result type?
 
     override fun toMeta(): Meta = buildMeta {
-        "name" to name
+        "name" put name.toString()
-        "meta" to meta
+        "meta" put meta
-        "dependsOn" to {
+        "dependsOn" put {
             val dataDependencies = dependencies.filterIsInstance<DataDependency>()
-            val taskDependencies = dependencies.filterIsInstance<TaskModelDependency>()
+            val taskDependencies = dependencies.filterIsInstance<TaskDependency<*>>()
             setIndexed("data".toName(), dataDependencies.map { it.toMeta() })
-            setIndexed("task".toName(), taskDependencies.map { it.toMeta() }) { taskDependencies[it].name }
+            setIndexed("task".toName(), taskDependencies.map { it.toMeta() }) { taskDependencies[it].name.toString() }
             //TODO ensure all dependencies are listed
         }
     }
 
     companion object {
-        const val MODEL_TARGET_KEY = "@target"
+        val MODEL_TARGET_KEY = "@target".asName()
     }
 }
 
@@ -52,9 +52,8 @@ data class TaskModel(
  */
 fun TaskModel.buildInput(workspace: Workspace): DataTree<Any> {
     return DataTreeBuilder(Any::class).apply {
-        dependencies.asSequence().flatMap { it.apply(workspace).dataSequence() }.forEach { (name, data) ->
+        dependencies.forEach { dep ->
-            //TODO add concise error on replacement
+            update(dep.apply(workspace))
-            this[name] = data
         }
     }.build()
 }
@@ -62,54 +61,88 @@ fun TaskModel.buildInput(workspace: Workspace): DataTree<Any> {
 @DslMarker
 annotation class TaskBuildScope
 
+interface TaskDependencyContainer {
+    val defaultMeta: Meta
+    fun add(dependency: Dependency)
+}
+
+/**
+ * Add dependency for a task defined in a workspace and resolved by
+ */
+fun TaskDependencyContainer.dependsOn(
+    name: Name,
+    placement: Name = EmptyName,
+    meta: Meta = defaultMeta
+): WorkspaceTaskDependency =
+    WorkspaceTaskDependency(name, meta, placement).also { add(it) }
+
+fun TaskDependencyContainer.dependsOn(
+    name: String,
+    placement: Name = EmptyName,
+    meta: Meta = defaultMeta
+): WorkspaceTaskDependency =
+    dependsOn(name.toName(), placement, meta)
+
+fun <T : Any> TaskDependencyContainer.dependsOn(
+    task: Task<T>,
+    placement: Name = EmptyName,
+    meta: Meta = defaultMeta
+): DirectTaskDependency<T> =
+    DirectTaskDependency(task, meta, placement).also { add(it) }
+
+fun <T : Any> TaskDependencyContainer.dependsOn(
+    task: Task<T>,
+    placement: String,
+    meta: Meta = defaultMeta
+): DirectTaskDependency<T> =
+    DirectTaskDependency(task, meta, placement.toName()).also { add(it) }
+
+fun <T : Any> TaskDependencyContainer.dependsOn(
+    task: Task<T>,
+    placement: Name = EmptyName,
+    metaBuilder: MetaBuilder.() -> Unit
+): DirectTaskDependency<T> =
+    dependsOn(task, placement, buildMeta(metaBuilder))
+
+/**
+ * Add custom data dependency
+ */
+fun TaskDependencyContainer.data(action: DataFilter.() -> Unit): DataDependency =
+    DataDependency(DataFilter.build(action)).also { add(it) }
+
+/**
+ * User-friendly way to add data dependency
+ */
+fun TaskDependencyContainer.data(pattern: String? = null, from: String? = null, to: String? = null): DataDependency =
+    data {
+        pattern?.let { this.pattern = it }
+        from?.let { this.from = it }
+        to?.let { this.to = it }
+    }
+
+/**
+ * Add all data as root node
+ */
+fun TaskDependencyContainer.allData(to: Name = EmptyName) = AllDataDependency(to).also { add(it) }
+
 /**
  * A builder for [TaskModel]
  */
-@TaskBuildScope
+class TaskModelBuilder(val name: Name, meta: Meta = EmptyMeta) : TaskDependencyContainer {
-class TaskModelBuilder(val name: String, meta: Meta = EmptyMeta) {
     /**
      * Meta for current task. By default uses the whole input meta
      */
     var meta: MetaBuilder = meta.builder()
     val dependencies = HashSet<Dependency>()
 
+    override val defaultMeta: Meta get() = meta
+
+    override fun add(dependency: Dependency) {
+        dependencies.add(dependency)
+    }
+
     var target: String by this.meta.string(key = MODEL_TARGET_KEY, default = "")
 
-    /**
-     * Add dependency for
-     */
-    fun dependsOn(name: String, meta: Meta = this.meta, placement: Name = EmptyName) {
-        dependencies.add(TaskModelDependency(name, meta, placement))
-    }
-
-    fun dependsOn(task: Task<*>, meta: Meta = this.meta, placement: Name = EmptyName) =
-        dependsOn(task.name, meta, placement)
-
-    fun dependsOn(task: Task<*>, placement: Name = EmptyName, metaBuilder: MetaBuilder.() -> Unit) =
-        dependsOn(task.name, buildMeta(metaBuilder), placement)
-
-    /**
-     * Add custom data dependency
-     */
-    fun data(action: DataFilter.() -> Unit) {
-        dependencies.add(DataDependency(DataFilter.build(action)))
-    }
-
-    /**
-     * User-friendly way to add data dependency
-     */
-    fun data(pattern: String? = null, from: String? = null, to: String? = null) = data {
-        pattern?.let { this.pattern = it }
-        from?.let { this.from = it }
-        to?.let { this.to = it }
-    }
-
-    /**
-     * Add all data as root node
-     */
-    fun allData(to: Name = EmptyName) {
-        dependencies.add(AllDataDependency(to))
-    }
 
     fun build(): TaskModel = TaskModel(name, meta.seal(), dependencies)
 }

dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/Workspace.kt

@@ -1,6 +1,8 @@
 package hep.dataforge.workspace
 
+import hep.dataforge.context.Context
 import hep.dataforge.context.ContextAware
+import hep.dataforge.context.Global
 import hep.dataforge.data.Data
 import hep.dataforge.data.DataNode
 import hep.dataforge.data.dataSequence
@@ -56,6 +58,8 @@ interface Workspace : ContextAware, Provider {
 
     companion object {
         const val TYPE = "workspace"
+        operator fun invoke(parent: Context = Global, block: SimpleWorkspaceBuilder.() -> Unit): SimpleWorkspace =
+            SimpleWorkspaceBuilder(parent).apply(block).build()
     }
 }
 
@@ -73,3 +77,6 @@ fun Workspace.run(task: String, meta: Meta) =
 
 fun Workspace.run(task: String, block: MetaBuilder.() -> Unit = {}) =
     run(task, buildMeta(block))
+
+fun <T: Any> Workspace.run(task: Task<T>, metaBuilder: MetaBuilder.() -> Unit = {}): DataNode<T> =
+    run(task, buildMeta(metaBuilder))

dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/WorkspaceBuilder.kt

@@ -2,12 +2,15 @@ package hep.dataforge.workspace
 
 import hep.dataforge.context.Context
 import hep.dataforge.context.ContextBuilder
-import hep.dataforge.data.Data
 import hep.dataforge.data.DataNode
 import hep.dataforge.data.DataTreeBuilder
 import hep.dataforge.meta.*
+import hep.dataforge.names.EmptyName
 import hep.dataforge.names.Name
+import hep.dataforge.names.isEmpty
 import hep.dataforge.names.toName
+import kotlin.jvm.JvmName
+import kotlin.reflect.KClass
 
 @TaskBuildScope
 interface WorkspaceBuilder {
@@ -28,32 +31,26 @@ fun WorkspaceBuilder.context(name: String = "WORKSPACE", block: ContextBuilder.(
     context = ContextBuilder(name, parentContext).apply(block).build()
 }
 
-fun WorkspaceBuilder.data(name: Name, data: Data<Any>) {
+inline fun <reified T : Any> WorkspaceBuilder.data(
-    this.data[name] = data
+    name: Name = EmptyName,
+    noinline block: DataTreeBuilder<T>.() -> Unit
+): DataNode<T> {
+    val node = DataTreeBuilder(T::class).apply(block)
+    if (name.isEmpty()) {
+        @Suppress("UNCHECKED_CAST")
+        data = node as DataTreeBuilder<Any>
+    } else {
+        data[name] = node
+    }
+    return node.build()
 }
 
-fun WorkspaceBuilder.data(name: String, data: Data<Any>) = data(name.toName(), data)
+@JvmName("rawData")
+fun WorkspaceBuilder.data(
+    name: Name = EmptyName,
+    block: DataTreeBuilder<Any>.() -> Unit
+): DataNode<Any> = data<Any>(name, block)
 
-fun WorkspaceBuilder.static(name: Name, data: Any, meta: Meta = EmptyMeta) =
-    data(name, Data.static(data, meta))
-
-fun WorkspaceBuilder.static(name: Name, data: Any, block: MetaBuilder.() -> Unit = {}) =
-    data(name, Data.static(data, buildMeta(block)))
-
-fun WorkspaceBuilder.static(name: String, data: Any, block: MetaBuilder.() -> Unit = {}) =
-    data(name, Data.static(data, buildMeta(block)))
-
-fun WorkspaceBuilder.data(name: Name, node: DataNode<Any>) {
-    this.data[name] = node
-}
-
-fun WorkspaceBuilder.data(name: String, node: DataNode<Any>) = data(name.toName(), node)
-
-fun WorkspaceBuilder.data(name: Name, block: DataTreeBuilder<Any>.() -> Unit) {
-    this.data[name] = DataNode.build(Any::class, block)
-}
-
-fun WorkspaceBuilder.data(name: String, block: DataTreeBuilder<Any>.() -> Unit) = data(name.toName(), block)
 
 fun WorkspaceBuilder.target(name: String, block: MetaBuilder.() -> Unit) {
     targets[name] = buildMeta(block).seal()
@@ -65,22 +62,34 @@ fun WorkspaceBuilder.target(name: String, block: MetaBuilder.() -> Unit) {
 fun WorkspaceBuilder.target(name: String, base: String, block: MetaBuilder.() -> Unit) {
     val parentTarget = targets[base] ?: error("Base target with name $base not found")
     targets[name] = parentTarget.builder()
-        .apply { "@baseTarget" to base }
+        .apply { "@baseTarget" put base }
         .apply(block)
         .seal()
 }
 
-fun WorkspaceBuilder.task(task: Task<Any>) {
+fun <T : Any> WorkspaceBuilder.task(
-    this.tasks.add(task)
+    name: String,
-}
+    type: KClass<out T>,
+    builder: TaskBuilder<T>.() -> Unit
+): Task<T> = TaskBuilder(name.toName(), type).apply(builder).build().also { tasks.add(it) }
 
+inline fun <reified T : Any> WorkspaceBuilder.task(
+    name: String,
+    noinline builder: TaskBuilder<T>.() -> Unit
+): Task<T> = task(name, T::class, builder)
+
+@JvmName("rawTask")
+fun WorkspaceBuilder.task(
+    name: String,
+    builder: TaskBuilder<Any>.() -> Unit
+): Task<Any> = task(name, Any::class, builder)
 
 /**
  * A builder for a simple workspace
  */
 class SimpleWorkspaceBuilder(override val parentContext: Context) : WorkspaceBuilder {
     override var context: Context = parentContext
-    override var data = DataTreeBuilder(Any::class)
+    override var data: DataTreeBuilder<Any> = DataTreeBuilder(Any::class)
     override var tasks: MutableSet<Task<Any>> = HashSet()
     override var targets: MutableMap<String, Meta> = HashMap()
 

dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/WorkspacePlugin.kt

@@ -1,19 +1,42 @@
 package hep.dataforge.workspace
 
 import hep.dataforge.context.AbstractPlugin
+import hep.dataforge.context.toMap
 import hep.dataforge.names.Name
 import hep.dataforge.names.toName
+import kotlin.reflect.KClass
 
 /**
  * An abstract plugin with some additional boilerplate to effectively work with workspace context
  */
 abstract class WorkspacePlugin : AbstractPlugin() {
-    abstract val tasks: Collection<Task<*>>
+    private val _tasks = HashSet<Task<*>>()
+    val tasks: Collection<Task<*>> get() = _tasks
 
     override fun provideTop(target: String): Map<Name, Any> {
-        return when(target){
+        return when (target) {
-            Task.TYPE -> tasks.associateBy { it.name.toName() }
+            Task.TYPE -> tasks.toMap()
             else -> emptyMap()
         }
     }
+
+    fun task(task: Task<*>){
+        _tasks.add(task)
+    }
+
+    fun <T : Any> task(
+        name: String,
+        type: KClass<out T>,
+        builder: TaskBuilder<T>.() -> Unit
+    ): GenericTask<T> = TaskBuilder(name.toName(), type).apply(builder).build().also {
+        _tasks.add(it)
+    }
+
+    inline fun <reified T : Any> task(
+        name: String,
+        noinline builder: TaskBuilder<T>.() -> Unit
+    ) = task(name, T::class, builder)
+
+//
+////TODO add delegates to build gradle-like tasks
 }

dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/fileData.kt

@@ -1,12 +1,14 @@
 package hep.dataforge.workspace
 
 import hep.dataforge.data.Data
+import hep.dataforge.data.DataNode
+import hep.dataforge.data.DataTreeBuilder
+import hep.dataforge.data.datum
 import hep.dataforge.descriptors.NodeDescriptor
 import hep.dataforge.io.*
 import hep.dataforge.meta.EmptyMeta
 import hep.dataforge.meta.Meta
 import kotlinx.coroutines.Dispatchers
-import kotlinx.coroutines.coroutineScope
 import kotlinx.coroutines.withContext
 import kotlinx.io.nio.asInput
 import kotlinx.io.nio.asOutput
@@ -16,71 +18,94 @@ import java.nio.file.StandardOpenOption
 import kotlin.reflect.KClass
 
 /**
- * Read meta from file in a given [format]
+ * Read meta from file in a given [MetaFormat]
  */
-suspend fun Path.readMeta(format: MetaFormat, descriptor: NodeDescriptor? = null): Meta {
+fun MetaFormat.readMetaFile(path: Path, descriptor: NodeDescriptor? = null): Meta {
-    return withContext(Dispatchers.IO) {
+    return Files.newByteChannel(path, StandardOpenOption.READ)
-        format.run {
+        .asInput()
-            Files.newByteChannel(this@readMeta, StandardOpenOption.READ)
+        .readMeta(descriptor)
-                .asInput()
-                .readMeta(descriptor)
-        }
-    }
 }
 
 /**
- * Write meta to file in a given [format]
+ * Write meta to file using given [MetaFormat]
  */
-suspend fun Meta.write(path: Path, format: MetaFormat, descriptor: NodeDescriptor? = null) {
+fun MetaFormat.writeMetaFile(path: Path, meta: Meta, descriptor: NodeDescriptor? = null) {
-    withContext(Dispatchers.IO) {
+    return Files.newByteChannel(path, StandardOpenOption.WRITE, StandardOpenOption.CREATE_NEW)
-        format.run {
+        .asOutput()
-            Files.newByteChannel(path, StandardOpenOption.WRITE, StandardOpenOption.CREATE_NEW)
+        .writeMeta(meta, descriptor)
-                .asOutput()
-                .writeMeta(this@write, descriptor)
-        }
-    }
 }
 
 /**
  * Read data with supported envelope format and binary format. If envelope format is null, then read binary directly from file.
+ * The operation is blocking since it must read meta header. The reading of envelope body is lazy
  * @param type explicit type of data read
- * @param format binary format
+ * @param dataFormat binary format
- * @param envelopeFormat the format of envelope. If null, file is read directly
+ * @param envelopeFormatFactory the format of envelope. If null, file is read directly
  * @param metaFile the relative file for optional meta override
  * @param metaFileFormat the meta format for override
  */
-suspend fun <T : Any> Path.readData(
+fun <T : Any> IOPlugin.readData(
+    path: Path,
     type: KClass<out T>,
-    format: IOFormat<T>,
+    dataFormat: IOFormat<T>,
-    envelopeFormat: EnvelopeFormat? = null,
+    envelopeFormatFactory: EnvelopeFormatFactory? = null,
-    metaFile: Path = resolveSibling("$fileName.meta"),
+    metaFile: Path = path.resolveSibling("${path.fileName}.meta"),
-    metaFileFormat: MetaFormat = JsonMetaFormat
+    metaFileFormat: MetaFormat = JsonMetaFormat.default
 ): Data<T> {
-    return coroutineScope {
+    val externalMeta = if (Files.exists(metaFile)) {
-        val externalMeta = if (Files.exists(metaFile)) {
+        metaFileFormat.readMetaFile(metaFile)
-            metaFile.readMeta(metaFileFormat)
+    } else {
-        } else {
+        null
-            null
+    }
-        }
+    return if (envelopeFormatFactory == null) {
-        if (envelopeFormat == null) {
+        Data(type, externalMeta ?: EmptyMeta) {
-            Data(type, externalMeta ?: EmptyMeta) {
+            withContext(Dispatchers.IO) {
-                withContext(Dispatchers.IO) {
+                dataFormat.run {
-                    format.run {
+                    Files.newByteChannel(path, StandardOpenOption.READ)
-                        Files.newByteChannel(this@readData, StandardOpenOption.READ)
+                        .asInput()
-                            .asInput()
+                        .readObject()
-                            .readThis()
-                    }
                 }
             }
-        } else {
+        }
-            withContext(Dispatchers.IO) {
+    } else {
-                readEnvelope(envelopeFormat).let {
+        readEnvelopeFile(path, envelopeFormatFactory).let {
-                    if (externalMeta == null) {
+            if (externalMeta == null) {
-                        it
+                it
-                    } else {
+            } else {
-                        it.withMetaLayers(externalMeta)
+                it.withMetaLayers(externalMeta)
-                    }
+            }
-                }.toData(type, format)
+        }.toData(type, dataFormat)
+    }
+}
+
+//TODO wants multi-receiver
+fun <T : Any> DataTreeBuilder<T>.file(
+    plugin: IOPlugin,
+    path: Path,
+    dataFormat: IOFormat<T>,
+    envelopeFormatFactory: EnvelopeFormatFactory? = null
+) {
+    plugin.run {
+        val data = readData(path, type, dataFormat, envelopeFormatFactory)
+        val name = path.fileName.toString().replace(".df", "")
+        datum(name, data)
+    }
+}
+
+/**
+ * Read the directory as a data node
+ */
+fun <T : Any> IOPlugin.readDataNode(
+    path: Path,
+    type: KClass<out T>,
+    dataFormat: IOFormat<T>,
+    envelopeFormatFactory: EnvelopeFormatFactory? = null
+): DataNode<T> {
+    if (!Files.isDirectory(path)) error("Provided path $this is not a directory")
+    return DataNode(type) {
+        Files.list(path).forEach { path ->
+            if (!path.fileName.toString().endsWith(".meta")) {
+                file(this@readDataNode,path, dataFormat, envelopeFormatFactory)
             }
         }
     }

dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/DataPropagationTest.kt

@@ -0,0 +1,95 @@
+package hep.dataforge.workspace
+
+import hep.dataforge.context.Context
+import hep.dataforge.context.PluginFactory
+import hep.dataforge.context.PluginTag
+import hep.dataforge.data.*
+import hep.dataforge.meta.Meta
+import hep.dataforge.names.asName
+import org.junit.Test
+import kotlin.reflect.KClass
+import kotlin.test.assertEquals
+
+
+class DataPropagationTestPlugin : WorkspacePlugin() {
+    override val tag: PluginTag = Companion.tag
+
+    val testAllData = task("allData", Int::class) {
+        model {
+            allData()
+        }
+        transform<Int> { data ->
+            return@transform DataNode {
+                val result = data.dataSequence().map { it.second.get() }.reduce { acc, pair -> acc + pair }
+                set("result".asName(), Data { result })
+            }
+        }
+    }
+
+
+    val testSingleData = task("singleData", Int::class) {
+        model {
+            data("myData\\[12\\]")
+        }
+        transform<Int> { data ->
+            return@transform DataNode {
+                val result = data.dataSequence().map { it.second.get() }.reduce { acc, pair -> acc + pair }
+                set("result".asName(), Data { result })
+            }
+        }
+    }
+
+    val testAllRegexData = task("allRegexData", Int::class) {
+        model {
+            data(pattern = "myData.*")
+        }
+        transform<Int> { data ->
+            return@transform DataNode {
+                val result = data.dataSequence().map { it.second.get() }.reduce { acc, pair -> acc + pair }
+                set("result".asName(), Data { result })
+            }
+        }
+    }
+
+
+    companion object : PluginFactory<DataPropagationTestPlugin> {
+
+        override val type: KClass<out DataPropagationTestPlugin> = DataPropagationTestPlugin::class
+
+        override fun invoke(meta: Meta, context: Context): DataPropagationTestPlugin =
+            DataPropagationTestPlugin(meta)
+
+        override val tag: PluginTag = PluginTag("Test")
+    }
+}
+
+class DataPropagationTest {
+    val testWorkspace = Workspace {
+        context {
+            plugin(DataPropagationTestPlugin())
+        }
+        data {
+            repeat(100) {
+                static("myData[$it]", it)
+            }
+        }
+    }
+
+    @Test
+    fun testAllData() {
+        val node = testWorkspace.run("Test.allData")
+        assertEquals(4950, node.first()!!.get())
+    }
+
+    @Test
+    fun testAllRegexData() {
+        val node = testWorkspace.run("Test.allRegexData")
+        assertEquals(4950, node.first()!!.get())
+    }
+
+    @Test
+    fun testSingleData() {
+        val node = testWorkspace.run("Test.singleData")
+        assertEquals(12, node.first()!!.get())
+    }
+}

dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/SimpleWorkspaceTest.kt

@@ -1,10 +1,12 @@
 package hep.dataforge.workspace
 
 import hep.dataforge.context.PluginTag
-import hep.dataforge.data.first
+import hep.dataforge.data.*
-import hep.dataforge.data.get
 import hep.dataforge.meta.boolean
+import hep.dataforge.meta.builder
 import hep.dataforge.meta.get
+import hep.dataforge.meta.int
+import hep.dataforge.names.plus
 import org.junit.Test
 import kotlin.test.assertEquals
 import kotlin.test.assertTrue
@@ -14,30 +16,36 @@ class SimpleWorkspaceTest {
     val testPlugin = object : WorkspacePlugin() {
         override val tag: PluginTag = PluginTag("test")
 
-        val contextTask = Workspace.task("test") {
+        val contextTask = task("test", Any::class) {
-            pipe<Any, Unit> {
+            map<Any> {
                 context.logger.info { "Test: $it" }
             }
         }
-        override val tasks: Collection<Task<*>> = listOf(contextTask)
     }
 
-    val workspace = SimpleWorkspace.build {
+    val workspace = Workspace {
 
         context {
             plugin(testPlugin)
         }
 
-        repeat(100) {
+        data {
-            static("myData[$it]", it)
+            repeat(100) {
+                static("myData[$it]", it)
+            }
         }
 
-
+        val filterTask = task<Int>("filterOne") {
-        task("square") {
             model {
-                allData()
+                data("myData\\[12\\]")
             }
-            pipe<Int, Int> { data ->
+            map<Int>{
+                it
+            }
+        }
+
+        val square = task<Int>("square") {
+            map<Int> { data ->
                 if (meta["testFlag"].boolean == true) {
                     println("flag")
                 }
@@ -46,30 +54,55 @@ class SimpleWorkspaceTest {
             }
         }
 
-        task("sum") {
+        val linear = task<Int>("linear") {
-            model {
+            map<Int> { data ->
-                dependsOn("square")
+                context.logger.info { "Starting linear on $data" }
+                data * 2 + 1
             }
-            join<Int, Int> { data ->
+        }
+
+        val fullSquare = task<Int>("fullsquare") {
+            model {
+                val squareDep = dependsOn(square, placement = "square")
+                val linearDep = dependsOn(linear, placement = "linear")
+            }
+            transform { data ->
+                val squareNode = data["square"].node!!.cast<Int>()//squareDep()
+                val linearNode = data["linear"].node!!.cast<Int>()//linearDep()
+                return@transform DataNode(Int::class) {
+                    squareNode.dataSequence().forEach { (name, _) ->
+                        val newData = Data {
+                            val squareValue = squareNode[name].data!!.get()
+                            val linearValue = linearNode[name].data!!.get()
+                            squareValue + linearValue
+                        }
+                        set(name, newData)
+                    }
+                }
+            }
+        }
+
+        task<Int>("sum") {
+            model {
+                dependsOn(square)
+            }
+            reduce<Int> { data ->
                 context.logger.info { "Starting sum" }
                 data.values.sum()
             }
         }
 
-        task("average") {
+        val average = task<Double>("average") {
-            model {
+            reduceByGroup<Int> { env ->
-                allData()
-            }
-            joinByGroup<Int, Double> { context ->
                 group("even", filter = { name, _ -> name.toString().toInt() % 2 == 0 }) {
                     result { data ->
-                        context.logger.info { "Starting even" }
+                        env.context.logger.info { "Starting even" }
                         data.values.average()
                     }
                 }
                 group("odd", filter = { name, _ -> name.toString().toInt() % 2 == 1 }) {
                     result { data ->
-                        context.logger.info { "Starting odd" }
+                        env.context.logger.info { "Starting odd" }
                         data.values.average()
                     }
                 }
@@ -78,13 +111,25 @@ class SimpleWorkspaceTest {
 
         task("delta") {
             model {
-                dependsOn("average")
+                dependsOn(average)
             }
-            join<Double, Double> { data ->
+            reduce<Double> { data ->
                 data["even"]!! - data["odd"]!!
             }
         }
 
+        val customPipeTask = task<Int>("custom") {
+            mapAction<Int> {
+                meta = meta.builder().apply {
+                    "newValue" put 22
+                }
+                name += "new"
+                result {
+                    meta["value"].int ?: 0 + it
+                }
+            }
+        }
+
         target("empty") {}
     }
 
@@ -97,7 +142,7 @@ class SimpleWorkspaceTest {
 
     @Test
     fun testMetaPropagation() {
-        val node = workspace.run("sum") { "testFlag" to true }
+        val node = workspace.run("sum") { "testFlag" put true }
         val res = node.first()?.get()
     }
 
@@ -107,4 +152,16 @@ class SimpleWorkspaceTest {
         assertTrue { tasks["test.test"] != null }
         //val node = workspace.run("test.test", "empty")
     }
+
+    @Test
+    fun testFullSquare() {
+        val node = workspace.run("fullsquare")
+        println(node.toMeta())
+    }
+
+    @Test
+    fun testGather() {
+        val node = workspace.run("filterOne")
+        assertEquals(12, node.first()?.get())
+    }
 }

gradle/wrapper/gradle-wrapper.properties

@@ -1,5 +1,5 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-5.1.1-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-5.6.2-bin.zip
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists