Merge pull request '0.7.0' (!79) from dev into master

Reviewed-on: #79

.github/workflows/build.yml

@@ -1,19 +1,41 @@
 name: Gradle build
 
-on: [ push ]
+on:
+  push:
+    branches: [ dev, master ]
+  pull_request:
 
 jobs:
   build:
-
-    runs-on: ubuntu-latest
-
+    strategy:
+      matrix:
+        os: [ macOS-latest, windows-latest ]
+    runs-on: ${{matrix.os}}
+    timeout-minutes: 40
     steps:
-      - uses: actions/checkout@v2
+      - name: Checkout the repo
+        uses: actions/checkout@v2
       - name: Set up JDK 11
-        uses: actions/setup-java@v1
+        uses: DeLaGuardo/setup-graalvm@4.0
         with:
-          java-version: 11
-      - name: Grant execute permission for gradlew
-        run: chmod +x gradlew
-      - name: Build with Gradle
-        run: ./gradlew build
+          graalvm: 21.2.0
+          java: java11
+          arch: amd64
+      - name: Cache gradle
+        uses: actions/cache@v2
+        with:
+          path: |
+            ~/.gradle/caches
+            ~/.gradle/wrapper
+          key: ${{ runner.os }}-gradle-${{ hashFiles('*.gradle.kts') }}
+          restore-keys: |
+            ${{ runner.os }}-gradle-
+      - name: Cache konan
+        uses: actions/cache@v2
+        with:
+          path: ~/.konan
+          key: ${{ runner.os }}-gradle-${{ hashFiles('*.gradle.kts') }}
+          restore-keys: |
+            ${{ runner.os }}-gradle-
+      - name: Build
+        run: ./gradlew build --build-cache --no-daemon --stacktrace

.github/workflows/pages.yml

@@ -0,0 +1,31 @@
+name: Dokka publication
+
+on:
+  workflow_dispatch:
+  release:
+    types: [ created ]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    timeout-minutes: 40
+    steps:
+      - uses: actions/checkout@v3.0.0
+      - uses: actions/setup-java@v3.0.0
+        with:
+          java-version: 11
+          distribution: liberica
+      - name: Cache konan
+        uses: actions/cache@v3.0.1
+        with:
+          path: ~/.konan
+          key: ${{ runner.os }}-gradle-${{ hashFiles('*.gradle.kts') }}
+          restore-keys: |
+            ${{ runner.os }}-gradle-
+      - uses: gradle/gradle-build-action@v2.4.2
+        with:
+          arguments: dokkaHtmlMultiModule --no-parallel
+      - uses: JamesIves/github-pages-deploy-action@v4.3.0
+        with:
+          branch: gh-pages
+          folder: build/dokka/htmlMultiModule

.github/workflows/publish.yml

@@ -1,40 +1,50 @@
-name: Bintray Publish
+name: Gradle publish
 
 on:
+  workflow_dispatch:
   release:
-    types:
-      - created
+    types: [ created ]
 
 jobs:
-  build-on-windows:
-    runs-on: windows-latest
+  publish:
+    environment:
+      name: publish
+    strategy:
+      matrix:
+        os: [ macOS-latest, windows-latest ]
+    runs-on: ${{matrix.os}}
     steps:
-      - uses: actions/checkout@v2
-      - name: Set up JDK 11
-        uses: actions/setup-java@v1
+      - uses: actions/checkout@v3.0.0
+      - uses: actions/setup-java@v3.10.0
         with:
           java-version: 11
-      - name: Grant execute permission for gradlew
-        run: chmod +x gradlew
-      - name: Gradle clean
-        run: ./gradlew clean
-      - name: Gradle build
-        run: ./gradlew build
-      - name: Run release task
-        run: ./gradlew release -PbintrayUser=${{ secrets.BINTRAY_USER }} -PbintrayApiKey=${{ secrets.BINTRAY_KEY }}
-  build-on-macos:
-    runs-on: macos-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: Set up JDK 11
-        uses: actions/setup-java@v1
+          distribution: liberica
+      - name: Cache konan
+        uses: actions/cache@v3.0.1
         with:
-          java-version: 11
-      - name: Grant execute permission for gradlew
-        run: chmod +x gradlew
-      - name: Gradle clean
-        run: ./gradlew clean
-      - name: Gradle build
-        run: ./gradlew build
-      - name: Run release task
-        run: ./gradlew release -PbintrayUser=${{ secrets.BINTRAY_USER }} -PbintrayApiKey=${{ secrets.BINTRAY_KEY }}
+          path: ~/.konan
+          key: ${{ runner.os }}-gradle-${{ hashFiles('*.gradle.kts') }}
+          restore-keys: |
+            ${{ runner.os }}-gradle-
+      - name: Publish Windows Artifacts
+        if: matrix.os == 'windows-latest'
+        uses: gradle/gradle-build-action@v2.4.2
+        with:
+          arguments: |
+            publishAllPublicationsToSpaceRepository
+            -Ppublishing.targets=all
+            -Ppublishing.space.user=${{ secrets.SPACE_APP_ID }}
+            -Ppublishing.space.token=${{ secrets.SPACE_APP_SECRET }}
+      - name: Publish Mac Artifacts
+        if: matrix.os == 'macOS-latest'
+        uses: gradle/gradle-build-action@v2.4.2
+        with:
+          arguments: |
+            publishMacosX64PublicationToSpaceRepository
+            publishMacosArm64PublicationToSpaceRepository
+            publishIosX64PublicationToSpaceRepository
+            publishIosArm64PublicationToSpaceRepository
+            publishIosSimulatorArm64PublicationToSpaceRepository
+            -Ppublishing.targets=all
+            -Ppublishing.space.user=${{ secrets.SPACE_APP_ID }}
+            -Ppublishing.space.token=${{ secrets.SPACE_APP_SECRET }}

CHANGELOG.md

@@ -1,6 +1,7 @@
 # Changelog
 
-## [Unreleased]
+## Unreleased
+
 ### Added
 
 ### Changed
@@ -13,8 +14,160 @@
 
 ### Security
 
-## [0.3.0]
+## 0.7.0 - 2023-11-26
+
 ### Added
+
+- Obligatory `type: KType` and `descriptor` property for `MetaConverters`
+- Added separate `Meta`, `SealedMeta` and `ObservableMutableMeta` builders.
+
+### Changed
+
+- Meta converter `metaToObject` returns a non-nullable type. Additional method `metaToObjectOrNull` for nullable return.
+- Kotlin 1.9.20.
+- Migrated from ktor-io to kotlinx-io.
+- `MutableMeta` builder now returns a simplified version of meta that does not hold listeners.
+- More concise names for read/write methods in IO.
+- Remove unnecessary confusion with `get`/`getMeta` by removing `getMeta` from the interface.
+
+### Deprecated
+
+- `String.parseValue` is replaced with `Value.parse`
+
+### Fixed
+
+- Memory leak in SealedMeta builder
+
+## 0.6.2 - 2023-07-29
+
+### Changed
+
+- Meta to Json serializer now serializes a single item with index as an array. It is important for plotly integration.
+- Meta to Json serializes Meta without children a value as literal or array instead of an object with `@value` field.
+
+## 0.6.1 - 2023-03-31
+
+### Added
+
+- File cache for workspace
+- Smart task metadata transformation for workspace
+- Add `readOnly` property to descriptors
+- Add `specOrNull` delegate to meta and Scheme
+- Suspended read methods to the `Binary`
+- Synchronously accessed `meta` to all `DataSet`s
+- More fine-grained types in Action builders.
+
+### Changed
+
+- `Name::replaceLast` API
+- `PluginFactory` no longer requires plugin class
+- Collection<Named> toMap -> associateByName
+- Simplified `DFTL` envelope format. Closing symbols are unnecessary. Properties are discontinued.
+- Meta `get` method allows nullable receiver
+- `withDefault` functions do not add new keys to meta children and are consistent.
+- `dataforge.meta.values` package is merged into `dataforge.meta` for better star imports
+- Kotlin 1.8.20
+- `Factory` is now `fun interface` and uses `build` instead of `invoke`. `invoke moved to an extension.
+- KTor 2.0
+- DataTree `items` call is blocking.
+- DataSet `getData` is no longer suspended and renamed to `get`
+- DataSet operates with sequences of data instead of flows
+- PartialEnvelope uses `Int` instead `UInt`.
+- `ActiveDataSet` renamed to `DataSource`
+- `selectOne`->`getByType`
+- Data traversal in `DataSet` is done via iterator
+- Remove all unnecessary properties for `IOFormat`
+- Separate interfaces for `IOReader` and `IOWriter`
+
+### Deprecated
+
+- Context.fetch -> Context.request
+
+### Fixed
+
+- `readDataDirectory` does not split names with dots
+- Front matter reader does not crash on non-UTF files
+- Meta file name in readMeta from directory
+- Tagless and FrontMatter envelope partial readers fix.
+
+## 0.5.2
+
+### Added
+
+- Yaml plugin
+- Partial fix to #53
+
+### Fixed
+
+- MutableMetaImpl attachment and checks
+- Listeners in observable meta are replaced by lists
+- JS number comparison bug.
+
+## 0.5.0
+
+### Added
+
+- Experimental `listOfSpec` delegate.
+
+### Changed
+
+- **API breaking** Config is deprecated, use `ObservableMeta` instead.
+- **API breaking** Descriptor no has a member property `defaultValue` instead of `defaultItem()` extension. It caches default value state on the first call. It is done because computing default on each call is too expensive.
+- Kotlin 1.5.10
+- Build tools 0.10.0
+- Relaxed type restriction on `MetaConverter`. Now nullables are available.
+- **Huge API-breaking refactoring of Meta**. Meta now can have both value and children. There is only one kind of descriptor now.
+- **API breaking** `String.toName()` is replaced by `Name.parse()`
+- **API breaking** Configurable`config` changed to `meta`
+
+### Removed
+
+- `Config`
+- Public PluginManager mutability
+- Tables and tables-exposed moved to the separate project `tables.kt`
+- BinaryMetaFormat. Use CBOR encoding instead
+
+### Fixed
+
+- Proper json array index treatment.
+- Proper json index for single-value array.
+
+## 0.4.0
+
+### Added
+
+- LogManager plugin
+- dataforge-context API dependency on SLF4j
+- Context `withEnv` and `fetch` methods to manipulate plugins without changing plugins after creation.
+- Split `ItemDescriptor` into builder and read-only part
+
+### Changed
+
+- Kotlin-logging moved from common to JVM and JS. Replaced by console for native.
+- Package changed to `space.kscience`
+- Scheme made observable
+- Global context is a variable (the singleton is hidden and will be deprecated in future)
+- Kotlin 1.5
+- Added blank builders for children context.
+- Refactor loggers
+
+### Deprecated
+
+- Direct use of PluginManager
+
+### Removed
+
+- Common dependency on Kotlin-logging
+- Kotlinx-io fork dependency. Replaced by Ktor-io.
+
+### Fixed
+
+- Scheme properties properly handle children property change.
+
+## 0.3.0
+
+### Added
+
 - Yaml meta format based on yaml.kt
 - `Path` builders
 - Special ValueType for lists
@@ -22,6 +175,7 @@
 - Multiplatform yaml meta
 
 ### Changed
+
 - `ListValue` and `DoubleArrayValue` implement `Iterable`.
 - Changed the logic of `Value::isList` to check for type instead of size
 - `Meta{}` builder made inline
@@ -35,18 +189,10 @@
 - \[Major breaking change\] Full refactor of DataTree/DataSource
 - \[Major Breaking change\] Replace KClass with KType in data. Remove direct access to constructors with types.
 
-### Deprecated
-
-### Removed
-
-### Fixed
-
-### Security
-
-## [0.2.0]
-### Added
+## 0.2.0
 
 ### Changed
+
 - Context content resolution refactor
 - Kotlin 1.4.10 (build tools 0.6.0)
 - Empty query in Name is null instead of ""
@@ -56,15 +202,16 @@
 - Configurable is no longer MutableItemProvider. All functionality moved to Scheme.
 
 ### Deprecated
+
 - Context activation API
 - TextRenderer
 
 ### Removed
+
 - Functional server prototype
 - `dataforge-output` module
 
 ### Fixed
+
 - Global context CoroutineScope resolution
 - Library mode compliance
-
-### Security

LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

README.md

@@ -3,55 +3,36 @@
 
 ![Gradle build](https://github.com/mipt-npm/dataforge-core/workflows/Gradle%20build/badge.svg)
 
-[ ![Download](https://api.bintray.com/packages/mipt-npm/dataforge/dataforge-meta/images/download.svg) ](https://bintray.com/mipt-npm/dataforge/dataforge-meta/_latestVersion)
 
-<hr/>
-
-* ### [dataforge-context](dataforge-context)
-> 
+### [dataforge-context](dataforge-context)
+> Context and provider definitions
 >
 > **Maturity**: DEVELOPMENT
-<hr/>
 
-* ### [dataforge-data](dataforge-data)
-> 
+### [dataforge-data](dataforge-data)
 >
 > **Maturity**: EXPERIMENTAL
-<hr/>
 
-* ### [dataforge-io](dataforge-io)
-> 
+### [dataforge-io](dataforge-io)
+> IO module
 >
-> **Maturity**: PROTOTYPE
-<hr/>
+> **Maturity**: EXPERIMENTAL
 
-* ### [dataforge-meta](dataforge-meta)
-> 
+### [dataforge-meta](dataforge-meta)
+> Meta definition and basic operations on meta
 >
 > **Maturity**: DEVELOPMENT
-<hr/>
 
-* ### [dataforge-scripting](dataforge-scripting)
-> 
+### [dataforge-scripting](dataforge-scripting)
 >
 > **Maturity**: PROTOTYPE
-<hr/>
 
-* ### [dataforge-tables](dataforge-tables)
-> 
->
-> **Maturity**: PROTOTYPE
-<hr/>
-
-* ### [dataforge-workspace](dataforge-workspace)
-> 
+### [dataforge-workspace](dataforge-workspace)
 >
 > **Maturity**: EXPERIMENTAL
-<hr/>
 
-* ### [dataforge-io-yaml](dataforge-io-yaml)
+### [dataforge-io/dataforge-io-yaml](dataforge-io/dataforge-io-yaml)
 > YAML meta converters and Front Matter envelope format
 >
 > **Maturity**: PROTOTYPE
-<hr/>
 

build.gradle.kts

@@ -1,20 +1,24 @@
+import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
+import space.kscience.gradle.useApache2Licence
+import space.kscience.gradle.useSPCTeam
+
 plugins {
-    id("ru.mipt.npm.project")
+    id("space.kscience.gradle.project")
 }
 
-val dataforgeVersion by extra("0.3.0")
-
-
-
 allprojects {
-    group = "hep.dataforge"
-    version = dataforgeVersion
-
-    apply<org.jetbrains.dokka.gradle.DokkaPlugin>()
+    group = "space.kscience"
+    version = "0.7.0"
 }
 
 subprojects {
-    apply(plugin = "ru.mipt.npm.publish")
+    apply(plugin = "maven-publish")
+
+    tasks.withType<KotlinCompile> {
+        kotlinOptions {
+            freeCompilerArgs = freeCompilerArgs + "-Xcontext-receivers"
+        }
+    }
 }
 
 readme {
@@ -22,7 +26,14 @@ readme {
 }
 
 ksciencePublish {
-    bintrayRepo = "dataforge"
-    githubProject = "dataforge-core"
-    spaceRepo = "https://maven.jetbrains.space/mipt-npm/p/df/maven"
+    pom("https://github.com/SciProgCentre/kmath") {
+        useApache2Licence()
+        useSPCTeam()
+    }
+    repository("spc","https://maven.sciprog.center/kscience")
+    sonatype()
+}
+
+apiValidation {
+    nonPublicMarkers.add("space.kscience.dataforge.misc.DFExperimental")
 }

dataforge-context/README.md

@@ -0,0 +1,23 @@
+# Module dataforge-context
+
+Context and provider definitions
+
+## Usage
+
+## Artifact:
+
+The Maven coordinates of this project are `space.kscience:dataforge-context:0.7.0`.
+
+**Gradle Kotlin DSL:**
+```kotlin
+repositories {
+    maven("https://repo.kotlin.link")
+    //uncomment to access development builds
+    //maven("https://maven.pkg.jetbrains.space/spc/p/sci/dev")
+    mavenCentral()
+}
+
+dependencies {
+    implementation("space.kscience:dataforge-context:0.7.0")
+}
+```

dataforge-context/api/dataforge-context.api

@@ -1,254 +1,267 @@
-public abstract class hep/dataforge/context/AbstractPlugin : hep/dataforge/context/Plugin {
+public abstract class space/kscience/dataforge/context/AbstractPlugin : space/kscience/dataforge/context/Plugin {
 	public fun <init> ()V
-	public fun <init> (Lhep/dataforge/meta/Meta;)V
-	public synthetic fun <init> (Lhep/dataforge/meta/Meta;ILkotlin/jvm/internal/DefaultConstructorMarker;)V
-	public fun attach (Lhep/dataforge/context/Context;)V
-	public fun content (Ljava/lang/String;)Ljava/util/Map;
-	public final fun dependsOn ()Ljava/util/Map;
+	public fun <init> (Lspace/kscience/dataforge/meta/Meta;)V
+	public synthetic fun <init> (Lspace/kscience/dataforge/meta/Meta;ILkotlin/jvm/internal/DefaultConstructorMarker;)V
+	public fun attach (Lspace/kscience/dataforge/context/Context;)V
+	public fun dependsOn ()Ljava/util/Map;
 	public fun detach ()V
-	public fun getContext ()Lhep/dataforge/context/Context;
-	public fun getDefaultChainTarget ()Ljava/lang/String;
-	public fun getDefaultTarget ()Ljava/lang/String;
-	public fun getMeta ()Lhep/dataforge/meta/Meta;
-	public fun getName ()Lhep/dataforge/names/Name;
-	protected final fun require (Lhep/dataforge/context/PluginFactory;Lhep/dataforge/meta/Meta;)Lkotlin/properties/ReadOnlyProperty;
-	public static synthetic fun require$default (Lhep/dataforge/context/AbstractPlugin;Lhep/dataforge/context/PluginFactory;Lhep/dataforge/meta/Meta;ILjava/lang/Object;)Lkotlin/properties/ReadOnlyProperty;
-	public fun toMeta ()Lhep/dataforge/meta/Meta;
+	public fun getContext ()Lspace/kscience/dataforge/context/Context;
+	public fun getMeta ()Lspace/kscience/dataforge/meta/Meta;
+	public fun isAttached ()Z
+	protected final fun require (Lspace/kscience/dataforge/context/PluginFactory;Lkotlin/reflect/KClass;Lspace/kscience/dataforge/meta/Meta;)Lkotlin/properties/ReadOnlyProperty;
+	public static synthetic fun require$default (Lspace/kscience/dataforge/context/AbstractPlugin;Lspace/kscience/dataforge/context/PluginFactory;Lkotlin/reflect/KClass;Lspace/kscience/dataforge/meta/Meta;ILjava/lang/Object;)Lkotlin/properties/ReadOnlyProperty;
 }
 
-public final class hep/dataforge/context/AbstractPluginKt {
-	public static final fun toMap (Ljava/util/Collection;)Ljava/util/Map;
+public final class space/kscience/dataforge/context/AbstractPluginKt {
+	public static final fun associateByName (Ljava/util/Collection;)Ljava/util/Map;
 }
 
-public final class hep/dataforge/context/ClassLoaderPlugin : hep/dataforge/context/AbstractPlugin {
-	public static final field Companion Lhep/dataforge/context/ClassLoaderPlugin$Companion;
+public final class space/kscience/dataforge/context/ClassLoaderPlugin : space/kscience/dataforge/context/AbstractPlugin {
+	public static final field Companion Lspace/kscience/dataforge/context/ClassLoaderPlugin$Companion;
 	public fun <init> (Ljava/lang/ClassLoader;)V
-	public fun getTag ()Lhep/dataforge/context/PluginTag;
+	public fun getTag ()Lspace/kscience/dataforge/context/PluginTag;
 	public final fun services (Lkotlin/reflect/KClass;)Lkotlin/sequences/Sequence;
 }
 
-public final class hep/dataforge/context/ClassLoaderPlugin$Companion {
-	public final fun getDEFAULT ()Lhep/dataforge/context/ClassLoaderPlugin;
+public final class space/kscience/dataforge/context/ClassLoaderPlugin$Companion {
+	public final fun getDEFAULT ()Lspace/kscience/dataforge/context/ClassLoaderPlugin;
 }
 
-public final class hep/dataforge/context/ClassLoaderPluginKt {
-	public static final fun getClassLoaderPlugin (Lhep/dataforge/context/Context;)Lhep/dataforge/context/ClassLoaderPlugin;
+public final class space/kscience/dataforge/context/ClassLoaderPluginKt {
+	public static final fun getClassLoaderPlugin (Lspace/kscience/dataforge/context/Context;)Lspace/kscience/dataforge/context/ClassLoaderPlugin;
 }
 
-public class hep/dataforge/context/Context : hep/dataforge/meta/MetaRepr, hep/dataforge/misc/Named, hep/dataforge/provider/Provider, kotlinx/coroutines/CoroutineScope {
-	public static final field Companion Lhep/dataforge/context/Context$Companion;
+public class space/kscience/dataforge/context/Context : kotlinx/coroutines/CoroutineScope, space/kscience/dataforge/meta/MetaRepr, space/kscience/dataforge/misc/Named, space/kscience/dataforge/provider/Provider {
+	public static final field Companion Lspace/kscience/dataforge/context/Context$Companion;
 	public static final field PROPERTY_TARGET Ljava/lang/String;
+	public final fun buildContext (Lspace/kscience/dataforge/names/Name;Lkotlin/jvm/functions/Function1;)Lspace/kscience/dataforge/context/Context;
+	public static synthetic fun buildContext$default (Lspace/kscience/dataforge/context/Context;Lspace/kscience/dataforge/names/Name;Lkotlin/jvm/functions/Function1;ILjava/lang/Object;)Lspace/kscience/dataforge/context/Context;
 	public fun close ()V
 	public fun content (Ljava/lang/String;)Ljava/util/Map;
 	public final fun content (Ljava/lang/String;Z)Ljava/util/Map;
 	public fun getCoroutineContext ()Lkotlin/coroutines/CoroutineContext;
-	public fun getDefaultChainTarget ()Ljava/lang/String;
 	public fun getDefaultTarget ()Ljava/lang/String;
-	public final fun getName ()Lhep/dataforge/names/Name;
-	public final fun getParent ()Lhep/dataforge/context/Context;
-	public final fun getPlugins ()Lhep/dataforge/context/PluginManager;
-	public fun toMeta ()Lhep/dataforge/meta/Meta;
+	public final fun getName ()Lspace/kscience/dataforge/names/Name;
+	public final fun getParent ()Lspace/kscience/dataforge/context/Context;
+	public final fun getPlugins ()Lspace/kscience/dataforge/context/PluginManager;
+	public final fun getProperties ()Lspace/kscience/dataforge/meta/Laminate;
+	public fun toMeta ()Lspace/kscience/dataforge/meta/Meta;
 }
 
-public final class hep/dataforge/context/Context$Companion {
+public final class space/kscience/dataforge/context/Context$Companion {
 }
 
-public abstract interface class hep/dataforge/context/ContextAware {
-	public abstract fun getContext ()Lhep/dataforge/context/Context;
+public abstract interface class space/kscience/dataforge/context/ContextAware {
+	public abstract fun getContext ()Lspace/kscience/dataforge/context/Context;
 }
 
-public final class hep/dataforge/context/ContextBuilder {
-	public fun <init> ()V
-	public fun <init> (Lhep/dataforge/context/Context;Ljava/lang/String;)V
-	public synthetic fun <init> (Lhep/dataforge/context/Context;Ljava/lang/String;ILkotlin/jvm/internal/DefaultConstructorMarker;)V
-	public final fun build ()Lhep/dataforge/context/Context;
-	public final fun getName ()Ljava/lang/String;
-	public final fun plugin (Lhep/dataforge/context/PluginFactory;Lkotlin/jvm/functions/Function1;)V
-	public final fun plugin (Lhep/dataforge/context/PluginTag;Lkotlin/jvm/functions/Function1;)V
+public final class space/kscience/dataforge/context/ContextBuilder {
+	public final fun build ()Lspace/kscience/dataforge/context/Context;
+	public final fun getName ()Lspace/kscience/dataforge/names/Name;
 	public final fun plugin (Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Lkotlin/jvm/functions/Function1;)V
-	public static synthetic fun plugin$default (Lhep/dataforge/context/ContextBuilder;Lhep/dataforge/context/PluginFactory;Lkotlin/jvm/functions/Function1;ILjava/lang/Object;)V
-	public static synthetic fun plugin$default (Lhep/dataforge/context/ContextBuilder;Lhep/dataforge/context/PluginTag;Lkotlin/jvm/functions/Function1;ILjava/lang/Object;)V
-	public static synthetic fun plugin$default (Lhep/dataforge/context/ContextBuilder;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Lkotlin/jvm/functions/Function1;ILjava/lang/Object;)V
+	public final fun plugin (Lspace/kscience/dataforge/context/Plugin;)V
+	public final fun plugin (Lspace/kscience/dataforge/context/PluginFactory;Lkotlin/jvm/functions/Function1;)V
+	public final fun plugin (Lspace/kscience/dataforge/context/PluginFactory;Lspace/kscience/dataforge/meta/Meta;)V
+	public final fun plugin (Lspace/kscience/dataforge/context/PluginTag;Lkotlin/jvm/functions/Function1;)V
+	public static synthetic fun plugin$default (Lspace/kscience/dataforge/context/ContextBuilder;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Lkotlin/jvm/functions/Function1;ILjava/lang/Object;)V
+	public static synthetic fun plugin$default (Lspace/kscience/dataforge/context/ContextBuilder;Lspace/kscience/dataforge/context/PluginFactory;Lkotlin/jvm/functions/Function1;ILjava/lang/Object;)V
+	public static synthetic fun plugin$default (Lspace/kscience/dataforge/context/ContextBuilder;Lspace/kscience/dataforge/context/PluginTag;Lkotlin/jvm/functions/Function1;ILjava/lang/Object;)V
 	public final fun properties (Lkotlin/jvm/functions/Function1;)V
-	public final fun setName (Ljava/lang/String;)V
 }
 
-public final class hep/dataforge/context/ContextKt {
-	public static final fun Context (Ljava/lang/String;Lhep/dataforge/context/Context;Lkotlin/jvm/functions/Function1;)Lhep/dataforge/context/Context;
-	public static synthetic fun Context$default (Ljava/lang/String;Lhep/dataforge/context/Context;Lkotlin/jvm/functions/Function1;ILjava/lang/Object;)Lhep/dataforge/context/Context;
+public final class space/kscience/dataforge/context/ContextBuilderKt {
 }
 
-public abstract interface class hep/dataforge/context/Factory {
-	public abstract fun invoke (Lhep/dataforge/meta/Meta;Lhep/dataforge/context/Context;)Ljava/lang/Object;
+public final class space/kscience/dataforge/context/DefaultLogManager : space/kscience/dataforge/context/AbstractPlugin, space/kscience/dataforge/context/LogManager {
+	public static final field Companion Lspace/kscience/dataforge/context/DefaultLogManager$Companion;
+	public fun <init> ()V
+	public fun getDefaultLogger ()Lspace/kscience/dataforge/context/Logger;
+	public fun getTag ()Lspace/kscience/dataforge/context/PluginTag;
+	public fun logger (Lspace/kscience/dataforge/names/Name;)Lspace/kscience/dataforge/context/Logger;
 }
 
-public final class hep/dataforge/context/Factory$DefaultImpls {
-	public static synthetic fun invoke$default (Lhep/dataforge/context/Factory;Lhep/dataforge/meta/Meta;Lhep/dataforge/context/Context;ILjava/lang/Object;)Ljava/lang/Object;
+public final class space/kscience/dataforge/context/DefaultLogManager$Companion : space/kscience/dataforge/context/PluginFactory {
+	public synthetic fun build (Lspace/kscience/dataforge/context/Context;Lspace/kscience/dataforge/meta/Meta;)Ljava/lang/Object;
+	public fun build (Lspace/kscience/dataforge/context/Context;Lspace/kscience/dataforge/meta/Meta;)Lspace/kscience/dataforge/context/DefaultLogManager;
+	public fun getTag ()Lspace/kscience/dataforge/context/PluginTag;
 }
 
-public final class hep/dataforge/context/Global : hep/dataforge/context/Context {
-	public static final field INSTANCE Lhep/dataforge/context/Global;
-	public fun close ()V
-	public final fun context (Ljava/lang/String;Lhep/dataforge/context/Context;Lkotlin/jvm/functions/Function1;)Lhep/dataforge/context/Context;
-	public static synthetic fun context$default (Lhep/dataforge/context/Global;Ljava/lang/String;Lhep/dataforge/context/Context;Lkotlin/jvm/functions/Function1;ILjava/lang/Object;)Lhep/dataforge/context/Context;
-	public final fun getContext (Ljava/lang/String;)Lhep/dataforge/context/Context;
-	public fun getCoroutineContext ()Lkotlin/coroutines/CoroutineContext;
+public abstract interface class space/kscience/dataforge/context/Factory {
+	public abstract fun build (Lspace/kscience/dataforge/context/Context;Lspace/kscience/dataforge/meta/Meta;)Ljava/lang/Object;
 }
 
-public final class hep/dataforge/context/LoggingKt {
-	public static final fun getLogger (Lhep/dataforge/context/Context;)Lmu/KLogger;
-	public static final fun getLogger (Lhep/dataforge/context/ContextAware;)Lmu/KLogger;
+public final class space/kscience/dataforge/context/FactoryKt {
+	public static final fun invoke (Lspace/kscience/dataforge/context/Factory;Lspace/kscience/dataforge/meta/Meta;Lspace/kscience/dataforge/context/Context;)Ljava/lang/Object;
+	public static synthetic fun invoke$default (Lspace/kscience/dataforge/context/Factory;Lspace/kscience/dataforge/meta/Meta;Lspace/kscience/dataforge/context/Context;ILjava/lang/Object;)Ljava/lang/Object;
 }
 
-public abstract interface class hep/dataforge/context/Plugin : hep/dataforge/context/ContextAware, hep/dataforge/meta/MetaRepr, hep/dataforge/misc/Named, hep/dataforge/provider/Provider {
-	public static final field Companion Lhep/dataforge/context/Plugin$Companion;
+public final class space/kscience/dataforge/context/GlobalKt {
+	public static final fun Context (Ljava/lang/String;Lkotlin/jvm/functions/Function1;)Lspace/kscience/dataforge/context/Context;
+	public static synthetic fun Context$default (Ljava/lang/String;Lkotlin/jvm/functions/Function1;ILjava/lang/Object;)Lspace/kscience/dataforge/context/Context;
+	public static final fun getGlobal ()Lspace/kscience/dataforge/context/Context;
+}
+
+public abstract interface class space/kscience/dataforge/context/LogManager : space/kscience/dataforge/context/Logger, space/kscience/dataforge/context/Plugin {
+	public static final field Companion Lspace/kscience/dataforge/context/LogManager$Companion;
+	public static final field DEBUG Ljava/lang/String;
+	public static final field ERROR Ljava/lang/String;
+	public static final field INFO Ljava/lang/String;
+	public static final field TRACE Ljava/lang/String;
+	public static final field WARNING Ljava/lang/String;
+	public abstract fun getDefaultLogger ()Lspace/kscience/dataforge/context/Logger;
+	public fun log (Ljava/lang/String;Lkotlin/jvm/functions/Function0;)V
+	public fun log (Lspace/kscience/dataforge/names/Name;Ljava/lang/String;Lkotlin/jvm/functions/Function0;)V
+	public abstract fun logger (Lspace/kscience/dataforge/names/Name;)Lspace/kscience/dataforge/context/Logger;
+}
+
+public final class space/kscience/dataforge/context/LogManager$Companion {
+	public static final field DEBUG Ljava/lang/String;
+	public static final field ERROR Ljava/lang/String;
+	public static final field INFO Ljava/lang/String;
+	public static final field TRACE Ljava/lang/String;
+	public static final field WARNING Ljava/lang/String;
+}
+
+public final class space/kscience/dataforge/context/LogManagerKt {
+	public static final fun debug (Lspace/kscience/dataforge/context/Logger;Lkotlin/jvm/functions/Function0;)V
+	public static final fun error (Lspace/kscience/dataforge/context/Logger;Ljava/lang/Throwable;Lkotlin/jvm/functions/Function0;)V
+	public static final fun error (Lspace/kscience/dataforge/context/Logger;Lkotlin/jvm/functions/Function0;)V
+	public static final fun getLogger (Lspace/kscience/dataforge/context/Context;)Lspace/kscience/dataforge/context/LogManager;
+	public static final fun getLogger (Lspace/kscience/dataforge/context/ContextAware;)Lspace/kscience/dataforge/context/Logger;
+	public static final fun info (Lspace/kscience/dataforge/context/Logger;Lkotlin/jvm/functions/Function0;)V
+	public static final fun trace (Lspace/kscience/dataforge/context/Logger;Lkotlin/jvm/functions/Function0;)V
+	public static final fun warn (Lspace/kscience/dataforge/context/Logger;Lkotlin/jvm/functions/Function0;)V
+}
+
+public abstract interface class space/kscience/dataforge/context/Logger {
+	public abstract fun log (Ljava/lang/String;Lkotlin/jvm/functions/Function0;)V
+}
+
+public abstract interface class space/kscience/dataforge/context/Plugin : space/kscience/dataforge/context/ContextAware, space/kscience/dataforge/meta/MetaRepr, space/kscience/dataforge/misc/Named, space/kscience/dataforge/provider/Provider {
+	public static final field Companion Lspace/kscience/dataforge/context/Plugin$Companion;
 	public static final field TARGET Ljava/lang/String;
-	public abstract fun attach (Lhep/dataforge/context/Context;)V
+	public abstract fun attach (Lspace/kscience/dataforge/context/Context;)V
 	public abstract fun dependsOn ()Ljava/util/Map;
 	public abstract fun detach ()V
-	public abstract fun getMeta ()Lhep/dataforge/meta/Meta;
-	public abstract fun getName ()Lhep/dataforge/names/Name;
-	public abstract fun getTag ()Lhep/dataforge/context/PluginTag;
-	public abstract fun toMeta ()Lhep/dataforge/meta/Meta;
+	public abstract fun getMeta ()Lspace/kscience/dataforge/meta/Meta;
+	public fun getName ()Lspace/kscience/dataforge/names/Name;
+	public abstract fun getTag ()Lspace/kscience/dataforge/context/PluginTag;
+	public abstract fun isAttached ()Z
+	public fun toMeta ()Lspace/kscience/dataforge/meta/Meta;
 }
 
-public final class hep/dataforge/context/Plugin$Companion {
+public final class space/kscience/dataforge/context/Plugin$Companion {
 	public static final field TARGET Ljava/lang/String;
 }
 
-public final class hep/dataforge/context/Plugin$DefaultImpls {
-	public static fun content (Lhep/dataforge/context/Plugin;Ljava/lang/String;)Ljava/util/Map;
-	public static fun getDefaultChainTarget (Lhep/dataforge/context/Plugin;)Ljava/lang/String;
-	public static fun getDefaultTarget (Lhep/dataforge/context/Plugin;)Ljava/lang/String;
-	public static fun getName (Lhep/dataforge/context/Plugin;)Lhep/dataforge/names/Name;
-	public static fun toMeta (Lhep/dataforge/context/Plugin;)Lhep/dataforge/meta/Meta;
+public final class space/kscience/dataforge/context/PluginBuilder {
+	public fun <init> (Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;)V
+	public synthetic fun <init> (Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;ILkotlin/jvm/internal/DefaultConstructorMarker;)V
+	public final fun build ()Lspace/kscience/dataforge/context/PluginFactory;
+	public final fun getTag ()Lspace/kscience/dataforge/context/PluginTag;
+	public final fun provides (Ljava/lang/String;Ljava/util/Map;)V
+	public final fun provides (Ljava/lang/String;[Lspace/kscience/dataforge/misc/Named;)V
+	public final fun requires (Lspace/kscience/dataforge/context/PluginFactory;Lspace/kscience/dataforge/meta/Meta;)V
+	public static synthetic fun requires$default (Lspace/kscience/dataforge/context/PluginBuilder;Lspace/kscience/dataforge/context/PluginFactory;Lspace/kscience/dataforge/meta/Meta;ILjava/lang/Object;)V
 }
 
-public abstract interface class hep/dataforge/context/PluginFactory : hep/dataforge/context/Factory {
-	public static final field Companion Lhep/dataforge/context/PluginFactory$Companion;
+public final class space/kscience/dataforge/context/PluginBuilderKt {
+	public static final fun PluginFactory (Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Lkotlin/jvm/functions/Function1;)Lspace/kscience/dataforge/context/PluginFactory;
+	public static synthetic fun PluginFactory$default (Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Lkotlin/jvm/functions/Function1;ILjava/lang/Object;)Lspace/kscience/dataforge/context/PluginFactory;
+}
+
+public abstract interface class space/kscience/dataforge/context/PluginFactory : space/kscience/dataforge/context/Factory {
+	public static final field Companion Lspace/kscience/dataforge/context/PluginFactory$Companion;
 	public static final field TYPE Ljava/lang/String;
-	public abstract fun getTag ()Lhep/dataforge/context/PluginTag;
-	public abstract fun getType ()Lkotlin/reflect/KClass;
+	public abstract fun getTag ()Lspace/kscience/dataforge/context/PluginTag;
 }
 
-public final class hep/dataforge/context/PluginFactory$Companion {
+public final class space/kscience/dataforge/context/PluginFactory$Companion {
 	public static final field TYPE Ljava/lang/String;
 }
 
-public final class hep/dataforge/context/PluginManager : hep/dataforge/context/ContextAware, java/lang/Iterable, kotlin/jvm/internal/markers/KMappedMarker {
-	public fun <init> (Lhep/dataforge/context/Context;)V
-	public final fun fetch (Lhep/dataforge/context/PluginFactory;Lhep/dataforge/meta/Meta;Z)Lhep/dataforge/context/Plugin;
-	public final fun fetch (Lhep/dataforge/context/PluginFactory;ZLkotlin/jvm/functions/Function1;)Lhep/dataforge/context/Plugin;
-	public static synthetic fun fetch$default (Lhep/dataforge/context/PluginManager;Lhep/dataforge/context/PluginFactory;Lhep/dataforge/meta/Meta;ZILjava/lang/Object;)Lhep/dataforge/context/Plugin;
-	public static synthetic fun fetch$default (Lhep/dataforge/context/PluginManager;Lhep/dataforge/context/PluginFactory;ZLkotlin/jvm/functions/Function1;ILjava/lang/Object;)Lhep/dataforge/context/Plugin;
-	public final fun find (ZLkotlin/jvm/functions/Function1;)Lhep/dataforge/context/Plugin;
-	public static synthetic fun find$default (Lhep/dataforge/context/PluginManager;ZLkotlin/jvm/functions/Function1;ILjava/lang/Object;)Lhep/dataforge/context/Plugin;
-	public final fun get (Lhep/dataforge/context/PluginTag;Z)Lhep/dataforge/context/Plugin;
-	public final fun get (Lkotlin/reflect/KClass;Lhep/dataforge/context/PluginTag;Z)Ljava/lang/Object;
-	public static synthetic fun get$default (Lhep/dataforge/context/PluginManager;Lhep/dataforge/context/PluginTag;ZILjava/lang/Object;)Lhep/dataforge/context/Plugin;
-	public static synthetic fun get$default (Lhep/dataforge/context/PluginManager;Lkotlin/reflect/KClass;Lhep/dataforge/context/PluginTag;ZILjava/lang/Object;)Ljava/lang/Object;
-	public fun getContext ()Lhep/dataforge/context/Context;
+public final class space/kscience/dataforge/context/PluginManager : java/lang/Iterable, kotlin/jvm/internal/markers/KMappedMarker, space/kscience/dataforge/context/ContextAware {
+	public final fun find (ZLkotlin/jvm/functions/Function1;)Lspace/kscience/dataforge/context/Plugin;
+	public static synthetic fun find$default (Lspace/kscience/dataforge/context/PluginManager;ZLkotlin/jvm/functions/Function1;ILjava/lang/Object;)Lspace/kscience/dataforge/context/Plugin;
+	public final fun get (Lspace/kscience/dataforge/context/PluginTag;Z)Lspace/kscience/dataforge/context/Plugin;
+	public static synthetic fun get$default (Lspace/kscience/dataforge/context/PluginManager;Lspace/kscience/dataforge/context/PluginTag;ZILjava/lang/Object;)Lspace/kscience/dataforge/context/Plugin;
+	public final fun getByType (Lkotlin/reflect/KClass;Lspace/kscience/dataforge/context/PluginTag;Z)Ljava/lang/Object;
+	public static synthetic fun getByType$default (Lspace/kscience/dataforge/context/PluginManager;Lkotlin/reflect/KClass;Lspace/kscience/dataforge/context/PluginTag;ZILjava/lang/Object;)Ljava/lang/Object;
+	public fun getContext ()Lspace/kscience/dataforge/context/Context;
 	public fun iterator ()Ljava/util/Iterator;
 	public final fun list (Z)Ljava/util/Collection;
-	public final fun load (Lhep/dataforge/context/Plugin;)Lhep/dataforge/context/Plugin;
-	public final fun load (Lhep/dataforge/context/PluginFactory;Lhep/dataforge/meta/Meta;)Lhep/dataforge/context/Plugin;
-	public final fun load (Lhep/dataforge/context/PluginFactory;Lkotlin/jvm/functions/Function1;)Lhep/dataforge/context/Plugin;
-	public static synthetic fun load$default (Lhep/dataforge/context/PluginManager;Lhep/dataforge/context/PluginFactory;Lhep/dataforge/meta/Meta;ILjava/lang/Object;)Lhep/dataforge/context/Plugin;
-	public final fun remove (Lhep/dataforge/context/Plugin;)V
 }
 
-public final class hep/dataforge/context/PluginTag : hep/dataforge/meta/MetaRepr {
-	public static final field Companion Lhep/dataforge/context/PluginTag$Companion;
+public final class space/kscience/dataforge/context/PluginTag : space/kscience/dataforge/meta/MetaRepr {
+	public static final field Companion Lspace/kscience/dataforge/context/PluginTag$Companion;
 	public static final field DATAFORGE_GROUP Ljava/lang/String;
 	public fun <init> (Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;)V
 	public synthetic fun <init> (Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;ILkotlin/jvm/internal/DefaultConstructorMarker;)V
 	public final fun component1 ()Ljava/lang/String;
 	public final fun component2 ()Ljava/lang/String;
 	public final fun component3 ()Ljava/lang/String;
-	public final fun copy (Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;)Lhep/dataforge/context/PluginTag;
-	public static synthetic fun copy$default (Lhep/dataforge/context/PluginTag;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;ILjava/lang/Object;)Lhep/dataforge/context/PluginTag;
+	public final fun copy (Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;)Lspace/kscience/dataforge/context/PluginTag;
+	public static synthetic fun copy$default (Lspace/kscience/dataforge/context/PluginTag;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;ILjava/lang/Object;)Lspace/kscience/dataforge/context/PluginTag;
 	public fun equals (Ljava/lang/Object;)Z
 	public final fun getGroup ()Ljava/lang/String;
 	public final fun getName ()Ljava/lang/String;
 	public final fun getVersion ()Ljava/lang/String;
 	public fun hashCode ()I
-	public final fun matches (Lhep/dataforge/context/PluginTag;)Z
-	public fun toMeta ()Lhep/dataforge/meta/Meta;
+	public final fun matches (Lspace/kscience/dataforge/context/PluginTag;)Z
+	public fun toMeta ()Lspace/kscience/dataforge/meta/Meta;
 	public fun toString ()Ljava/lang/String;
 }
 
-public final class hep/dataforge/context/PluginTag$Companion {
-	public final fun fromString (Ljava/lang/String;)Lhep/dataforge/context/PluginTag;
+public final class space/kscience/dataforge/context/PluginTag$$serializer : kotlinx/serialization/internal/GeneratedSerializer {
+	public static final field INSTANCE Lspace/kscience/dataforge/context/PluginTag$$serializer;
+	public fun childSerializers ()[Lkotlinx/serialization/KSerializer;
+	public synthetic fun deserialize (Lkotlinx/serialization/encoding/Decoder;)Ljava/lang/Object;
+	public fun deserialize (Lkotlinx/serialization/encoding/Decoder;)Lspace/kscience/dataforge/context/PluginTag;
+	public fun getDescriptor ()Lkotlinx/serialization/descriptors/SerialDescriptor;
+	public synthetic fun serialize (Lkotlinx/serialization/encoding/Encoder;Ljava/lang/Object;)V
+	public fun serialize (Lkotlinx/serialization/encoding/Encoder;Lspace/kscience/dataforge/context/PluginTag;)V
+	public fun typeParametersSerializers ()[Lkotlinx/serialization/KSerializer;
 }
 
-public final class hep/dataforge/context/ResolveKt {
-	public static final fun gather (Lhep/dataforge/context/Context;Ljava/lang/String;Lkotlin/reflect/KClass;Z)Ljava/util/Map;
-	public static synthetic fun gather$default (Lhep/dataforge/context/Context;Ljava/lang/String;Lkotlin/reflect/KClass;ZILjava/lang/Object;)Ljava/util/Map;
-	public static final fun gatherInSequence (Lhep/dataforge/context/Context;Ljava/lang/String;Lkotlin/reflect/KClass;Z)Lkotlin/sequences/Sequence;
-	public static synthetic fun gatherInSequence$default (Lhep/dataforge/context/Context;Ljava/lang/String;Lkotlin/reflect/KClass;ZILjava/lang/Object;)Lkotlin/sequences/Sequence;
+public final class space/kscience/dataforge/context/PluginTag$Companion {
+	public final fun fromString (Ljava/lang/String;)Lspace/kscience/dataforge/context/PluginTag;
+	public final fun serializer ()Lkotlinx/serialization/KSerializer;
+}
+
+public final class space/kscience/dataforge/context/ResolveKt {
+	public static final fun gather (Lspace/kscience/dataforge/context/Context;Ljava/lang/String;Lkotlin/reflect/KClass;Z)Ljava/util/Map;
+	public static synthetic fun gather$default (Lspace/kscience/dataforge/context/Context;Ljava/lang/String;Lkotlin/reflect/KClass;ZILjava/lang/Object;)Ljava/util/Map;
 	public static final fun getValues (Lkotlin/sequences/Sequence;)Lkotlin/sequences/Sequence;
-	public static final fun resolve (Lhep/dataforge/context/Context;Ljava/lang/String;Lhep/dataforge/names/Name;Lkotlin/reflect/KClass;)Ljava/lang/Object;
+	public static final fun resolve (Lspace/kscience/dataforge/context/Context;Ljava/lang/String;Lspace/kscience/dataforge/names/Name;Lkotlin/reflect/KClass;)Ljava/lang/Object;
 }
 
-public abstract interface annotation class hep/dataforge/descriptors/Attribute : java/lang/annotation/Annotation {
-	public abstract fun key ()Ljava/lang/String;
-	public abstract fun value ()Ljava/lang/String;
+public final class space/kscience/dataforge/context/SlfLogManager : space/kscience/dataforge/context/AbstractPlugin, space/kscience/dataforge/context/LogManager {
+	public static final field Companion Lspace/kscience/dataforge/context/SlfLogManager$Companion;
+	public fun <init> ()V
+	public fun getDefaultLogger ()Lspace/kscience/dataforge/context/Logger;
+	public fun getTag ()Lspace/kscience/dataforge/context/PluginTag;
+	public fun logger (Lspace/kscience/dataforge/names/Name;)Lspace/kscience/dataforge/context/Logger;
 }
 
-public abstract interface annotation class hep/dataforge/descriptors/Attributes : java/lang/annotation/Annotation {
-	public abstract fun attrs ()[Lhep/dataforge/descriptors/Attribute;
+public final class space/kscience/dataforge/context/SlfLogManager$Companion : space/kscience/dataforge/context/PluginFactory {
+	public synthetic fun build (Lspace/kscience/dataforge/context/Context;Lspace/kscience/dataforge/meta/Meta;)Ljava/lang/Object;
+	public fun build (Lspace/kscience/dataforge/context/Context;Lspace/kscience/dataforge/meta/Meta;)Lspace/kscience/dataforge/context/SlfLogManager;
+	public fun getTag ()Lspace/kscience/dataforge/context/PluginTag;
 }
 
-public abstract interface annotation class hep/dataforge/descriptors/ItemDef : java/lang/annotation/Annotation {
-	public abstract fun info ()Ljava/lang/String;
-	public abstract fun multiple ()Z
-	public abstract fun required ()Z
+public final class space/kscience/dataforge/properties/PropertyKt {
 }
 
-public abstract interface annotation class hep/dataforge/descriptors/ValueDef : java/lang/annotation/Annotation {
-	public abstract fun allowed ()[Ljava/lang/String;
-	public abstract fun def ()Ljava/lang/String;
-	public abstract fun enumeration ()Ljava/lang/Class;
-	public abstract fun type ()[Lhep/dataforge/values/ValueType;
+public final class space/kscience/dataforge/properties/SchemePropertyKt {
 }
 
-public final class hep/dataforge/properties/ConfigProperty : hep/dataforge/properties/Property {
-	public fun <init> (Lhep/dataforge/meta/Config;Lhep/dataforge/names/Name;Lhep/dataforge/meta/transformations/MetaConverter;)V
-	public final fun getConfig ()Lhep/dataforge/meta/Config;
-	public final fun getConverter ()Lhep/dataforge/meta/transformations/MetaConverter;
-	public final fun getName ()Lhep/dataforge/names/Name;
-	public fun getValue ()Ljava/lang/Object;
-	public fun onChange (Ljava/lang/Object;Lkotlin/jvm/functions/Function1;)V
-	public fun removeChangeListener (Ljava/lang/Object;)V
-	public fun setValue (Ljava/lang/Object;)V
+public final class space/kscience/dataforge/provider/DfTypeKt {
 }
 
-public abstract interface class hep/dataforge/properties/Property {
-	public abstract fun getValue ()Ljava/lang/Object;
-	public abstract fun onChange (Ljava/lang/Object;Lkotlin/jvm/functions/Function1;)V
-	public abstract fun removeChangeListener (Ljava/lang/Object;)V
-	public abstract fun setValue (Ljava/lang/Object;)V
-}
-
-public final class hep/dataforge/properties/Property$DefaultImpls {
-	public static synthetic fun onChange$default (Lhep/dataforge/properties/Property;Ljava/lang/Object;Lkotlin/jvm/functions/Function1;ILjava/lang/Object;)V
-	public static synthetic fun removeChangeListener$default (Lhep/dataforge/properties/Property;Ljava/lang/Object;ILjava/lang/Object;)V
-}
-
-public final class hep/dataforge/properties/PropertyKt {
-	public static final fun bind (Lhep/dataforge/properties/Property;Lhep/dataforge/properties/Property;)V
-	public static final fun mirror (Lhep/dataforge/properties/Property;Lhep/dataforge/properties/Property;Lkotlinx/coroutines/CoroutineScope;)V
-	public static final fun toFlow (Lhep/dataforge/properties/Property;)Lkotlinx/coroutines/flow/StateFlow;
-}
-
-public final class hep/dataforge/provider/DfTypeKt {
-	public static final fun getDfType (Lkotlin/reflect/KClass;)Ljava/lang/String;
-}
-
-public final class hep/dataforge/provider/Path : java/lang/Iterable, kotlin/jvm/internal/markers/KMappedMarker {
-	public static final field Companion Lhep/dataforge/provider/Path$Companion;
+public final class space/kscience/dataforge/provider/Path : java/lang/Iterable, kotlin/jvm/internal/markers/KMappedMarker {
+	public static final field Companion Lspace/kscience/dataforge/provider/Path$Companion;
 	public static final field PATH_SEGMENT_SEPARATOR Ljava/lang/String;
-	public static final synthetic fun box-impl (Ljava/util/List;)Lhep/dataforge/provider/Path;
+	public static final synthetic fun box-impl (Ljava/util/List;)Lspace/kscience/dataforge/provider/Path;
 	public static fun constructor-impl (Ljava/util/List;)Ljava/util/List;
 	public fun equals (Ljava/lang/Object;)Z
 	public static fun equals-impl (Ljava/util/List;Ljava/lang/Object;)Z
@@ -263,57 +276,52 @@ public final class hep/dataforge/provider/Path : java/lang/Iterable, kotlin/jvm/
 	public final synthetic fun unbox-impl ()Ljava/util/List;
 }
 
-public final class hep/dataforge/provider/Path$Companion {
-	public final fun parse-AnEnhig (Ljava/lang/String;)Ljava/util/List;
+public final class space/kscience/dataforge/provider/Path$Companion {
+	public final fun parse-X5wN5Vs (Ljava/lang/String;)Ljava/util/List;
 }
 
-public final class hep/dataforge/provider/PathKt {
-	public static final fun Path ([Lhep/dataforge/names/Name;)Ljava/util/List;
+public final class space/kscience/dataforge/provider/PathKt {
 	public static final fun Path ([Lkotlin/Pair;)Ljava/util/List;
-	public static final fun asPath (Lhep/dataforge/names/Name;Ljava/lang/String;)Ljava/util/List;
-	public static final fun asPath (Lhep/dataforge/provider/PathToken;)Ljava/util/List;
-	public static synthetic fun asPath$default (Lhep/dataforge/names/Name;Ljava/lang/String;ILjava/lang/Object;)Ljava/util/List;
-	public static final fun getHead-Ipm_iiw (Ljava/util/List;)Lhep/dataforge/provider/PathToken;
-	public static final fun getLength-Ipm_iiw (Ljava/util/List;)I
-	public static final fun getTail-Ipm_iiw (Ljava/util/List;)Ljava/util/List;
-	public static final fun plus-BlTXZnM (Ljava/util/List;Ljava/util/List;)Ljava/util/List;
+	public static final fun Path ([Lspace/kscience/dataforge/names/Name;)Ljava/util/List;
+	public static final fun asPath (Lspace/kscience/dataforge/names/Name;Ljava/lang/String;)Ljava/util/List;
+	public static final fun asPath (Lspace/kscience/dataforge/provider/PathToken;)Ljava/util/List;
+	public static synthetic fun asPath$default (Lspace/kscience/dataforge/names/Name;Ljava/lang/String;ILjava/lang/Object;)Ljava/util/List;
+	public static final fun getHead-LGjlSZY (Ljava/util/List;)Lspace/kscience/dataforge/provider/PathToken;
+	public static final fun getLength-LGjlSZY (Ljava/util/List;)I
+	public static final fun getTail-LGjlSZY (Ljava/util/List;)Ljava/util/List;
+	public static final fun plus-sn2Gq0g (Ljava/util/List;Ljava/util/List;)Ljava/util/List;
 }
 
-public final class hep/dataforge/provider/PathToken {
-	public static final field Companion Lhep/dataforge/provider/PathToken$Companion;
+public final class space/kscience/dataforge/provider/PathToken {
+	public static final field Companion Lspace/kscience/dataforge/provider/PathToken$Companion;
 	public static final field TARGET_SEPARATOR Ljava/lang/String;
-	public fun <init> (Lhep/dataforge/names/Name;Ljava/lang/String;)V
-	public synthetic fun <init> (Lhep/dataforge/names/Name;Ljava/lang/String;ILkotlin/jvm/internal/DefaultConstructorMarker;)V
-	public final fun component1 ()Lhep/dataforge/names/Name;
+	public fun <init> (Lspace/kscience/dataforge/names/Name;Ljava/lang/String;)V
+	public synthetic fun <init> (Lspace/kscience/dataforge/names/Name;Ljava/lang/String;ILkotlin/jvm/internal/DefaultConstructorMarker;)V
+	public final fun component1 ()Lspace/kscience/dataforge/names/Name;
 	public final fun component2 ()Ljava/lang/String;
-	public final fun copy (Lhep/dataforge/names/Name;Ljava/lang/String;)Lhep/dataforge/provider/PathToken;
-	public static synthetic fun copy$default (Lhep/dataforge/provider/PathToken;Lhep/dataforge/names/Name;Ljava/lang/String;ILjava/lang/Object;)Lhep/dataforge/provider/PathToken;
+	public final fun copy (Lspace/kscience/dataforge/names/Name;Ljava/lang/String;)Lspace/kscience/dataforge/provider/PathToken;
+	public static synthetic fun copy$default (Lspace/kscience/dataforge/provider/PathToken;Lspace/kscience/dataforge/names/Name;Ljava/lang/String;ILjava/lang/Object;)Lspace/kscience/dataforge/provider/PathToken;
 	public fun equals (Ljava/lang/Object;)Z
-	public final fun getName ()Lhep/dataforge/names/Name;
+	public final fun getName ()Lspace/kscience/dataforge/names/Name;
 	public final fun getTarget ()Ljava/lang/String;
 	public fun hashCode ()I
 	public fun toString ()Ljava/lang/String;
 }
 
-public final class hep/dataforge/provider/PathToken$Companion {
-	public final fun parse (Ljava/lang/String;)Lhep/dataforge/provider/PathToken;
+public final class space/kscience/dataforge/provider/PathToken$Companion {
+	public final fun parse (Ljava/lang/String;Z)Lspace/kscience/dataforge/provider/PathToken;
+	public static synthetic fun parse$default (Lspace/kscience/dataforge/provider/PathToken$Companion;Ljava/lang/String;ZILjava/lang/Object;)Lspace/kscience/dataforge/provider/PathToken;
 }
 
-public abstract interface class hep/dataforge/provider/Provider {
-	public abstract fun content (Ljava/lang/String;)Ljava/util/Map;
-	public abstract fun getDefaultChainTarget ()Ljava/lang/String;
-	public abstract fun getDefaultTarget ()Ljava/lang/String;
+public abstract interface class space/kscience/dataforge/provider/Provider {
+	public fun content (Ljava/lang/String;)Ljava/util/Map;
+	public fun getDefaultChainTarget ()Ljava/lang/String;
+	public fun getDefaultTarget ()Ljava/lang/String;
 }
 
-public final class hep/dataforge/provider/Provider$DefaultImpls {
-	public static fun content (Lhep/dataforge/provider/Provider;Ljava/lang/String;)Ljava/util/Map;
-	public static fun getDefaultChainTarget (Lhep/dataforge/provider/Provider;)Ljava/lang/String;
-	public static fun getDefaultTarget (Lhep/dataforge/provider/Provider;)Ljava/lang/String;
-}
-
-public final class hep/dataforge/provider/ProviderKt {
-	public static final fun provide-mzxrFLw (Lhep/dataforge/provider/Provider;Ljava/util/List;Ljava/lang/String;)Ljava/lang/Object;
-	public static synthetic fun provide-mzxrFLw$default (Lhep/dataforge/provider/Provider;Ljava/util/List;Ljava/lang/String;ILjava/lang/Object;)Ljava/lang/Object;
-	public static final fun top (Lhep/dataforge/provider/Provider;Ljava/lang/String;Lkotlin/reflect/KClass;)Ljava/util/Map;
+public final class space/kscience/dataforge/provider/ProviderKt {
+	public static final fun provide-CSkoCSg (Lspace/kscience/dataforge/provider/Provider;Ljava/util/List;Ljava/lang/String;)Ljava/lang/Object;
+	public static synthetic fun provide-CSkoCSg$default (Lspace/kscience/dataforge/provider/Provider;Ljava/util/List;Ljava/lang/String;ILjava/lang/Object;)Ljava/lang/Object;
+	public static final fun top (Lspace/kscience/dataforge/provider/Provider;Ljava/lang/String;Lkotlin/reflect/KClass;)Ljava/util/Map;
 }
 

dataforge-context/build.gradle.kts

@@ -1,35 +1,24 @@
 plugins {
-    id("ru.mipt.npm.mpp")
-    id("ru.mipt.npm.native")
+    id("space.kscience.gradle.mpp")
 }
 
 description = "Context and provider definitions"
 
 kscience {
+    jvm()
+    js()
+    native()
     useCoroutines()
-}
-
-kotlin {
-    sourceSets {
-        val commonMain by getting {
-            dependencies {
-                api(project(":dataforge-meta"))
-                api("io.github.microutils:kotlin-logging:1.9.0-dev-npm-2")
-            }
-        }
-        val jvmMain by getting {
-            dependencies {
-                api(kotlin("reflect"))
-                api("ch.qos.logback:logback-classic:1.2.3")
-            }
-        }
-        val jsMain by getting {
-            dependencies {
-            }
-        }
+    useSerialization()
+    dependencies {
+        api(project(":dataforge-meta"))
+    }
+    dependencies(jvmMain){
+        api(kotlin("reflect"))
+        api("org.slf4j:slf4j-api:1.7.30")
     }
 }
 
-readme{
-    maturity = ru.mipt.npm.gradle.Maturity.DEVELOPMENT
+readme {
+    maturity = space.kscience.gradle.Maturity.DEVELOPMENT
 }

dataforge-context/src/commonMain/kotlin/hep/dataforge/context/AbstractPlugin.kt

@@ -1,42 +0,0 @@
-package hep.dataforge.context
-
-import hep.dataforge.meta.Meta
-import hep.dataforge.misc.Named
-import hep.dataforge.names.Name
-import kotlin.properties.ReadOnlyProperty
-import kotlin.reflect.KClass
-import kotlin.reflect.KProperty
-
-public abstract class AbstractPlugin(override val meta: Meta = Meta.EMPTY) : Plugin {
-    private var _context: Context? = null
-    private val dependencies = HashMap<PluginFactory<*>, Meta>()
-
-    override val context: Context
-        get() = _context ?: error("Plugin $tag is not attached")
-
-    override fun attach(context: Context) {
-        this._context = context
-    }
-
-    override fun detach() {
-        this._context = null
-    }
-
-    final override fun dependsOn(): Map<PluginFactory<*>, Meta> = dependencies
-
-    /**
-     * Register plugin dependency and return a delegate which provides lazily initialized reference to dependent plugin
-     */
-    protected fun <P : Plugin> require(factory: PluginFactory<P>, meta: Meta = Meta.EMPTY): ReadOnlyProperty<AbstractPlugin, P> {
-        dependencies[factory] = meta
-        return PluginDependencyDelegate(factory.type)
-    }
-}
-
-public fun <T : Named> Collection<T>.toMap(): Map<Name, T> = associate { it.name to it }
-
-private class PluginDependencyDelegate<P : Plugin>(val type: KClass<out P>) : ReadOnlyProperty<AbstractPlugin, P> {
-    override fun getValue(thisRef: AbstractPlugin, property: KProperty<*>): P {
-        return thisRef.context.plugins[type] ?: error("Plugin with type $type not found")
-    }
-}

dataforge-context/src/commonMain/kotlin/hep/dataforge/context/ContextBuilder.kt

@@ -1,52 +0,0 @@
-package hep.dataforge.context
-
-import hep.dataforge.meta.Meta
-import hep.dataforge.meta.MetaBuilder
-import hep.dataforge.meta.seal
-import hep.dataforge.misc.DFBuilder
-import hep.dataforge.misc.DFExperimental
-import hep.dataforge.names.toName
-import kotlin.collections.HashMap
-import kotlin.collections.component1
-import kotlin.collections.component2
-import kotlin.collections.forEach
-import kotlin.collections.set
-
-/**
- * A convenience builder for context
- */
-@DFBuilder
-public class ContextBuilder(private val parent: Context = Global, public var name: String = "@anonymous") {
-    private val factories = HashMap<PluginFactory<*>, Meta>()
-    private var meta = MetaBuilder()
-
-    public fun properties(action: MetaBuilder.() -> Unit) {
-        meta.action()
-    }
-
-    @OptIn(DFExperimental::class)
-    private fun findPluginFactory(tag: PluginTag): PluginFactory<*> =
-        parent.gatherInSequence<PluginFactory<*>>(PluginFactory.TYPE).values
-            .find { it.tag.matches(tag) } ?: error("Can't resolve plugin factory for $tag")
-
-    public fun plugin(tag: PluginTag, metaBuilder: MetaBuilder.() -> Unit = {}) {
-        val factory = findPluginFactory(tag)
-        factories[factory] = Meta(metaBuilder)
-    }
-
-    public fun plugin(factory: PluginFactory<*>, metaBuilder: MetaBuilder.() -> Unit = {}) {
-        factories[factory] = Meta(metaBuilder)
-    }
-
-    public fun plugin(name: String, group: String = "", version: String = "", action: MetaBuilder.() -> Unit = {}) {
-        plugin(PluginTag(name, group, version), action)
-    }
-
-    public fun build(): Context {
-        return Context(name.toName(), parent, meta.seal()).apply {
-            factories.forEach { (factory, meta) ->
-                plugins.load(factory, meta)
-            }
-        }
-    }
-}

dataforge-context/src/commonMain/kotlin/hep/dataforge/context/Factory.kt

@@ -1,7 +0,0 @@
-package hep.dataforge.context
-
-import hep.dataforge.meta.Meta
-
-public interface Factory<out T : Any> {
-    public operator fun invoke(meta: Meta = Meta.EMPTY, context: Context = Global): T
-}

dataforge-context/src/commonMain/kotlin/hep/dataforge/context/Global.kt

@@ -1,48 +0,0 @@
-package hep.dataforge.context
-
-import hep.dataforge.meta.Meta
-import hep.dataforge.names.asName
-import kotlinx.coroutines.GlobalScope
-import kotlinx.coroutines.SupervisorJob
-import kotlin.coroutines.CoroutineContext
-import kotlin.native.concurrent.ThreadLocal
-
-/**
- * A global root context. Closing [Global] terminates the framework.
- */
-@ThreadLocal
-public object Global : Context("GLOBAL".asName(), null, Meta.EMPTY) {
-
-    override val coroutineContext: CoroutineContext = GlobalScope.coroutineContext + SupervisorJob()
-
-    /**
-     * Closing all contexts
-     *
-     * @throws Exception
-     */
-    override fun close() {
-        logger.info { "Shutting down GLOBAL" }
-        for (ctx in contextRegistry.values) {
-            ctx.close()
-        }
-        super.close()
-    }
-
-    private val contextRegistry = HashMap<String, Context>()
-
-    /**
-     * Get previously built context
-     *
-     * @param name
-     * @return
-     */
-    public fun getContext(name: String): Context? {
-        return contextRegistry[name]
-    }
-
-    public fun context(name: String, parent: Context = this, block: ContextBuilder.() -> Unit = {}): Context =
-        ContextBuilder(parent, name).apply(block).build().also {
-            contextRegistry[name] = it
-        }
-
-}

dataforge-context/src/commonMain/kotlin/hep/dataforge/context/PluginFactory.kt

@@ -1,14 +0,0 @@
-package hep.dataforge.context
-
-import hep.dataforge.misc.Type
-import kotlin.reflect.KClass
-
-@Type(PluginFactory.TYPE)
-public interface PluginFactory<T : Plugin> : Factory<T> {
-    public val tag: PluginTag
-    public val type: KClass<out T>
-
-    public companion object {
-        public const val TYPE: String = "pluginFactory"
-    }
-}

dataforge-context/src/commonMain/kotlin/hep/dataforge/context/PluginManager.kt

@@ -1,143 +0,0 @@
-package hep.dataforge.context
-
-import hep.dataforge.meta.Meta
-import hep.dataforge.meta.MetaBuilder
-import kotlin.reflect.KClass
-
-
-/**
- * The manager for plugin system. Should monitor plugin dependencies and locks.
- *
- * @property context A context for this plugin manager
- * @author Alexander Nozik
- */
-public class PluginManager(override val context: Context) : ContextAware, Iterable<Plugin> {
-
-    //TODO refactor to read-only container
-
-    /**
-     * A set of loaded plugins
-     */
-    private val plugins: HashSet<Plugin> = HashSet()
-
-    init {
-        plugins.forEach { it.attach(context) }
-    }
-
-    /**
-     * A [PluginManager] of parent context if it is present
-     */
-    private val parent: PluginManager? = context.parent?.plugins
-
-    /**
-     * List plugins stored in this [PluginManager]. If [inherit] is true, include parent plugins as well
-     */
-    public fun list(inherit: Boolean): Collection<Plugin> {
-        return if (inherit && parent != null) {
-            plugins + parent.list(true)
-        } else {
-            plugins
-        }
-    }
-
-    /**
-     * Get existing plugin or return null if not present. Only first matching plugin is returned.
-     * @param inherit search for parent [PluginManager] plugins
-     * @param predicate condition for the plugin
-     */
-    public fun find(inherit: Boolean = true, predicate: (Plugin) -> Boolean): Plugin? =
-        list(inherit).find(predicate)
-
-    /**
-     * Find a loaded plugin via its tag
-     *
-     * @param tag
-     * @return
-     */
-    public operator fun get(tag: PluginTag, inherit: Boolean = true): Plugin? =
-        find(inherit) { tag.matches(it.tag) }
-
-    /**
-     * Find a loaded plugin via its class. This method does not check if the result is unique and just returns first
-     * plugin matching the class condition.
-     * For safe search provide a tag since tags are checked on load and plugins with the same tag are not allowed
-     * in the same context.
-     *
-     * @param tag
-     * @param type
-     * @param <T>
-     * @return
-     */
-    @Suppress("UNCHECKED_CAST")
-    public operator fun <T : Any> get(type: KClass<out T>, tag: PluginTag? = null, recursive: Boolean = true): T? =
-        find(recursive) { type.isInstance(it) && (tag == null || tag.matches(it.tag)) } as T?
-
-    public inline operator fun <reified T : Any> get(tag: PluginTag? = null, recursive: Boolean = true): T? =
-        get(T::class, tag, recursive)
-
-    public inline operator fun <reified T : Plugin> get(factory: PluginFactory<T>, recursive: Boolean = true): T? =
-        get(factory.type, factory.tag, recursive)
-
-    /**
-     * Load given plugin into this manager and return loaded instance.
-     * Throw error if plugin of the same type and tag already exists in manager.
-     *
-     * @param plugin
-     * @return
-     */
-    public fun <T : Plugin> load(plugin: T): T {
-        if (get(plugin::class, plugin.tag, recursive = false) != null) {
-            error("Plugin with tag ${plugin.tag} already exists in ${context.name}")
-        } else {
-            for ((factory, meta) in plugin.dependsOn()) {
-                fetch(factory, meta, true)
-            }
-
-            logger.info { "Loading plugin ${plugin.name} into ${context.name}" }
-            plugin.attach(context)
-            plugins.add(plugin)
-            return plugin
-        }
-    }
-
-    /**
-     * Load a plugin using its factory
-     */
-    public fun <T : Plugin> load(factory: PluginFactory<T>, meta: Meta = Meta.EMPTY): T =
-        load(factory(meta, context))
-
-    public fun <T : Plugin> load(factory: PluginFactory<T>, metaBuilder: MetaBuilder.() -> Unit): T =
-        load(factory, Meta(metaBuilder))
-
-    /**
-     * Remove a plugin from [PluginManager]
-     */
-    public fun remove(plugin: Plugin) {
-        if (plugins.contains(plugin)) {
-            logger.info { "Removing plugin ${plugin.name} from ${context.name}" }
-            plugin.detach()
-            plugins.remove(plugin)
-        }
-    }
-
-    /**
-     * Get an existing plugin with given meta or load new one using provided factory
-     */
-    public fun <T : Plugin> fetch(factory: PluginFactory<T>, meta: Meta = Meta.EMPTY, recursive: Boolean = true): T {
-        val loaded = get(factory.type, factory.tag, recursive)
-        return when {
-            loaded == null -> load(factory(meta, context))
-            loaded.meta == meta -> loaded // if meta is the same, return existing plugin
-            else -> throw RuntimeException("Can't load plugin with tag ${factory.tag}. Plugin with this tag and different configuration already exists in context.")
-        }
-    }
-
-    public fun <T : Plugin> fetch(
-        factory: PluginFactory<T>,
-        recursive: Boolean = true,
-        metaBuilder: MetaBuilder.() -> Unit,
-    ): T = fetch(factory, Meta(metaBuilder), recursive)
-
-    override fun iterator(): Iterator<Plugin> = plugins.iterator()
-
-}

dataforge-context/src/commonMain/kotlin/hep/dataforge/context/logging.kt

@@ -1,22 +0,0 @@
-package hep.dataforge.context
-
-import hep.dataforge.misc.Named
-import hep.dataforge.provider.Path
-import mu.KLogger
-import mu.KotlinLogging
-
-/**
- * The logger specific to this context
- */
-public val Context.logger: KLogger get() = KotlinLogging.logger(name.toString())
-
-/**
- * The logger
- */
-public val ContextAware.logger: KLogger
-    get() = if (this is Named) {
-        KotlinLogging.logger(Path(context.name, this.name).toString())
-    } else {
-        context.logger
-    }
-

dataforge-context/src/commonMain/kotlin/hep/dataforge/properties/ConfigProperty.kt

@@ -1,34 +0,0 @@
-package hep.dataforge.properties
-
-import hep.dataforge.meta.Config
-import hep.dataforge.meta.get
-import hep.dataforge.meta.set
-import hep.dataforge.meta.transformations.MetaConverter
-import hep.dataforge.meta.transformations.nullableItemToObject
-import hep.dataforge.meta.transformations.nullableObjectToMetaItem
-import hep.dataforge.misc.DFExperimental
-import hep.dataforge.names.Name
-
-@DFExperimental
-public class ConfigProperty<T : Any>(
-    public val config: Config,
-    public val name: Name,
-    public val converter: MetaConverter<T>,
-) : Property<T?> {
-
-    override var value: T?
-        get() = converter.nullableItemToObject(config[name])
-        set(value) {
-            config[name] = converter.nullableObjectToMetaItem(value)
-        }
-
-    override fun onChange(owner: Any?, callback: (T?) -> Unit) {
-        config.onChange(owner) { name, oldItem, newItem ->
-            if (name == this.name && oldItem != newItem) callback(converter.nullableItemToObject(newItem))
-        }
-    }
-
-    override fun removeChangeListener(owner: Any?) {
-        config.removeListener(owner)
-    }
-}

dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/context/AbstractPlugin.kt

@@ -0,0 +1,57 @@
+package space.kscience.dataforge.context
+
+import space.kscience.dataforge.meta.Meta
+import space.kscience.dataforge.misc.DFInternal
+import space.kscience.dataforge.misc.Named
+import space.kscience.dataforge.names.Name
+import kotlin.properties.ReadOnlyProperty
+import kotlin.reflect.KClass
+import kotlin.reflect.KProperty
+
+public abstract class AbstractPlugin(override val meta: Meta = Meta.EMPTY) : Plugin {
+    private var _context: Context? = null
+    private val dependencies = HashMap<PluginFactory<*>, Meta>()
+
+    override val isAttached: Boolean get() = _context != null
+
+    override val context: Context
+        get() = _context ?: error("Plugin $tag is not attached")
+
+    override fun attach(context: Context) {
+        this._context = context
+    }
+
+    override fun detach() {
+        this._context = null
+    }
+
+    override fun dependsOn(): Map<PluginFactory<*>, Meta> = dependencies
+
+    protected fun <P : Plugin> require(
+        factory: PluginFactory<P>,
+        type: KClass<P>,
+        meta: Meta = Meta.EMPTY,
+    ): ReadOnlyProperty<AbstractPlugin, P> {
+        dependencies[factory] = meta
+        return PluginDependencyDelegate(factory, type)
+    }
+
+    /**
+     * Register plugin dependency and return a delegate which provides lazily initialized reference to dependent plugin
+     */
+    protected inline fun <reified P : Plugin> require(
+        factory: PluginFactory<P>,
+        meta: Meta = Meta.EMPTY,
+    ): ReadOnlyProperty<AbstractPlugin, P> = require(factory, P::class, meta)
+}
+
+public fun <T : Named> Collection<T>.associateByName(): Map<Name, T> = associate { it.name to it }
+
+private class PluginDependencyDelegate<P : Plugin>(val factory: PluginFactory<P>, val type: KClass<P>) :
+    ReadOnlyProperty<AbstractPlugin, P> {
+    @OptIn(DFInternal::class)
+    override fun getValue(thisRef: AbstractPlugin, property: KProperty<*>): P {
+        if (!thisRef.isAttached) error("Plugin dependency must not be called eagerly during initialization.")
+        return thisRef.context.plugins.getByType(type, factory.tag) ?: error("Plugin ${factory.tag} not found")
+    }
+}

dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/context/Context.kt

@@ -1,15 +1,14 @@
-package hep.dataforge.context
+package space.kscience.dataforge.context
 
-import hep.dataforge.meta.Laminate
-import hep.dataforge.meta.Meta
-import hep.dataforge.meta.MetaRepr
-import hep.dataforge.meta.itemSequence
-import hep.dataforge.misc.Named
-import hep.dataforge.names.Name
-import hep.dataforge.provider.Provider
 import kotlinx.coroutines.CoroutineScope
 import kotlinx.coroutines.Job
 import kotlinx.coroutines.SupervisorJob
+import space.kscience.dataforge.meta.*
+import space.kscience.dataforge.misc.DFExperimental
+import space.kscience.dataforge.misc.Named
+import space.kscience.dataforge.misc.ThreadSafe
+import space.kscience.dataforge.names.Name
+import space.kscience.dataforge.provider.Provider
 import kotlin.coroutines.CoroutineContext
 
 /**
@@ -25,35 +24,37 @@ import kotlin.coroutines.CoroutineContext
 public open class Context internal constructor(
     final override val name: Name,
     public val parent: Context?,
+    plugins: Set<Plugin>, // set of unattached plugins
     meta: Meta,
 ) : Named, MetaRepr, Provider, CoroutineScope {
 
     /**
      * Context properties. Working as substitute for environment variables
      */
-    private val properties: Laminate = if (parent == null) {
+    public val properties: Laminate = if (parent == null) {
         Laminate(meta)
     } else {
         Laminate(meta, parent.properties)
     }
 
+
     /**
      * A [PluginManager] for current context
      */
-    public val plugins: PluginManager by lazy { PluginManager(this) }
+    public val plugins: PluginManager by lazy { PluginManager(this, plugins) }
 
     override val defaultTarget: String get() = Plugin.TARGET
 
     public fun content(target: String, inherit: Boolean): Map<Name, Any> {
         return if (inherit) {
             when (target) {
-                PROPERTY_TARGET -> properties.itemSequence().toMap()
+                PROPERTY_TARGET -> properties.nodeSequence().toMap()
                 Plugin.TARGET -> plugins.list(true).associateBy { it.name }
                 else -> emptyMap()
             }
         } else {
             when (target) {
-                PROPERTY_TARGET -> properties.layers.firstOrNull()?.itemSequence()?.toMap() ?: emptyMap()
+                PROPERTY_TARGET -> properties.layers.firstOrNull()?.nodeSequence()?.toMap() ?: emptyMap()
                 Plugin.TARGET -> plugins.list(false).associateBy { it.name }
                 else -> emptyMap()
             }
@@ -68,18 +69,35 @@ public open class Context internal constructor(
         }
     }
 
+    private val childrenContexts = HashMap<Name, Context>()
+
     /**
-     * Detach all plugins and terminate context
+     * Get and validate existing context or build and register a new child context.
+     * @param name the relative (tail) name of the new context. If null, uses context hash code as a marker.
+     */
+    @OptIn(DFExperimental::class)
+    @ThreadSafe
+    public fun buildContext(name: Name? = null, block: ContextBuilder.() -> Unit = {}): Context {
+        val existing = name?.let { childrenContexts[name] }
+        return existing?.modify(block) ?: ContextBuilder(this, name).apply(block).build().also {
+            childrenContexts[it.name] = it
+        }
+    }
+
+    /**
+     * Detach all plugins, and close child contexts
      */
     public open fun close() {
+        //recursively closed child context
+        childrenContexts.forEach { it.value.close() }
         //detach all plugins
         plugins.forEach { it.detach() }
     }
 
     override fun toMeta(): Meta = Meta {
         "parent" to parent?.name
-        "properties" put properties.layers.firstOrNull()
-        "plugins" put plugins.map { it.toMeta() }
+        properties.layers.firstOrNull()?.let { set("properties", it) }
+        "plugins" putIndexed plugins.map { it.toMeta() }
     }
 
     public companion object {
@@ -87,9 +105,6 @@ public open class Context internal constructor(
     }
 }
 
-public fun Context(name: String, parent: Context = Global, block: ContextBuilder.() -> Unit = {}): Context =
-    Global.context(name, parent, block)
-
 /**
  * The interface for something that encapsulated in context
  *

dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/context/ContextBuilder.kt

@@ -0,0 +1,111 @@
+package space.kscience.dataforge.context
+
+import space.kscience.dataforge.meta.Meta
+import space.kscience.dataforge.meta.MutableMeta
+import space.kscience.dataforge.meta.seal
+import space.kscience.dataforge.meta.toMutableMeta
+import space.kscience.dataforge.misc.DFBuilder
+import space.kscience.dataforge.misc.DFExperimental
+import space.kscience.dataforge.names.Name
+import space.kscience.dataforge.names.NameToken
+import space.kscience.dataforge.names.asName
+import space.kscience.dataforge.names.plus
+import kotlin.collections.component1
+import kotlin.collections.component2
+import kotlin.collections.set
+
+/**
+ * A convenience builder for context
+ */
+@DFBuilder
+public class ContextBuilder internal constructor(
+    private val parent: Context,
+    public val name: Name? = null,
+    meta: Meta = Meta.EMPTY,
+) {
+    internal val factories = HashMap<PluginFactory<*>, Meta>()
+    internal var meta = meta.toMutableMeta()
+
+    public fun properties(action: MutableMeta.() -> Unit) {
+        meta.action()
+    }
+
+    @OptIn(DFExperimental::class)
+    private fun findPluginFactory(tag: PluginTag): PluginFactory<*> =
+        parent.gatherInSequence<PluginFactory<*>>(PluginFactory.TYPE).values
+            .find { it.tag.matches(tag) } ?: error("Can't resolve plugin factory for $tag")
+
+    public fun plugin(tag: PluginTag, mutableMeta: MutableMeta.() -> Unit = {}) {
+        val factory = findPluginFactory(tag)
+        factories[factory] = Meta(mutableMeta)
+    }
+
+    public fun plugin(factory: PluginFactory<*>, meta: Meta) {
+        factories[factory] = meta
+    }
+
+    public fun plugin(factory: PluginFactory<*>, mutableMeta: MutableMeta.() -> Unit = {}) {
+        factories[factory] = Meta(mutableMeta)
+    }
+
+    public fun plugin(name: String, group: String = "", version: String = "", action: MutableMeta.() -> Unit = {}) {
+        plugin(PluginTag(name, group, version), action)
+    }
+
+    /**
+     * Add de-facto existing plugin as a dependency
+     */
+    public fun plugin(plugin: Plugin) {
+        plugin(DeFactoPluginFactory(plugin))
+    }
+
+    public fun build(): Context {
+        val contextName = name ?: NameToken("@auto",hashCode().toUInt().toString(16)).asName()
+        val plugins = HashMap<PluginTag, Plugin>()
+
+        fun addPlugin(factory: PluginFactory<*>, meta: Meta) {
+            val existing = plugins[factory.tag]
+            // Add if does not exist
+            if (existing == null) {
+                //TODO bypass if parent already has plugin with given meta?
+                val plugin = factory.build(parent, meta)
+
+                for ((depFactory, deoMeta) in plugin.dependsOn()) {
+                    addPlugin(depFactory, deoMeta)
+                }
+
+                parent.logger.info { "Loading plugin ${plugin.name} into $contextName" }
+                plugins[plugin.tag] = plugin
+            } else if (existing.meta != meta) {
+                error("Plugin with tag ${factory.tag} and meta $meta already exists in $contextName")
+            }
+            //bypass if exists with the same meta
+        }
+
+        factories.forEach { (factory, meta) ->
+            addPlugin(factory, meta)
+        }
+
+        return Context(contextName, parent, plugins.values.toSet(), meta.seal())
+    }
+}
+
+/**
+ * Check if current context contains all plugins required by the builder and return it does or forks to a new context
+ * if it does not.
+ */
+@DFExperimental
+public fun Context.modify(block: ContextBuilder.() -> Unit): Context {
+
+    fun Context.contains(factory: PluginFactory<*>, meta: Meta): Boolean {
+        val loaded = plugins[factory.tag] ?: return false
+        return loaded.meta == meta
+    }
+
+    val builder = ContextBuilder(this, name + "mod", properties).apply(block)
+    val requiresFork = builder.factories.any { (factory, meta) ->
+        !contains(factory, meta)
+    } || ((properties as Meta) == builder.meta)
+
+    return if (requiresFork) builder.build() else this
+}

dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/context/Factory.kt

@@ -0,0 +1,12 @@
+package space.kscience.dataforge.context
+
+import space.kscience.dataforge.meta.Meta
+
+public fun interface Factory<out T> {
+    public fun build(context: Context, meta: Meta): T
+}
+
+public operator fun <T> Factory<T>.invoke(
+    meta: Meta = Meta.EMPTY,
+    context: Context = Global,
+): T = build(context, meta)

dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/context/Global.kt

@@ -0,0 +1,24 @@
+package space.kscience.dataforge.context
+
+import kotlinx.coroutines.CoroutineName
+import kotlinx.coroutines.Job
+import space.kscience.dataforge.meta.Meta
+import space.kscience.dataforge.names.asName
+import space.kscience.dataforge.names.parseAsName
+import kotlin.coroutines.CoroutineContext
+import kotlin.native.concurrent.ThreadLocal
+
+internal expect fun getGlobalLoggerFactory(): PluginFactory<out LogManager>
+
+/**
+ * A global root context. Closing [Global] terminates the framework.
+ */
+@ThreadLocal
+private object GlobalContext : Context("GLOBAL".asName(), null, emptySet(), Meta.EMPTY) {
+    override val coroutineContext: CoroutineContext = Job() + CoroutineName("GlobalContext")
+}
+
+public val Global: Context get() = GlobalContext
+
+public fun Context(name: String? = null, block: ContextBuilder.() -> Unit = {}): Context =
+    Global.buildContext(name?.parseAsName(), block)

dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/context/LogManager.kt

@@ -0,0 +1,89 @@
+package space.kscience.dataforge.context
+
+import space.kscience.dataforge.meta.Meta
+import space.kscience.dataforge.misc.Named
+import space.kscience.dataforge.names.Name
+import space.kscience.dataforge.names.plus
+
+public fun interface Logger {
+    public fun log(tag: String, body: () -> String)
+}
+
+public interface LogManager : Plugin, Logger {
+    public fun logger(name: Name): Logger
+
+    public val defaultLogger: Logger
+
+    override fun log(tag: String, body: () -> String): Unit = defaultLogger.log(tag, body)
+
+    public fun log(name: Name, tag: String, body: () -> String): Unit = logger(name).log(tag, body)
+
+    public companion object {
+        public const val TRACE: String = "TRACE"
+        public const val INFO: String = "INFO"
+        public const val DEBUG: String = "DEBUG"
+        public const val WARNING: String = "WARNING"
+        public const val ERROR: String = "ERROR"
+    }
+}
+
+public fun Logger.trace(body: () -> String): Unit = log(LogManager.TRACE, body)
+public fun Logger.info(body: () -> String): Unit = log(LogManager.INFO, body)
+public fun Logger.debug(body: () -> String): Unit = log(LogManager.DEBUG, body)
+public fun Logger.warn(body: () -> String): Unit = log(LogManager.WARNING, body)
+public fun Logger.error(body: () -> String): Unit = log(LogManager.ERROR, body)
+
+internal val (() -> String).safe: String
+    get() = try {
+        invoke()
+    } catch (t: Throwable) {
+        "Error while evaluating log string: ${t.message}"
+    }
+
+
+public fun Logger.error(throwable: Throwable?, body: () -> String): Unit = log(LogManager.ERROR) {
+    buildString {
+        appendLine(body())
+        throwable?.let { appendLine(throwable.stackTraceToString()) }
+    }
+}
+
+
+public class DefaultLogManager : AbstractPlugin(), LogManager {
+
+    override fun logger(name: Name): Logger = Logger { tag, body ->
+        val message: String = body.safe
+        println("$tag $name: [${context.name}] $message")
+    }
+
+    override val defaultLogger: Logger = logger(Name.EMPTY)
+
+
+    override val tag: PluginTag get() = Companion.tag
+
+    public companion object : PluginFactory<DefaultLogManager> {
+        override fun build(context: Context, meta: Meta): DefaultLogManager = DefaultLogManager()
+
+        override val tag: PluginTag = PluginTag(group = PluginTag.DATAFORGE_GROUP, name = "log.default")
+    }
+}
+
+/**
+ * Context log manager inherited from parent
+ */
+public val Context.logger: LogManager
+    get() = plugins.find(inherit = true) { it is LogManager } as? LogManager
+        ?: getGlobalLoggerFactory().build(context = Global, meta = Meta.EMPTY).apply { attach(Global) }
+
+/**
+ * The named proxy logger for a context member
+ */
+public val ContextAware.logger: Logger
+    get() = if (this is Named) {
+        Logger { tag, body ->
+            context.logger.log(this@logger.name + name, tag, body)
+        }
+    } else {
+        context.logger
+    }
+

dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/context/Plugin.kt

@@ -1,13 +1,13 @@
-package hep.dataforge.context
+package space.kscience.dataforge.context
 
-import hep.dataforge.context.Plugin.Companion.TARGET
-import hep.dataforge.meta.Meta
-import hep.dataforge.meta.MetaRepr
-import hep.dataforge.misc.Named
-import hep.dataforge.misc.Type
-import hep.dataforge.names.Name
-import hep.dataforge.names.toName
-import hep.dataforge.provider.Provider
+import space.kscience.dataforge.context.Plugin.Companion.TARGET
+import space.kscience.dataforge.meta.Meta
+import space.kscience.dataforge.meta.MetaRepr
+import space.kscience.dataforge.misc.DfId
+import space.kscience.dataforge.misc.Named
+import space.kscience.dataforge.names.Name
+import space.kscience.dataforge.names.parseAsName
+import space.kscience.dataforge.provider.Provider
 
 /**
  * The interface to define a Context plugin. A plugin stores all runtime features of a context.
@@ -18,7 +18,7 @@ import hep.dataforge.provider.Provider
  *
  * create - configure - attach - detach - destroy
  */
-@Type(TARGET)
+@DfId(TARGET)
 public interface Plugin : Named, ContextAware, Provider, MetaRepr {
 
     /**
@@ -31,7 +31,7 @@ public interface Plugin : Named, ContextAware, Provider, MetaRepr {
     /**
      * The name of this plugin ignoring version and group
      */
-    override val name: Name get() = tag.name.toName()
+    override val name: Name get() = tag.name.parseAsName()
 
     /**
      * Plugin dependencies which are required to attach this plugin. Plugin
@@ -53,6 +53,8 @@ public interface Plugin : Named, ContextAware, Provider, MetaRepr {
      */
     public fun detach()
 
+    public val isAttached: Boolean
+
     override fun toMeta(): Meta = Meta {
         "context" put context.name.toString()
         "type" to this::class.simpleName
@@ -63,5 +65,4 @@ public interface Plugin : Named, ContextAware, Provider, MetaRepr {
     public companion object {
         public const val TARGET: String = "plugin"
     }
-
 }

dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/context/PluginBuilder.kt

@@ -0,0 +1,57 @@
+package space.kscience.dataforge.context
+
+import space.kscience.dataforge.meta.Meta
+import space.kscience.dataforge.misc.Named
+import space.kscience.dataforge.names.Name
+
+/**
+ * A convenience factory to build simple plugins
+ */
+public class PluginBuilder(
+    name: String,
+    group: String = "",
+    version: String = "",
+) {
+    public val tag: PluginTag = PluginTag(name, group, version)
+
+    private val content = HashMap<String, MutableMap<Name, Any>>()
+    private val dependencies = HashMap<PluginFactory<*>, Meta>()
+
+    public fun requires(
+        factory: PluginFactory<*>,
+        meta: Meta = Meta.EMPTY,
+    ) {
+        dependencies[factory] = meta
+    }
+
+    public fun provides(target: String, items: Map<Name, Any>) {
+        content.getOrPut(target) { HashMap() }.putAll(items)
+    }
+
+    public fun provides(target: String, vararg items: Named) {
+        provides(target, items.associateBy { it.name })
+    }
+
+    public fun build(): PluginFactory<*> {
+
+        return object : PluginFactory<Plugin> {
+            override val tag: PluginTag get() = this@PluginBuilder.tag
+
+            override fun build(context: Context, meta: Meta): Plugin = object : AbstractPlugin() {
+                override val tag: PluginTag get() = this@PluginBuilder.tag
+
+                override fun content(target: String): Map<Name, Any> = this@PluginBuilder.content[target] ?: emptyMap()
+
+                override fun dependsOn(): Map<PluginFactory<*>, Meta> = this@PluginBuilder.dependencies
+            }
+
+        }
+    }
+}
+
+public fun PluginFactory(
+    name: String,
+    group: String = "",
+    version: String = "",
+    block: PluginBuilder.() -> Unit,
+): PluginFactory<*> = PluginBuilder(name, group, version).apply(block).build()

dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/context/PluginFactory.kt

@@ -0,0 +1,21 @@
+package space.kscience.dataforge.context
+
+import space.kscience.dataforge.meta.Meta
+import space.kscience.dataforge.misc.DfId
+
+@DfId(PluginFactory.TYPE)
+public interface PluginFactory<T : Plugin> : Factory<T> {
+    public val tag: PluginTag
+
+    public companion object {
+        public const val TYPE: String = "pluginFactory"
+    }
+}
+
+/**
+ * Plugin factory created for the specific actual plugin
+ */
+internal class DeFactoPluginFactory<T : Plugin>(val plugin: T) : PluginFactory<T> {
+    override fun build(context: Context, meta: Meta): T = plugin
+    override val tag: PluginTag get() = plugin.tag
+}

dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/context/PluginManager.kt

@@ -0,0 +1,100 @@
+package space.kscience.dataforge.context
+
+import space.kscience.dataforge.meta.Meta
+import space.kscience.dataforge.misc.DFInternal
+import space.kscience.dataforge.names.plus
+import kotlin.reflect.KClass
+import kotlin.reflect.cast
+
+
+/**
+ * The manager for plugin system. Should monitor plugin dependencies and locks.
+ *
+ * @property context A context for this plugin manager
+ * @author Alexander Nozik
+ */
+public class PluginManager internal constructor(
+    override val context: Context,
+    private val plugins: Set<Plugin>,
+) : ContextAware, Iterable<Plugin> {
+
+    init {
+        plugins.forEach { it.attach(context) }
+    }
+
+    /**
+     * A [PluginManager] of parent context if it is present
+     */
+    private val parent: PluginManager? = context.parent?.plugins
+
+    /**
+     * List plugins stored in this [PluginManager]. If [inherit] is true, include parent plugins as well
+     */
+    public fun list(inherit: Boolean): Collection<Plugin> {
+        return if (inherit && parent != null) {
+            plugins + parent.list(true)
+        } else {
+            plugins
+        }
+    }
+
+    /**
+     * Get existing plugin or return null if not present. Only first matching plugin is returned.
+     * @param inherit search for parent [PluginManager] plugins
+     * @param predicate condition for the plugin
+     */
+    public fun find(inherit: Boolean = true, predicate: (Plugin) -> Boolean): Plugin? =
+        list(inherit).find(predicate)
+
+    /**
+     * Find a loaded plugin via its tag
+     *
+     * @param tag
+     * @return
+     */
+    public operator fun get(tag: PluginTag, inherit: Boolean = true): Plugin? =
+        find(inherit) { tag.matches(it.tag) }
+
+    /**
+     * Find a loaded plugin via its class. This method does not check if the result is unique and just returns first
+     * plugin matching the class condition.
+     * For safe search provide a tag since tags are checked on load and plugins with the same tag are not allowed
+     * in the same context.
+     *
+     * @param tag
+     * @param type
+     * @param <T>
+     * @return
+     */
+    @DFInternal
+    public fun <T : Any> getByType(type: KClass<T>, tag: PluginTag? = null, inherit: Boolean = true): T? =
+        find(inherit) { type.isInstance(it) && (tag == null || tag.matches(it.tag)) }?.let { type.cast(it) }
+
+    @OptIn(DFInternal::class)
+    public inline operator fun <reified T : Any> get(tag: PluginTag? = null, recursive: Boolean = true): T? =
+        getByType(T::class, tag, recursive)
+
+    @OptIn(DFInternal::class)
+    public inline operator fun <reified T : Plugin> get(factory: PluginFactory<T>, recursive: Boolean = true): T? =
+        getByType(T::class, factory.tag, recursive)
+
+    override fun iterator(): Iterator<Plugin> = plugins.iterator()
+}
+
+/**
+ * Fetch a plugin with given meta from the context. If the plugin (with given meta) is already registered, it is returned.
+ * Otherwise, new child context with the plugin is created. In the later case the context could be retrieved from the plugin.
+ */
+public inline fun <reified T : Plugin> Context.request(factory: PluginFactory<T>, meta: Meta = Meta.EMPTY): T {
+    val existing = plugins[factory]
+    return if (existing != null && existing.meta == meta) existing
+    else {
+        buildContext(name = this@request.name + factory.tag.name) {
+            plugin(factory, meta)
+        }.plugins[factory]!!
+    }
+}
+
+@Deprecated("Replace with request", ReplaceWith("request(factory, meta)"))
+public inline fun <reified T : Plugin> Context.fetch(factory: PluginFactory<T>, meta: Meta = Meta.EMPTY): T =
+    request(factory, meta)

dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/context/PluginTag.kt

@@ -1,7 +1,8 @@
-package hep.dataforge.context
+package space.kscience.dataforge.context
 
-import hep.dataforge.meta.Meta
-import hep.dataforge.meta.MetaRepr
+import kotlinx.serialization.Serializable
+import space.kscience.dataforge.meta.Meta
+import space.kscience.dataforge.meta.MetaRepr
 
 /**
  * The tag which contains information about name, group and version of some
@@ -9,6 +10,7 @@ import hep.dataforge.meta.MetaRepr
  *
  * @author Alexander Nozik
  */
+@Serializable
 public data class PluginTag(
     val name: String,
     val group: String = "",
@@ -43,7 +45,7 @@ public data class PluginTag(
 
     public companion object {
 
-        public const val DATAFORGE_GROUP: String = "hep.dataforge"
+        public const val DATAFORGE_GROUP: String = "dataforge"
 
         /**
          * Build new PluginTag from standard string representation

dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/context/resolve.kt

@@ -1,10 +1,10 @@
-package hep.dataforge.context
+package space.kscience.dataforge.context
 
-import hep.dataforge.misc.DFExperimental
-import hep.dataforge.names.Name
-import hep.dataforge.names.plus
-import hep.dataforge.provider.Provider
-import hep.dataforge.provider.top
+import space.kscience.dataforge.misc.DFExperimental
+import space.kscience.dataforge.names.Name
+import space.kscience.dataforge.names.plus
+import space.kscience.dataforge.provider.Provider
+import space.kscience.dataforge.provider.top
 import kotlin.reflect.KClass
 import kotlin.reflect.cast
 

dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/properties/MetaProperty.kt

@@ -0,0 +1,35 @@
+package space.kscience.dataforge.properties
+
+
+import space.kscience.dataforge.meta.Meta
+import space.kscience.dataforge.meta.ObservableMutableMeta
+import space.kscience.dataforge.meta.transformations.MetaConverter
+import space.kscience.dataforge.meta.transformations.nullableMetaToObject
+import space.kscience.dataforge.meta.transformations.nullableObjectToMeta
+import space.kscience.dataforge.misc.DFExperimental
+import space.kscience.dataforge.names.Name
+import space.kscience.dataforge.names.startsWith
+
+@DFExperimental
+public class MetaProperty<T : Any>(
+    public val meta: ObservableMutableMeta,
+    public val name: Name,
+    public val converter: MetaConverter<T>,
+) : Property<T?> {
+
+    override var value: T?
+        get() = converter.nullableMetaToObject(meta[name])
+        set(value) {
+            meta[name] = converter.nullableObjectToMeta(value) ?: Meta.EMPTY
+        }
+
+    override fun onChange(owner: Any?, callback: (T?) -> Unit) {
+        meta.onChange(owner) { name ->
+            if (name.startsWith(this@MetaProperty.name)) callback(converter.nullableMetaToObject(this[name]))
+        }
+    }
+
+    override fun removeChangeListener(owner: Any?) {
+        meta.removeListener(owner)
+    }
+}

dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/properties/Property.kt

@@ -1,10 +1,9 @@
-package hep.dataforge.properties
+package space.kscience.dataforge.properties
 
-import hep.dataforge.misc.DFExperimental
-import kotlinx.coroutines.CoroutineScope
 import kotlinx.coroutines.ExperimentalCoroutinesApi
 import kotlinx.coroutines.flow.MutableStateFlow
 import kotlinx.coroutines.flow.StateFlow
+import space.kscience.dataforge.misc.DFExperimental
 
 @DFExperimental
 public interface Property<T> {
@@ -23,12 +22,12 @@ public fun <T> Property<T>.toFlow(): StateFlow<T> = MutableStateFlow(value).also
 }
 
 /**
- * Reflect all changes in the [source] property onto this property
+ * Reflect all changes in the [source] property onto this property. Does not reflect changes back.
  *
  * @return a mirroring job
  */
 @DFExperimental
-public fun <T> Property<T>.mirror(source: Property<T>, scope: CoroutineScope) {
+public fun <T> Property<T>.mirror(source: Property<T>) {
     source.onChange(this) {
         this.value = it
     }

dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/properties/schemeProperty.kt

@@ -0,0 +1,31 @@
+package space.kscience.dataforge.properties
+
+
+import space.kscience.dataforge.meta.Scheme
+import space.kscience.dataforge.misc.DFExperimental
+import space.kscience.dataforge.names.parseAsName
+import space.kscience.dataforge.names.startsWith
+import kotlin.reflect.KMutableProperty1
+
+@DFExperimental
+public fun <S : Scheme, T : Any> S.property(property: KMutableProperty1<S, T?>): Property<T?> =
+    object : Property<T?> {
+        override var value: T?
+            get() = property.get(this@property)
+            set(value) {
+                property.set(this@property, value)
+            }
+
+        override fun onChange(owner: Any?, callback: (T?) -> Unit) {
+            this@property.meta.onChange(this) { name ->
+                if (name.startsWith(property.name.parseAsName(true))) {
+                    callback(property.get(this@property))
+                }
+            }
+        }
+
+        override fun removeChangeListener(owner: Any?) {
+            this@property.meta.removeListener(this@property)
+        }
+
+    }

dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/provider/Path.kt

@@ -13,16 +13,18 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package hep.dataforge.provider
+package space.kscience.dataforge.provider
 
-import hep.dataforge.names.Name
-import hep.dataforge.names.toName
+import space.kscience.dataforge.names.Name
+import space.kscience.dataforge.names.parseAsName
+import kotlin.jvm.JvmInline
 
 /**
  * Path interface.
  *
  */
-public inline class Path(public val tokens: List<PathToken>) : Iterable<PathToken> {
+@JvmInline
+public value class Path(public val tokens: List<PathToken>) : Iterable<PathToken> {
 
     override fun iterator(): Iterator<PathToken> = tokens.iterator()
 
@@ -31,11 +33,7 @@ public inline class Path(public val tokens: List<PathToken>) : Iterable<PathToke
     public companion object {
         public const val PATH_SEGMENT_SEPARATOR: String = "/"
 
-        public fun parse(path: String): Path {
-            val head = path.substringBefore(PATH_SEGMENT_SEPARATOR)
-            val tail = path.substringAfter(PATH_SEGMENT_SEPARATOR)
-            return PathToken.parse(head).asPath() + parse(tail)
-        }
+        public fun parse(path: String): Path = Path(path.split(PATH_SEGMENT_SEPARATOR).map { PathToken.parse(it) })
     }
 }
 
@@ -63,9 +61,10 @@ public data class PathToken(val name: Name, val target: String? = null) {
 
     public companion object {
         public const val TARGET_SEPARATOR: String = "::"
-        public fun parse(token: String): PathToken {
+
+        public fun parse(token: String, cache: Boolean = false): PathToken {
             val target = token.substringBefore(TARGET_SEPARATOR, "")
-            val name = token.substringAfter(TARGET_SEPARATOR).toName()
+            val name = token.substringAfter(TARGET_SEPARATOR).parseAsName(cache)
             if (target.contains("[")) TODO("target separators in queries are not supported")
             return PathToken(name, target)
         }

dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/provider/Provider.kt

@@ -13,9 +13,9 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package hep.dataforge.provider
+package space.kscience.dataforge.provider
 
-import hep.dataforge.names.Name
+import space.kscience.dataforge.names.Name
 import kotlin.reflect.KClass
 import kotlin.reflect.safeCast
 
@@ -75,10 +75,8 @@ public inline fun <reified T : Any> Provider.provide(path: String, targetOverrid
 /**
  *  Typed top level content
  */
-public fun <T : Any> Provider.top(target: String, type: KClass<out T>): Map<Name, T> {
-    return content(target).mapValues {
-        type.safeCast(it.value) ?: error("The type of element $it is ${it::class} but $type is expected")
-    }
+public fun <T : Any> Provider.top(target: String, type: KClass<out T>): Map<Name, T> = content(target).mapValues {
+    type.safeCast(it.value) ?: error("The type of element ${it.value} is ${it.value::class} but $type is expected")
 }
 
 /**

dataforge-context/src/commonTest/kotlin/space/kscience/dataforge/context/ContextTest.kt

@@ -1,8 +1,7 @@
-package hep.dataforge.context
+package space.kscience.dataforge.context
 
-import hep.dataforge.names.Name
-import hep.dataforge.names.appendLeft
-import hep.dataforge.names.toName
+import space.kscience.dataforge.names.Name
+import space.kscience.dataforge.names.appendFirst
 import kotlin.test.Test
 import kotlin.test.assertEquals
 
@@ -12,23 +11,22 @@ class ContextTest {
         override val tag get() = PluginTag("test")
 
         override fun content(target: String): Map<Name, Any> {
-            return when(target){
-                "test" -> listOf("a", "b", "c.d").associate { it.toName() to it.toName() }
+            return when (target) {
+                "test" -> listOf("a", "b", "c.d").associate { Name.parse(it) to Name.parse(it) }
                 else -> emptyMap()
             }
         }
-
     }
 
     @Test
     fun testPluginManager() {
-        val context = Global.context("test")
-        context.plugins.load(DummyPlugin())
-        //Global.plugins.load(DummyPlugin())
+        val context = Context("test") {
+            plugin(DummyPlugin())
+        }
         val members = context.gather<Name>("test")
         assertEquals(3, members.count())
         members.forEach {
-            assertEquals(it.key, it.value.appendLeft("test"))
+            assertEquals(it.key, it.value.appendFirst("test"))
         }
     }
 

dataforge-context/src/commonTest/kotlin/space/kscience/dataforge/properties/MetaPropertiesTest.kt

@@ -0,0 +1,28 @@
+package space.kscience.dataforge.properties
+
+import space.kscience.dataforge.meta.Scheme
+import space.kscience.dataforge.meta.SchemeSpec
+import space.kscience.dataforge.meta.int
+import space.kscience.dataforge.misc.DFExperimental
+import kotlin.test.Test
+import kotlin.test.assertEquals
+
+internal class TestScheme : Scheme() {
+    var a by int()
+    var b by int()
+    companion object : SchemeSpec<TestScheme>(::TestScheme)
+}
+
+@DFExperimental
+class MetaPropertiesTest {
+    @Test
+    fun testBinding() {
+        val scheme = TestScheme.empty()
+        val a = scheme.property(TestScheme::a)
+        val b = scheme.property(TestScheme::b)
+        a.bind(b)
+        scheme.a = 2
+        assertEquals(2, scheme.b)
+        assertEquals(2, b.value)
+    }
+}

dataforge-context/src/commonTest/kotlin/space/kscience/dataforge/provider/PathTest.kt

@@ -0,0 +1,14 @@
+package space.kscience.dataforge.provider
+
+import kotlin.test.Test
+import kotlin.test.assertEquals
+
+class PathTest {
+    @Test
+    fun testParse(){
+        val nameString = "a.b.c.d"
+        val pathString = "a.b/c.d"
+        assertEquals(1, Path.parse(nameString).length)
+        assertEquals(2, Path.parse(pathString).length)
+    }
+}

dataforge-context/src/jsMain/kotlin/space/kscience/dataforge/context/loggingJs.kt

@@ -0,0 +1,32 @@
+package space.kscience.dataforge.context
+
+import space.kscience.dataforge.meta.Meta
+import space.kscience.dataforge.names.Name
+
+public class ConsoleLogManager : AbstractPlugin(), LogManager {
+
+    override fun logger(name: Name): Logger  = Logger { tag, body ->
+        val message: String = body.safe
+        when (tag) {
+            // TODO depends on https://youtrack.jetbrains.com/issue/KT-33595/
+            LogManager.DEBUG -> console.asDynamic().debug("[${context.name}] $name: $message")
+            LogManager.INFO -> console.info("[${context.name}] $name: $message")
+            LogManager.WARNING ->  console.warn("[${context.name}] $name: $message")
+            LogManager.ERROR -> console.error("[${context.name}] $name: $message")
+            else ->  console.log("[${context.name}] $name: $message")
+        }
+    }
+
+    override val defaultLogger: Logger = logger(Name.EMPTY)
+
+
+    override val tag: PluginTag get() = Companion.tag
+
+    public companion object : PluginFactory<ConsoleLogManager> {
+        override fun build(context: Context, meta: Meta): ConsoleLogManager = ConsoleLogManager()
+
+        override val tag: PluginTag = PluginTag(group = PluginTag.DATAFORGE_GROUP, name = "log.jsConsole")
+    }
+}
+
+internal actual fun getGlobalLoggerFactory(): PluginFactory<out LogManager> = ConsoleLogManager

dataforge-context/src/jsMain/kotlin/space/kscience/dataforge/properties/bindings.kt

@@ -1,7 +1,7 @@
-package hep.dataforge.properties
+package space.kscience.dataforge.properties
 
-import hep.dataforge.misc.DFExperimental
 import org.w3c.dom.HTMLInputElement
+import space.kscience.dataforge.misc.DFExperimental
 
 @DFExperimental
 public fun HTMLInputElement.bindValue(property: Property<String>) {

dataforge-context/src/jvmMain/kotlin/hep/dataforge/provider/dfType.kt

@@ -1,37 +0,0 @@
-package hep.dataforge.provider
-
-import hep.dataforge.context.Context
-import hep.dataforge.context.gather
-import hep.dataforge.misc.DFExperimental
-import hep.dataforge.misc.Type
-import hep.dataforge.names.Name
-import kotlin.reflect.KClass
-import kotlin.reflect.full.findAnnotation
-
-
-@DFExperimental
-public val KClass<*>.dfType: String
-    get() = findAnnotation<Type>()?.id ?: simpleName ?: ""
-
-/**
- * Provide an object with given name inferring target from its type using [Type] annotation
- */
-@DFExperimental
-public inline fun <reified T : Any> Provider.provideByType(name: String): T? {
-    val target = T::class.dfType
-    return provide(target, name)
-}
-
-@DFExperimental
-public inline fun <reified T : Any> Provider.top(): Map<Name, T> {
-    val target = T::class.dfType
-    return top(target)
-}
-
-/**
- * All objects provided by plugins with given target and type
- */
-@DFExperimental
-public inline fun <reified T : Any> Context.gather(inherit: Boolean = true): Map<Name, T> =
-    gather<T>(T::class.dfType, inherit)
-

dataforge-context/src/jvmMain/kotlin/space/kscience/dataforge/context/ClassLoaderPlugin.kt

@@ -13,7 +13,7 @@
  *  See the License for the specific language governing permissions and
  *  limitations under the License.
  */
-package hep.dataforge.context
+package space.kscience.dataforge.context
 
 import java.util.*
 import kotlin.reflect.KClass

dataforge-context/src/jvmMain/kotlin/space/kscience/dataforge/context/loggingJvm.kt

@@ -0,0 +1,32 @@
+package space.kscience.dataforge.context
+
+import org.slf4j.LoggerFactory
+import space.kscience.dataforge.meta.Meta
+import space.kscience.dataforge.names.Name
+
+public class SlfLogManager : AbstractPlugin(), LogManager {
+
+    override fun logger(name: Name): Logger = Logger { tag, body ->
+        val logger = LoggerFactory.getLogger("[${context.name}] $name") //KotlinLogging.logger("[${context.name}] $name")
+        val message = body.safe
+        when (tag) {
+            LogManager.DEBUG -> logger.debug(message)
+            LogManager.INFO -> logger.info(message)
+            LogManager.WARNING -> logger.warn(message)
+            LogManager.ERROR -> logger.error(message)
+            else -> logger.trace(message)
+        }
+    }
+
+    override val defaultLogger: Logger = logger(Name.EMPTY)
+
+    override val tag: PluginTag get() = Companion.tag
+
+    public companion object : PluginFactory<SlfLogManager> {
+        override fun build(context: Context, meta: Meta): SlfLogManager = SlfLogManager()
+
+        override val tag: PluginTag = PluginTag(group = PluginTag.DATAFORGE_GROUP, name = "log.kotlinLogging")
+    }
+}
+
+internal actual fun getGlobalLoggerFactory(): PluginFactory<out LogManager> = SlfLogManager

dataforge-context/src/jvmMain/kotlin/space/kscience/dataforge/descriptors/annotations.kt

@@ -14,37 +14,34 @@
  *  limitations under the License.
  */
 
-package hep.dataforge.descriptors
+package space.kscience.dataforge.descriptors
 
-import hep.dataforge.values.ValueType
-import kotlin.reflect.KClass
-
-@MustBeDocumented
-annotation class Attribute(
-    val key: String,
-    val value: String
-)
-
-@MustBeDocumented
-annotation class Attributes(
-    val attrs: Array<Attribute>
-)
-
-@MustBeDocumented
-annotation class ItemDef(
-    val info: String = "",
-    val multiple: Boolean = false,
-    val required: Boolean = false
-)
-
-@Target(AnnotationTarget.PROPERTY)
-@MustBeDocumented
-annotation class ValueDef(
-    val type: Array<ValueType> = [ValueType.STRING],
-    val def: String = "",
-    val allowed: Array<String> = [],
-    val enumeration: KClass<*> = Any::class
-)
+//@MustBeDocumented
+//annotation class Attribute(
+//    val key: String,
+//    val value: String
+//)
+//
+//@MustBeDocumented
+//annotation class Attributes(
+//    val attrs: Array<Attribute>
+//)
+//
+//@MustBeDocumented
+//annotation class ItemDef(
+//    val info: String = "",
+//    val multiple: Boolean = false,
+//    val required: Boolean = false
+//)
+//
+//@Target(AnnotationTarget.PROPERTY)
+//@MustBeDocumented
+//annotation class ValueDef(
+//    val type: Array<ValueType> = [ValueType.STRING],
+//    val def: String = "",
+//    val allowed: Array<String> = [],
+//    val enumeration: KClass<*> = Any::class
+//)
 
 ///**
 // * Description text for meta property, node or whole object

dataforge-context/src/jvmMain/kotlin/space/kscience/dataforge/descriptors/reflectiveDescriptors.kt

@@ -1,4 +1,4 @@
-package hep.dataforge.descriptors
+package space.kscience.dataforge.descriptors
 
 
 //inline fun <reified T : Scheme> T.buildDescriptor(): NodeDescriptor = NodeDescriptor {

dataforge-context/src/jvmMain/kotlin/space/kscience/dataforge/provider/dfType.kt

@@ -0,0 +1,49 @@
+package space.kscience.dataforge.provider
+
+import space.kscience.dataforge.context.Context
+import space.kscience.dataforge.context.PluginBuilder
+import space.kscience.dataforge.context.gather
+import space.kscience.dataforge.misc.DFExperimental
+import space.kscience.dataforge.misc.DfId
+import space.kscience.dataforge.misc.Named
+import space.kscience.dataforge.names.Name
+import kotlin.reflect.KClass
+import kotlin.reflect.full.findAnnotation
+
+
+@DFExperimental
+public val KClass<*>.dfId: String
+    get() = findAnnotation<DfId>()?.id ?: simpleName ?: ""
+
+/**
+ * Provide an object with given name inferring target from its type using [DfId] annotation
+ */
+@DFExperimental
+public inline fun <reified T : Any> Provider.provideByType(name: String): T? {
+    val target = T::class.dfId
+    return provide(target, name)
+}
+
+@DFExperimental
+public inline fun <reified T : Any> Provider.top(): Map<Name, T> {
+    val target = T::class.dfId
+    return top(target)
+}
+
+/**
+ * All objects provided by plugins with given target and type
+ */
+@DFExperimental
+public inline fun <reified T : Any> Context.gather(inherit: Boolean = true): Map<Name, T> =
+    gather<T>(T::class.dfId, inherit)
+
+
+@DFExperimental
+public inline fun <reified T : Any> PluginBuilder.provides(items: Map<Name, T>) {
+    provides(T::class.dfId, items)
+}
+
+@DFExperimental
+public inline fun <reified T : Any> PluginBuilder.provides(vararg items: Named) {
+    provides(T::class.dfId, *items)
+}

dataforge-context/src/nativeMain/kotlin/space/kscience/dataforge/context/loggingNative.kt

@@ -0,0 +1,4 @@
+package space.kscience.dataforge.context
+
+
+internal actual fun getGlobalLoggerFactory(): PluginFactory<out LogManager> = DefaultLogManager

dataforge-data/README.md

@@ -0,0 +1,23 @@
+# Module dataforge-data
+
+
+
+## Usage
+
+## Artifact:
+
+The Maven coordinates of this project are `space.kscience:dataforge-data:0.7.0`.
+
+**Gradle Kotlin DSL:**
+```kotlin
+repositories {
+    maven("https://repo.kotlin.link")
+    //uncomment to access development builds
+    //maven("https://maven.pkg.jetbrains.space/spc/p/sci/dev")
+    mavenCentral()
+}
+
+dependencies {
+    implementation("space.kscience:dataforge-data:0.7.0")
+}
+```

dataforge-data/build.gradle.kts

@@ -1,23 +1,18 @@
 plugins {
-    id("ru.mipt.npm.mpp")
-    id("ru.mipt.npm.native")
+    id("space.kscience.gradle.mpp")
 }
 
 kscience{
+    jvm()
+    js()
+    native()
     useCoroutines()
-}
-
-kotlin {
-    sourceSets {
-        commonMain{
-            dependencies {
-                api(project(":dataforge-meta"))
-                api(kotlin("reflect"))
-            }
-        }
+    dependencies {
+        api(project(":dataforge-meta"))
+        api(kotlin("reflect"))
     }
 }
 
 readme{
-    maturity = ru.mipt.npm.gradle.Maturity.EXPERIMENTAL
+    maturity = space.kscience.gradle.Maturity.EXPERIMENTAL
 }

dataforge-data/src/commonMain/kotlin/hep/dataforge/actions/Action.kt

@@ -1,41 +0,0 @@
-package hep.dataforge.actions
-
-import hep.dataforge.data.DataSet
-import hep.dataforge.meta.Meta
-import hep.dataforge.misc.DFExperimental
-import kotlinx.coroutines.CoroutineScope
-
-/**
- * A simple data transformation on a data node. Actions should avoid doing actual dependency evaluation in [execute].
- */
-public interface Action<in T : Any, out R : Any> {
-    /**
-     * Transform the data in the node, producing a new node. By default it is assumed that all calculations are lazy
-     * so not actual computation is started at this moment.
-     *
-     * [scope] context used to compute the initial result, also it is used for updates propagation
-     */
-    public suspend fun execute(dataSet: DataSet<T>, meta: Meta = Meta.EMPTY, scope: CoroutineScope? = null): DataSet<R>
-
-    public companion object
-}
-
-/**
- * Action composition. The result is terminal if one of its parts is terminal
- */
-public infix fun <T : Any, I : Any, R : Any> Action<T, I>.then(action: Action<I, R>): Action<T, R> {
-    // TODO introduce composite action and add optimize by adding action to the list
-    return object : Action<T, R> {
-        override suspend fun execute(dataSet: DataSet<T>, meta: Meta, scope: CoroutineScope?): DataSet<R> {
-            return action.execute(this@then.execute(dataSet, meta, scope), meta, scope)
-        }
-    }
-}
-
-@DFExperimental
-public suspend fun <T : Any, R : Any> DataSet<T>.transformWith(
-    action: Action<T, R>,
-    meta: Meta = Meta.EMPTY,
-    scope: CoroutineScope? = null,
-): DataSet<R> = action.execute(this, meta, scope)
-

dataforge-data/src/commonMain/kotlin/hep/dataforge/actions/MapAction.kt

@@ -1,104 +0,0 @@
-package hep.dataforge.actions
-
-import hep.dataforge.data.*
-import hep.dataforge.meta.Meta
-import hep.dataforge.meta.MetaBuilder
-import hep.dataforge.meta.seal
-import hep.dataforge.meta.toMutableMeta
-import hep.dataforge.misc.DFBuilder
-import hep.dataforge.misc.DFExperimental
-import hep.dataforge.misc.DFInternal
-import hep.dataforge.names.Name
-import kotlinx.coroutines.CoroutineScope
-import kotlinx.coroutines.flow.collect
-import kotlinx.coroutines.flow.map
-import kotlinx.coroutines.launch
-import kotlin.reflect.KType
-import kotlin.reflect.typeOf
-
-/**
- * Action environment includes data name, data meta and action configuration meta
- */
-public data class ActionEnv(
-    val name: Name,
-    val meta: Meta,
-    val actionMeta: Meta,
-)
-
-/**
- * Action environment
- */
-@DFBuilder
-public class MapActionBuilder<T, R>(public var name: Name, public var meta: MetaBuilder, public val actionMeta: Meta) {
-    public lateinit var result: suspend ActionEnv.(T) -> R
-
-    /**
-     * Calculate the result of goal
-     */
-    public fun result(f: suspend ActionEnv.(T) -> R) {
-        result = f;
-    }
-}
-
-@PublishedApi
-internal class MapAction<in T : Any, out R : Any>(
-    private val outputType: KType,
-    private val block: MapActionBuilder<T, R>.() -> Unit,
-) : Action<T, R> {
-
-    override suspend fun execute(
-        dataSet: DataSet<T>,
-        meta: Meta,
-        scope: CoroutineScope?,
-    ): DataSet<R> {
-        suspend fun mapOne(data: NamedData<T>): NamedData<R> {
-            // Creating a new environment for action using **old** name, old meta and task meta
-            val env = ActionEnv(data.name, data.meta, meta)
-
-            //applying transformation from builder
-            val builder = MapActionBuilder<T, R>(
-                data.name,
-                data.meta.toMutableMeta(), // using data meta
-                meta
-            ).apply(block)
-
-            //getting new name
-            val newName = builder.name
-
-            //getting new meta
-            val newMeta = builder.meta.seal()
-
-            @OptIn(DFInternal::class) val newData = Data(outputType, newMeta, dependencies = listOf(data)) {
-                builder.result(env, data.await())
-            }
-            //setting the data node
-            return newData.named(newName)
-        }
-
-        val flow = dataSet.flow().map(::mapOne)
-
-        return ActiveDataTree(outputType) {
-            populate(flow)
-            scope?.launch {
-                dataSet.updates.collect { name ->
-                    //clear old nodes
-                    remove(name)
-                    //collect new items
-                    populate(dataSet.flowChildren(name).map(::mapOne))
-                }
-            }
-        }
-    }
-}
-
-
-/**
- * A one-to-one mapping action
- */
-@DFExperimental
-@Suppress("FunctionName")
-public inline fun <T : Any, reified R : Any> Action.Companion.map(
-    noinline builder: MapActionBuilder<T, R>.() -> Unit,
-): Action<T, R> = MapAction(typeOf<R>(), builder)
-
-

dataforge-data/src/commonMain/kotlin/hep/dataforge/actions/ReduceAction.kt

@@ -1,116 +0,0 @@
-package hep.dataforge.actions
-
-import hep.dataforge.data.*
-import hep.dataforge.meta.Meta
-import hep.dataforge.meta.MetaBuilder
-import hep.dataforge.misc.DFBuilder
-import hep.dataforge.misc.DFExperimental
-import hep.dataforge.misc.DFInternal
-import hep.dataforge.names.Name
-import hep.dataforge.names.toName
-import kotlinx.coroutines.CoroutineScope
-import kotlinx.coroutines.flow.Flow
-import kotlinx.coroutines.flow.flow
-import kotlinx.coroutines.flow.fold
-import kotlin.reflect.KType
-import kotlin.reflect.typeOf
-
-
-public class JoinGroup<T : Any, R : Any>(public var name: String, internal val set: DataSet<T>) {
-
-    public var meta: MetaBuilder = MetaBuilder()
-
-    public lateinit var result: suspend ActionEnv.(Map<Name, T>) -> R
-
-    public fun result(f: suspend ActionEnv.(Map<Name, T>) -> R) {
-        this.result = f;
-    }
-
-}
-
-@DFBuilder
-public class ReduceGroupBuilder<T : Any, R : Any>(
-    private val inputType: KType,
-    private val scope: CoroutineScope,
-    public val actionMeta: Meta,
-) {
-    private val groupRules: MutableList<suspend (DataSet<T>) -> List<JoinGroup<T, R>>> = ArrayList();
-
-    /**
-     * introduce grouping by meta value
-     */
-    public fun byValue(tag: String, defaultTag: String = "@default", action: JoinGroup<T, R>.() -> Unit) {
-        groupRules += { node ->
-            GroupRule.byMetaValue(scope, tag, defaultTag).gather(node).map {
-                JoinGroup<T, R>(it.key, it.value).apply(action)
-            }
-        }
-    }
-
-    public fun group(
-        groupName: String,
-        filter: suspend (Name, Data<T>) -> Boolean,
-        action: JoinGroup<T, R>.() -> Unit,
-    ) {
-        groupRules += { source ->
-            listOf(
-                JoinGroup<T, R>(groupName, source.filter(filter)).apply(action)
-            )
-        }
-    }
-
-    /**
-     * Apply transformation to the whole node
-     */
-    public fun result(resultName: String, f: suspend ActionEnv.(Map<Name, T>) -> R) {
-        groupRules += { node ->
-            listOf(JoinGroup<T, R>(resultName, node).apply { result(f) })
-        }
-    }
-
-    internal suspend fun buildGroups(input: DataSet<T>): List<JoinGroup<T, R>> {
-        return groupRules.flatMap { it.invoke(input) }
-    }
-
-}
-
-@PublishedApi
-internal class ReduceAction<T : Any, R : Any>(
-    private val inputType: KType,
-    outputType: KType,
-    private val action: ReduceGroupBuilder<T, R>.() -> Unit,
-) : CachingAction<T, R>(outputType) {
-    //TODO optimize reduction. Currently the whole action recalculates on push
-
-
-    override fun CoroutineScope.transform(set: DataSet<T>, meta: Meta, key: Name): Flow<NamedData<R>> = flow {
-        ReduceGroupBuilder<T, R>(inputType, this@transform, meta).apply(action).buildGroups(set).forEach { group ->
-            val dataFlow: Map<Name, Data<T>> = group.set.flow().fold(HashMap()) { acc, value ->
-                acc.apply {
-                    acc[value.name] = value.data
-                }
-            }
-
-            val groupName: String = group.name
-
-            val groupMeta = group.meta
-
-            val env = ActionEnv(groupName.toName(), groupMeta, meta)
-            @OptIn(DFInternal::class) val res: Data<R> = dataFlow.reduceToData(
-                outputType,
-                meta = groupMeta
-            ) { group.result.invoke(env, it) }
-
-            emit(res.named(env.name))
-        }
-    }
-}
-
-/**
- * A one-to-one mapping action
- */
-@DFExperimental
-@Suppress("FunctionName")
-public inline fun <reified T : Any, reified R : Any> Action.Companion.reduce(
-    noinline builder: ReduceGroupBuilder<T, R>.() -> Unit,
-): Action<T, R> = ReduceAction(typeOf<T>(), typeOf<R>(), builder)

dataforge-data/src/commonMain/kotlin/hep/dataforge/actions/SplitAction.kt

@@ -1,96 +0,0 @@
-package hep.dataforge.actions
-
-import hep.dataforge.data.*
-import hep.dataforge.meta.Laminate
-import hep.dataforge.meta.Meta
-import hep.dataforge.meta.MetaBuilder
-import hep.dataforge.meta.toMutableMeta
-import hep.dataforge.misc.DFExperimental
-import hep.dataforge.misc.DFInternal
-import hep.dataforge.names.Name
-import hep.dataforge.names.toName
-import kotlinx.coroutines.CoroutineScope
-import kotlinx.coroutines.FlowPreview
-import kotlinx.coroutines.flow.*
-import kotlinx.coroutines.launch
-import kotlin.collections.set
-import kotlin.reflect.KType
-import kotlin.reflect.typeOf
-
-
-public class SplitBuilder<T : Any, R : Any>(public val name: Name, public val meta: Meta) {
-
-    public class FragmentRule<T : Any, R : Any>(public val name: Name, public var meta: MetaBuilder) {
-        public lateinit var result: suspend (T) -> R
-
-        public fun result(f: suspend (T) -> R) {
-            result = f;
-        }
-    }
-
-    internal val fragments: MutableMap<Name, FragmentRule<T, R>.() -> Unit> = HashMap()
-
-    /**
-     * Add new fragment building rule. If the framgent not defined, result won't be available even if it is present in the map
-     * @param name the name of a fragment
-     * @param rule the rule to transform fragment name and meta using
-     */
-    public fun fragment(name: String, rule: FragmentRule<T, R>.() -> Unit) {
-        fragments[name.toName()] = rule
-    }
-}
-
-/**
- * Action that splits each incoming element into a number of fragments defined in builder
- */
-@PublishedApi
-internal class SplitAction<T : Any, R : Any>(
-    private val outputType: KType,
-    private val action: SplitBuilder<T, R>.() -> Unit,
-) : Action<T, R> {
-
-    @OptIn(FlowPreview::class)
-    override suspend fun execute(
-        dataSet: DataSet<T>,
-        meta: Meta,
-        scope: CoroutineScope?,
-    ): DataSet<R> {
-
-        suspend fun splitOne(data: NamedData<T>): Flow<NamedData<R>> {
-            val laminate = Laminate(data.meta, meta)
-
-            val split = SplitBuilder<T, R>(data.name, data.meta).apply(action)
-
-
-            // apply individual fragment rules to result
-            return split.fragments.entries.asFlow().map { (fragmentName, rule) ->
-                val env = SplitBuilder.FragmentRule<T, R>(fragmentName, laminate.toMutableMeta()).apply(rule)
-                //data.map<R>(outputType, meta = env.meta) { env.result(it) }.named(fragmentName)
-                @OptIn(DFInternal::class) Data(outputType, meta = env.meta, dependencies = listOf(data)) {
-                    env.result(data.await())
-                }.named(fragmentName)
-            }
-        }
-
-        return ActiveDataTree<R>(outputType) {
-            populate(dataSet.flow().flatMapConcat(transform = ::splitOne))
-            scope?.launch {
-                dataSet.updates.collect { name ->
-                    //clear old nodes
-                    remove(name)
-                    //collect new items
-                    populate(dataSet.flowChildren(name).flatMapConcat(transform = ::splitOne))
-                }
-            }
-        }
-    }
-}
-
-/**
- * Action that splits each incoming element into a number of fragments defined in builder
- */
-@DFExperimental
-@Suppress("FunctionName")
-public inline fun <T : Any, reified R : Any> Action.Companion.split(
-    noinline builder: SplitBuilder<T, R>.() -> Unit,
-): Action<T, R> = SplitAction(typeOf<R>(), builder)

dataforge-data/src/commonMain/kotlin/hep/dataforge/data/ActiveDataTree.kt

@@ -1,118 +0,0 @@
-package hep.dataforge.data
-
-import hep.dataforge.meta.*
-import hep.dataforge.names.*
-import kotlinx.coroutines.CoroutineScope
-import kotlinx.coroutines.Job
-import kotlinx.coroutines.flow.*
-import kotlinx.coroutines.launch
-import kotlinx.coroutines.sync.Mutex
-import kotlinx.coroutines.sync.withLock
-import kotlin.reflect.KType
-import kotlin.reflect.typeOf
-
-/**
- * A mutable [DataTree.Companion.active]. It
- */
-public class ActiveDataTree<T : Any>(
-    override val dataType: KType,
-) : DataTree<T>, DataSetBuilder<T>, ActiveDataSet<T> {
-    private val mutex = Mutex()
-    private val treeItems = HashMap<NameToken, DataTreeItem<T>>()
-
-    override suspend fun items(): Map<NameToken, DataTreeItem<T>> = mutex.withLock {
-        treeItems.filter { !it.key.body.startsWith("@") }
-    }
-
-    private val _updates = MutableSharedFlow<Name>()
-
-    override val updates: Flow<Name>
-        get() = _updates
-
-    private suspend fun remove(token: NameToken) {
-        mutex.withLock {
-            if (treeItems.remove(token) != null) {
-                _updates.emit(token.asName())
-            }
-        }
-    }
-
-    override suspend fun remove(name: Name) {
-        if (name.isEmpty()) error("Can't remove the root node")
-        (getItem(name.cutLast()).tree as? ActiveDataTree)?.remove(name.lastOrNull()!!)
-    }
-
-    private suspend fun set(token: NameToken, data: Data<T>) {
-        mutex.withLock {
-            treeItems[token] = DataTreeItem.Leaf(data)
-        }
-    }
-
-    private suspend fun getOrCreateNode(token: NameToken): ActiveDataTree<T> =
-        (treeItems[token] as? DataTreeItem.Node<T>)?.tree as? ActiveDataTree<T>
-            ?: ActiveDataTree<T>(dataType).also {
-                mutex.withLock {
-                    treeItems[token] = DataTreeItem.Node(it)
-                }
-            }
-
-    private suspend fun getOrCreateNode(name: Name): ActiveDataTree<T> {
-        return when (name.length) {
-            0 -> this
-            1 -> getOrCreateNode(name.firstOrNull()!!)
-            else -> getOrCreateNode(name.firstOrNull()!!).getOrCreateNode(name.cutFirst())
-        }
-    }
-
-    override suspend fun emit(name: Name, data: Data<T>?) {
-        if (data == null) {
-            remove(name)
-        } else {
-            when (name.length) {
-                0 -> error("Can't add data with empty name")
-                1 -> set(name.firstOrNull()!!, data)
-                2 -> getOrCreateNode(name.cutLast()).set(name.lastOrNull()!!, data)
-            }
-        }
-        _updates.emit(name)
-    }
-
-    /**
-     * Copy given data set and mirror its changes to this [ActiveDataTree] in [this@setAndObserve]. Returns an update [Job]
-     */
-    public fun CoroutineScope.setAndObserve(name: Name, dataSet: DataSet<T>): Job = launch {
-        emit(name, dataSet)
-        dataSet.updates.collect { nameInBranch ->
-            emit(name + nameInBranch, dataSet.getData(nameInBranch))
-        }
-    }
-}
-
-/**
- * Create a dynamic tree. Initial data is placed synchronously. Updates are propagated via [updatesScope]
- */
-@Suppress("FunctionName")
-public suspend fun <T : Any> ActiveDataTree(
-    type: KType,
-    block: suspend ActiveDataTree<T>.() -> Unit,
-): ActiveDataTree<T> {
-    val tree = ActiveDataTree<T>(type)
-    tree.block()
-    return tree
-}
-
-@Suppress("FunctionName")
-public suspend inline fun <reified T : Any> ActiveDataTree(
-    crossinline block: suspend ActiveDataTree<T>.() -> Unit,
-): ActiveDataTree<T> = ActiveDataTree<T>(typeOf<T>()).apply { block() }
-
-
-public suspend inline fun <reified T : Any> ActiveDataTree<T>.emit(
-    name: Name,
-    noinline block: suspend ActiveDataTree<T>.() -> Unit,
-): Unit = emit(name, ActiveDataTree(typeOf<T>(), block))
-
-public suspend inline fun <reified T : Any> ActiveDataTree<T>.emit(
-    name: String,
-    noinline block: suspend ActiveDataTree<T>.() -> Unit,
-): Unit = emit(name.toName(), ActiveDataTree(typeOf<T>(), block))

dataforge-data/src/commonMain/kotlin/hep/dataforge/data/CachingAction.kt

@@ -1,52 +0,0 @@
-package hep.dataforge.data
-
-import hep.dataforge.actions.Action
-import hep.dataforge.meta.Meta
-import hep.dataforge.names.Name
-import hep.dataforge.names.startsWith
-import kotlinx.coroutines.CoroutineScope
-import kotlinx.coroutines.coroutineScope
-import kotlinx.coroutines.flow.Flow
-import kotlinx.coroutines.flow.collect
-import kotlin.reflect.KType
-
-/**
- * Remove all values with keys starting with [name]
- */
-internal fun MutableMap<Name, *>.removeWhatStartsWith(name: Name) {
-    val toRemove = keys.filter { it.startsWith(name) }
-    toRemove.forEach(::remove)
-}
-
-/**
- * An action that caches results on-demand and recalculates them on source push
- */
-public abstract class CachingAction<in T : Any, out R : Any>(
-    public val outputType: KType,
-) : Action<T, R> {
-
-    protected abstract fun CoroutineScope.transform(
-        set: DataSet<T>,
-        meta: Meta,
-        key: Name = Name.EMPTY,
-    ): Flow<NamedData<R>>
-
-    override suspend fun execute(
-        dataSet: DataSet<T>,
-        meta: Meta,
-        scope: CoroutineScope?,
-    ): DataSet<R> = ActiveDataTree<R>(outputType) {
-        coroutineScope {
-            populate(transform(dataSet, meta))
-        }
-        scope?.let {
-            dataSet.updates.collect {
-                //clear old nodes
-                remove(it)
-                //collect new items
-                populate(scope.transform(dataSet, meta, it))
-                //FIXME if the target is data, updates are fired twice
-            }
-        }
-    }
-}

dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataSet.kt

@@ -1,99 +0,0 @@
-package hep.dataforge.data
-
-import hep.dataforge.data.Data.Companion.TYPE_OF_NOTHING
-import hep.dataforge.meta.Meta
-import hep.dataforge.meta.set
-import hep.dataforge.names.*
-import kotlinx.coroutines.*
-import kotlinx.coroutines.flow.*
-import kotlin.reflect.KType
-
-public interface DataSet<out T : Any> {
-
-    /**
-     * The minimal common ancestor to all data in the node
-     */
-    public val dataType: KType
-
-    /**
-     * Traverse this provider or its child. The order is not guaranteed.
-     * [root] points to a root name for traversal. If it is empty, traverse this source, if it points to a [Data],
-     * return flow, that contains single [Data], if it points to a node with children, return children.
-     */
-    public fun flow(): Flow<NamedData<T>>
-
-    /**
-     * Get data with given name.
-     */
-    public suspend fun getData(name: Name): Data<T>?
-
-    /**
-     * Get a snapshot of names of children of given node. Empty if node does not exist or is a leaf.
-     *
-     * By default traverses the whole tree. Could be optimized in descendants
-     */
-    public suspend fun listChildren(prefix: Name = Name.EMPTY): List<Name> =
-        flow().map { it.name }.filter { it.startsWith(prefix) && (it.length == prefix.length + 1) }.toList()
-
-    public companion object {
-        public val META_KEY: Name = "@meta".asName()
-
-        /**
-         * An empty [DataSet] that suits all types
-         */
-        public val EMPTY: DataSet<Nothing> = object : DataSet<Nothing> {
-            override val dataType: KType = TYPE_OF_NOTHING
-
-            private val nothing: Nothing get() = error("this is nothing")
-
-            override fun flow(): Flow<NamedData<Nothing>> = emptyFlow()
-
-            override suspend fun getData(name: Name): Data<Nothing>? = null
-        }
-    }
-}
-
-public interface ActiveDataSet<T : Any> : DataSet<T> {
-    /**
-     * A flow of updated item names. Updates are propagated in a form of [Flow] of names of updated nodes.
-     * Those can include new data items and replacement of existing ones. The replaced items could update existing data content
-     * and replace it completely, so they should be pulled again.
-     *
-     */
-    public val updates: Flow<Name>
-}
-
-public val <T : Any> DataSet<T>.updates: Flow<Name> get() = if (this is ActiveDataSet) updates else emptyFlow()
-
-/**
- * Flow all data nodes with names starting with [branchName]
- */
-public fun <T : Any> DataSet<T>.flowChildren(branchName: Name): Flow<NamedData<T>> = this@flowChildren.flow().filter {
-    it.name.startsWith(branchName)
-}
-
-/**
- * Start computation for all goals in data node and return a job for the whole node
- */
-public fun <T : Any> DataSet<T>.startAll(coroutineScope: CoroutineScope): Job = coroutineScope.launch {
-    flow().map {
-        it.launch(this@launch)
-    }.toList().joinAll()
-}
-
-public suspend fun <T : Any> DataSet<T>.join(): Unit = coroutineScope { startAll(this).join() }
-
-public suspend fun DataSet<*>.toMeta(): Meta = Meta {
-    flow().collect {
-        if (it.name.endsWith(DataSet.META_KEY)) {
-            set(it.name, it.meta)
-        } else {
-            it.name put {
-                "type" put it.type.toString()
-                "meta" put it.meta
-            }
-        }
-    }
-}
-
-public val <T : Any> DataSet<T>.updatesWithData: Flow<NamedData<T>> get() = updates.mapNotNull { getData(it)?.named(it) }

dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataSetBuilder.kt

@@ -1,145 +0,0 @@
-package hep.dataforge.data
-
-import hep.dataforge.meta.Meta
-import hep.dataforge.meta.MetaBuilder
-import hep.dataforge.misc.DFExperimental
-import hep.dataforge.names.Name
-import hep.dataforge.names.plus
-import hep.dataforge.names.toName
-import kotlinx.coroutines.coroutineScope
-import kotlinx.coroutines.flow.Flow
-import kotlinx.coroutines.flow.collect
-import kotlin.reflect.KType
-
-public interface DataSetBuilder<in T : Any> {
-    public val dataType: KType
-
-    /**
-     * Remove all data items starting with [name]
-     */
-    public suspend fun remove(name: Name)
-
-    public suspend fun emit(name: Name, data: Data<T>?)
-
-    /**
-     * Set a current state of given [dataSet] into a branch [name]. Does not propagate updates
-     */
-    public suspend fun emit(name: Name, dataSet: DataSet<T>) {
-        //remove previous items
-        if (name != Name.EMPTY) {
-            remove(name)
-        }
-
-        //Set new items
-        dataSet.flow().collect {
-            emit(name + it.name, it.data)
-        }
-    }
-
-    /**
-     * Append data to node
-     */
-    public suspend infix fun String.put(data: Data<T>): Unit = emit(toName(), data)
-
-    /**
-     * Append node
-     */
-    public suspend infix fun String.put(dataSet: DataSet<T>): Unit = emit(toName(), dataSet)
-
-    /**
-     * Build and append node
-     */
-    public suspend infix fun String.put(block: suspend DataSetBuilder<T>.() -> Unit): Unit = emit(toName(), block)
-}
-
-private class SubSetBuilder<in T : Any>(
-    private val parent: DataSetBuilder<T>,
-    private val branch: Name,
-) : DataSetBuilder<T> {
-    override val dataType: KType get() = parent.dataType
-
-    override suspend fun remove(name: Name) {
-        parent.remove(branch + name)
-    }
-
-    override suspend fun emit(name: Name, data: Data<T>?) {
-        parent.emit(branch + name, data)
-    }
-
-    override suspend fun emit(name: Name, dataSet: DataSet<T>) {
-        parent.emit(branch + name, dataSet)
-    }
-}
-
-public suspend fun <T : Any> DataSetBuilder<T>.emit(name: Name, block: suspend DataSetBuilder<T>.() -> Unit) {
-    SubSetBuilder(this, name).apply { block() }
-}
-
-
-public suspend fun <T : Any> DataSetBuilder<T>.emit(name: String, data: Data<T>) {
-    emit(name.toName(), data)
-}
-
-public suspend fun <T : Any> DataSetBuilder<T>.emit(name: String, set: DataSet<T>) {
-    this.emit(name.toName(), set)
-}
-
-public suspend fun <T : Any> DataSetBuilder<T>.emit(name: String, block: suspend DataSetBuilder<T>.() -> Unit): Unit =
-    this@emit.emit(name.toName(), block)
-
-public suspend fun <T : Any> DataSetBuilder<T>.emit(data: NamedData<T>) {
-    emit(data.name, data.data)
-}
-
-/**
- * Produce lazy [Data] and emit it into the [DataSetBuilder]
- */
-public suspend inline fun <reified T : Any> DataSetBuilder<T>.produce(
-    name: String,
-    meta: Meta = Meta.EMPTY,
-    noinline producer: suspend () -> T,
-) {
-    val data = Data(meta, block = producer)
-    emit(name, data)
-}
-
-public suspend inline fun <reified T : Any> DataSetBuilder<T>.produce(
-    name: Name,
-    meta: Meta = Meta.EMPTY,
-    noinline producer: suspend () -> T,
-) {
-    val data = Data(meta, block = producer)
-    emit(name, data)
-}
-
-/**
- * Emit a static data with the fixed value
- */
-public suspend inline fun <reified T : Any> DataSetBuilder<T>.static(name: String, data: T, meta: Meta = Meta.EMPTY): Unit =
-    emit(name, Data.static(data, meta))
-
-public suspend inline fun <reified T : Any> DataSetBuilder<T>.static(name: Name, data: T, meta: Meta = Meta.EMPTY): Unit =
-    emit(name, Data.static(data, meta))
-
-public suspend inline fun <reified T : Any> DataSetBuilder<T>.static(
-    name: String,
-    data: T,
-    metaBuilder: MetaBuilder.() -> Unit,
-): Unit = emit(name.toName(), Data.static(data, Meta(metaBuilder)))
-
-/**
- * Update data with given node data and meta with node meta.
- */
-@DFExperimental
-public suspend fun <T : Any> DataSetBuilder<T>.populate(tree: DataSet<T>): Unit = coroutineScope {
-    tree.flow().collect {
-        //TODO check if the place is occupied
-        emit(it.name, it.data)
-    }
-}
-
-public suspend fun <T : Any> DataSetBuilder<T>.populate(flow: Flow<NamedData<T>>) {
-    flow.collect {
-        emit(it.name, it.data)
-    }
-}

dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataTree.kt

@@ -1,96 +0,0 @@
-package hep.dataforge.data
-
-import hep.dataforge.misc.Type
-import hep.dataforge.names.*
-import kotlinx.coroutines.flow.Flow
-import kotlinx.coroutines.flow.emitAll
-import kotlinx.coroutines.flow.flow
-import kotlinx.coroutines.flow.map
-import kotlin.collections.component1
-import kotlin.collections.component2
-import kotlin.reflect.KType
-
-public sealed class DataTreeItem<out T : Any> {
-    public class Node<out T : Any>(public val tree: DataTree<T>) : DataTreeItem<T>()
-    public class Leaf<out T : Any>(public val data: Data<T>) : DataTreeItem<T>()
-}
-
-public val <T : Any> DataTreeItem<T>.type: KType
-    get() = when (this) {
-        is DataTreeItem.Node -> tree.dataType
-        is DataTreeItem.Leaf -> data.type
-    }
-
-/**
- * A tree-like [DataSet] grouped into the node. All data inside the node must inherit its type
- */
-@Type(DataTree.TYPE)
-public interface DataTree<out T : Any> : DataSet<T> {
-
-    /**
-     * Children items of this [DataTree] provided asynchronously
-     */
-    public suspend fun items(): Map<NameToken, DataTreeItem<T>>
-
-    override fun flow(): Flow<NamedData<T>> = flow {
-        items().forEach { (token, childItem: DataTreeItem<T>) ->
-            if(!token.body.startsWith("@")) {
-                when (childItem) {
-                    is DataTreeItem.Leaf -> emit(childItem.data.named(token.asName()))
-                    is DataTreeItem.Node -> emitAll(childItem.tree.flow().map { it.named(token + it.name) })
-                }
-            }
-        }
-    }
-
-    override suspend fun listChildren(prefix: Name): List<Name> =
-        getItem(prefix).tree?.items()?.keys?.map { prefix + it } ?: emptyList()
-
-    override suspend fun getData(name: Name): Data<T>? = when (name.length) {
-        0 -> null
-        1 -> items()[name.firstOrNull()!!].data
-        else -> items()[name.firstOrNull()!!].tree?.getData(name.cutFirst())
-    }
-
-    public companion object {
-        public const val TYPE: String = "dataTree"
-    }
-}
-
-public suspend fun <T: Any> DataSet<T>.getData(name: String): Data<T>? = getData(name.toName())
-
-/**
- * Get a [DataTreeItem] with given [name] or null if the item does not exist
- */
-public tailrec suspend fun <T : Any> DataTree<T>.getItem(name: Name): DataTreeItem<T>? = when (name.length) {
-    0 -> DataTreeItem.Node(this)
-    1 -> items()[name.firstOrNull()]
-    else -> items()[name.firstOrNull()!!].tree?.getItem(name.cutFirst())
-}
-
-public val <T : Any> DataTreeItem<T>?.tree: DataTree<T>? get() = (this as? DataTreeItem.Node<T>)?.tree
-public val <T : Any> DataTreeItem<T>?.data: Data<T>? get() = (this as? DataTreeItem.Leaf<T>)?.data
-
-/**
- * Flow of all children including nodes
- */
-public fun <T : Any> DataTree<T>.itemFlow(): Flow<Pair<Name, DataTreeItem<T>>> = flow {
-    items().forEach { (head, item) ->
-        emit(head.asName() to item)
-        if (item is DataTreeItem.Node) {
-            val subSequence = item.tree.itemFlow()
-                .map { (name, data) -> (head.asName() + name) to data }
-            emitAll(subSequence)
-        }
-    }
-}
-
-/**
- * Get a branch of this [DataTree] with a given [branchName].
- * The difference from similar method for [DataSet] is that internal logic is more simple and the return value is a [DataTree]
- */
-public fun <T : Any> DataTree<T>.branch(branchName: Name): DataTree<T> = object : DataTree<T> {
-    override val dataType: KType get() = this@branch.dataType
-
-    override suspend fun items(): Map<NameToken, DataTreeItem<T>> = getItem(branchName).tree?.items() ?: emptyMap()
-}

dataforge-data/src/commonMain/kotlin/hep/dataforge/data/GroupRule.kt

@@ -1,64 +0,0 @@
-/* 
- * Copyright 2015 Alexander Nozik.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package hep.dataforge.data
-
-import hep.dataforge.meta.get
-import hep.dataforge.meta.string
-import kotlinx.coroutines.CoroutineScope
-import kotlinx.coroutines.flow.collect
-import kotlinx.coroutines.launch
-
-public interface GroupRule {
-    public suspend fun <T : Any> gather(set: DataSet<T>): Map<String, DataSet<T>>
-
-    public companion object {
-        /**
-         * Create grouping rule that creates groups for different values of value
-         * field with name [key]
-         *
-         * @param key
-         * @param defaultTagValue
-         * @return
-         */
-        public fun byMetaValue(
-            scope: CoroutineScope,
-            key: String,
-            defaultTagValue: String,
-        ): GroupRule = object : GroupRule {
-
-            override suspend fun <T : Any> gather(
-                set: DataSet<T>,
-            ): Map<String, DataSet<T>> {
-                val map = HashMap<String, ActiveDataTree<T>>()
-
-                set.flow().collect { data ->
-                    val tagValue = data.meta[key]?.string ?: defaultTagValue
-                    map.getOrPut(tagValue) { ActiveDataTree(set.dataType) }.emit(data.name, data.data)
-                }
-
-                scope.launch {
-                    set.updates.collect { name ->
-                        val data = set.getData(name)
-                        val tagValue = data?.meta[key]?.string ?: defaultTagValue
-                        map.getOrPut(tagValue) { ActiveDataTree(set.dataType) }.emit(name, data)
-                    }
-                }
-
-                return map
-            }
-        }
-    }
-}

dataforge-data/src/commonMain/kotlin/hep/dataforge/data/StaticDataTree.kt

@@ -1,76 +0,0 @@
-package hep.dataforge.data
-
-import hep.dataforge.names.*
-import kotlinx.coroutines.coroutineScope
-import kotlinx.coroutines.flow.collect
-import kotlin.reflect.KType
-import kotlin.reflect.typeOf
-
-@PublishedApi
-internal class StaticDataTree<T : Any>(
-    override val dataType: KType,
-) : DataSetBuilder<T>, DataTree<T> {
-
-    private val items: MutableMap<NameToken, DataTreeItem<T>> = HashMap()
-
-    override suspend fun items(): Map<NameToken, DataTreeItem<T>> = items.filter { !it.key.body.startsWith("@") }
-
-    override suspend fun remove(name: Name) {
-        when (name.length) {
-            0 -> error("Can't remove root tree node")
-            1 -> items.remove(name.firstOrNull()!!)
-            else -> (items[name.firstOrNull()!!].tree as? StaticDataTree<T>)?.remove(name.cutFirst())
-        }
-    }
-
-    private fun getOrCreateNode(name: Name): StaticDataTree<T> = when (name.length) {
-        0 -> this
-        1 -> {
-            val itemName = name.firstOrNull()!!
-            (items[itemName].tree as? StaticDataTree<T>) ?: StaticDataTree<T>(dataType).also {
-                items[itemName] = DataTreeItem.Node(it)
-            }
-        }
-        else -> getOrCreateNode(name.cutLast()).getOrCreateNode(name.lastOrNull()!!.asName())
-    }
-
-    private suspend fun set(name: Name, item: DataTreeItem<T>?) {
-        if (name.isEmpty()) error("Can't set top level tree node")
-        if (item == null) {
-            remove(name)
-        } else {
-            getOrCreateNode(name.cutLast()).items[name.lastOrNull()!!] = item
-        }
-    }
-
-    override suspend fun emit(name: Name, data: Data<T>?) {
-        set(name, data?.let { DataTreeItem.Leaf(it) })
-    }
-
-    override suspend fun emit(name: Name, dataSet: DataSet<T>) {
-        if (dataSet is StaticDataTree) {
-            set(name, DataTreeItem.Node(dataSet))
-        } else {
-            coroutineScope {
-                dataSet.flow().collect {
-                    emit(name + it.name, it.data)
-                }
-            }
-        }
-    }
-}
-
-@Suppress("FunctionName")
-public suspend fun <T : Any> DataTree(
-    dataType: KType,
-    block: suspend DataSetBuilder<T>.() -> Unit,
-): DataTree<T> = StaticDataTree<T>(dataType).apply { block() }
-
-@Suppress("FunctionName")
-public suspend inline fun <reified T : Any> DataTree(
-    noinline block: suspend DataSetBuilder<T>.() -> Unit,
-): DataTree<T> = DataTree(typeOf<T>(), block)
-
-public suspend fun <T : Any> DataSet<T>.seal(): DataTree<T> = DataTree(dataType){
-    populate(this@seal)
-}

dataforge-data/src/commonMain/kotlin/hep/dataforge/data/dataFilter.kt

@@ -1,71 +0,0 @@
-package hep.dataforge.data
-
-import hep.dataforge.misc.DFExperimental
-import hep.dataforge.names.*
-import kotlinx.coroutines.flow.Flow
-import kotlinx.coroutines.flow.filter
-import kotlinx.coroutines.flow.map
-import kotlinx.coroutines.flow.mapNotNull
-import kotlin.reflect.KType
-
-
-/**
- * A stateless filtered [DataSet]
- */
-public fun <T : Any> DataSet<T>.filter(
-    predicate: suspend (Name, Data<T>) -> Boolean,
-): ActiveDataSet<T> = object : ActiveDataSet<T> {
-    override val dataType: KType get() = this@filter.dataType
-
-    override fun flow(): Flow<NamedData<T>> =
-        this@filter.flow().filter { predicate(it.name, it.data) }
-
-    override suspend fun getData(name: Name): Data<T>? = this@filter.getData(name)?.takeIf {
-        predicate(name, it)
-    }
-
-    override val updates: Flow<Name> = this@filter.updates.filter flowFilter@{ name ->
-        val theData = this@filter.getData(name) ?: return@flowFilter false
-        predicate(name, theData)
-    }
-}
-
-/**
- * Generate a wrapper data set with a given name prefix appended to all names
- */
-public fun <T : Any> DataSet<T>.withNamePrefix(prefix: Name): DataSet<T> = if (prefix.isEmpty()) this
-else object : ActiveDataSet<T> {
-    override val dataType: KType get() = this@withNamePrefix.dataType
-
-    override fun flow(): Flow<NamedData<T>> = this@withNamePrefix.flow().map { it.data.named(prefix + it.name) }
-
-    override suspend fun getData(name: Name): Data<T>? =
-        name.removeHeadOrNull(name)?.let { this@withNamePrefix.getData(it) }
-
-    override val updates: Flow<Name> get() = this@withNamePrefix.updates.map { prefix + it }
-}
-
-/**
- * Get a subset of data starting with a given [branchName]
- */
-public fun <T : Any> DataSet<T>.branch(branchName: Name): DataSet<T> = if (branchName.isEmpty()) {
-    this
-} else object : ActiveDataSet<T> {
-    override val dataType: KType get() = this@branch.dataType
-
-    override fun flow(): Flow<NamedData<T>> = this@branch.flow().mapNotNull {
-        it.name.removeHeadOrNull(branchName)?.let { name ->
-            it.data.named(name)
-        }
-    }
-
-    override suspend fun getData(name: Name): Data<T>? = this@branch.getData(branchName + name)
-
-    override val updates: Flow<Name> get() = this@branch.updates.mapNotNull { it.removeHeadOrNull(branchName) }
-}
-
-public fun <T : Any> DataSet<T>.branch(branchName: String): DataSet<T> = this@branch.branch(branchName.toName())
-
-@DFExperimental
-public suspend fun <T : Any> DataSet<T>.rootData(): Data<T>? = getData(Name.EMPTY)
-

dataforge-data/src/commonMain/kotlin/hep/dataforge/data/dataSetMeta.kt

@@ -1,20 +0,0 @@
-package hep.dataforge.data
-
-import hep.dataforge.meta.Meta
-import hep.dataforge.meta.MetaBuilder
-
-
-/**
- * Get a metadata node for this set if it is present
- */
-public suspend fun DataSet<*>.getMeta(): Meta? = getData(DataSet.META_KEY)?.meta
-
-/**
- * Add meta-data node to a [DataSet]
- */
-public suspend fun DataSetBuilder<*>.meta(meta: Meta): Unit = emit(DataSet.META_KEY, Data.empty(meta))
-
-/**
- * Add meta-data node to a [DataSet]
- */
-public suspend fun DataSetBuilder<*>.meta(metaBuilder: MetaBuilder.() -> Unit): Unit = meta(Meta(metaBuilder))

dataforge-data/src/commonMain/kotlin/hep/dataforge/data/dataTransform.kt

@@ -1,181 +0,0 @@
-package hep.dataforge.data
-
-import hep.dataforge.meta.Meta
-import hep.dataforge.meta.MetaBuilder
-import hep.dataforge.meta.seal
-import hep.dataforge.meta.toMutableMeta
-import hep.dataforge.misc.DFInternal
-import kotlinx.coroutines.flow.*
-import kotlin.contracts.InvocationKind
-import kotlin.contracts.contract
-import kotlin.coroutines.CoroutineContext
-import kotlin.coroutines.EmptyCoroutineContext
-import kotlin.reflect.KType
-import kotlin.reflect.typeOf
-
-/**
- * Lazily transform this data to another data. By convention [block] should not use external data (be pure).
- * @param coroutineContext additional [CoroutineContext] elements used for data computation.
- * @param meta for the resulting data. By default equals input data.
- * @param block the transformation itself
- */
-public inline fun <T : Any, reified R : Any> Data<T>.map(
-    coroutineContext: CoroutineContext = EmptyCoroutineContext,
-    meta: Meta = this.meta,
-    crossinline block: suspend (T) -> R,
-): Data<R> = Data(meta, coroutineContext, listOf(this)) {
-    block(await())
-}
-
-/**
- * Combine this data with the other data using [block]. See [map] for other details
- */
-public inline fun <T1 : Any, T2 : Any, reified R : Any> Data<T1>.combine(
-    other: Data<T2>,
-    coroutineContext: CoroutineContext = EmptyCoroutineContext,
-    meta: Meta = this.meta,
-    crossinline block: suspend (left: T1, right: T2) -> R,
-): Data<R> = Data(meta, coroutineContext, listOf(this, other)) {
-    block(await(), other.await())
-}
-
-
-//data collection operations
-
-/**
- * Lazily reduce a collection of [Data] to a single data.
- */
-public inline fun <T : Any, reified R : Any> Collection<Data<T>>.reduceToData(
-    coroutineContext: CoroutineContext = EmptyCoroutineContext,
-    meta: Meta = Meta.EMPTY,
-    crossinline block: suspend (Collection<T>) -> R,
-): Data<R> = Data(
-    meta,
-    coroutineContext,
-    this
-) {
-    block(map { it.await() })
-}
-
-@DFInternal
-public fun <K, T : Any, R : Any> Map<K, Data<T>>.reduceToData(
-    outputType: KType,
-    coroutineContext: CoroutineContext = EmptyCoroutineContext,
-    meta: Meta = Meta.EMPTY,
-    block: suspend (Map<K, T>) -> R,
-): Data<R> = Data(
-    outputType,
-    meta,
-    coroutineContext,
-    this.values
-) {
-    block(mapValues { it.value.await() })
-}
-
-
-/**
- * Lazily reduce a [Map] of [Data] with any static key.
- * @param K type of the map key
- * @param T type of the input goal
- * @param R type of the result goal
- */
-public inline fun <K, T : Any, reified R : Any> Map<K, Data<T>>.reduceToData(
-    coroutineContext: CoroutineContext = EmptyCoroutineContext,
-    meta: Meta = Meta.EMPTY,
-    noinline block: suspend (Map<K, T>) -> R,
-): Data<R> = Data(
-    meta,
-    coroutineContext,
-    this.values
-) {
-    block(mapValues { it.value.await() })
-}
-
-//flow operations
-
-/**
- * Transform a [Flow] of [NamedData] to a single [Data].
- */
-@DFInternal
-public suspend fun <T : Any, R : Any> Flow<NamedData<T>>.reduceToData(
-    outputType: KType,
-    coroutineContext: CoroutineContext = EmptyCoroutineContext,
-    meta: Meta = Meta.EMPTY,
-    transformation: suspend (Flow<NamedData<T>>) -> R,
-): Data<R> = Data(
-    outputType,
-    meta,
-    coroutineContext,
-    toList()
-) {
-    transformation(this)
-}
-
-@OptIn(DFInternal::class)
-public suspend inline fun <T : Any, reified R : Any> Flow<NamedData<T>>.reduceToData(
-    coroutineContext: CoroutineContext = EmptyCoroutineContext,
-    meta: Meta = Meta.EMPTY,
-    noinline transformation: suspend (Flow<NamedData<T>>) -> R,
-): Data<R> = reduceToData(typeOf<R>(), coroutineContext, meta) {
-    transformation(it)
-}
-
-/**
- * Fold a flow of named data into a single [Data]
- */
-public suspend inline fun <T : Any, reified R : Any> Flow<NamedData<T>>.foldToData(
-    initial: R,
-    coroutineContext: CoroutineContext = EmptyCoroutineContext,
-    meta: Meta = Meta.EMPTY,
-    noinline block: suspend (result: R, data: NamedData<T>) -> R,
-): Data<R> = reduceToData(
-    coroutineContext, meta
-) {
-    it.fold(initial, block)
-}
-
-//DataSet operations
-
-@DFInternal
-public suspend fun <T : Any, R : Any> DataSet<T>.map(
-    outputType: KType,
-    coroutineContext: CoroutineContext = EmptyCoroutineContext,
-    metaTransform: MetaBuilder.() -> Unit = {},
-    block: suspend (T) -> R,
-): DataTree<R> = DataTree<R>(outputType) {
-    populate(
-        flow().map {
-            val newMeta = it.meta.toMutableMeta().apply(metaTransform).seal()
-            Data(outputType, newMeta, coroutineContext, listOf(it)) {
-                block(it.await())
-            }.named(it.name)
-        }
-    )
-}
-
-@OptIn(DFInternal::class)
-public suspend inline fun <T : Any, reified R : Any> DataSet<T>.map(
-    coroutineContext: CoroutineContext = EmptyCoroutineContext,
-    noinline metaTransform: MetaBuilder.() -> Unit = {},
-    noinline block: suspend (T) -> R,
-): DataTree<R> = map(typeOf<R>(), coroutineContext, metaTransform, block)
-
-public suspend fun <T : Any> DataSet<T>.forEach(block: suspend (NamedData<T>) -> Unit) {
-    contract { callsInPlace(block, InvocationKind.EXACTLY_ONCE) }
-    flow().collect {
-        block(it)
-    }
-}
-
-public suspend inline fun <T : Any, reified R : Any> DataSet<T>.reduceToData(
-    coroutineContext: CoroutineContext = EmptyCoroutineContext,
-    meta: Meta = Meta.EMPTY,
-    noinline transformation: suspend (Flow<NamedData<T>>) -> R,
-): Data<R> = flow().reduceToData(coroutineContext, meta, transformation)
-
-public suspend inline fun <T : Any, reified R : Any> DataSet<T>.foldToData(
-    initial: R,
-    coroutineContext: CoroutineContext = EmptyCoroutineContext,
-    meta: Meta = Meta.EMPTY,
-    noinline block: suspend (result: R, data: NamedData<T>) -> R,
-): Data<R> = flow().foldToData(initial, coroutineContext, meta, block)

dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/AbstractAction.kt

@@ -0,0 +1,65 @@
+package space.kscience.dataforge.actions
+
+import kotlinx.coroutines.launch
+import space.kscience.dataforge.data.*
+import space.kscience.dataforge.meta.Meta
+import space.kscience.dataforge.misc.DFInternal
+import space.kscience.dataforge.names.Name
+import space.kscience.dataforge.names.startsWith
+import kotlin.reflect.KType
+
+/**
+ * Remove all values with keys starting with [name]
+ */
+internal fun MutableMap<Name, *>.removeWhatStartsWith(name: Name) {
+    val toRemove = keys.filter { it.startsWith(name) }
+    toRemove.forEach(::remove)
+}
+
+/**
+ * An action that caches results on-demand and recalculates them on source push
+ */
+public abstract class AbstractAction<in T : Any, R : Any>(
+    public val outputType: KType,
+) : Action<T, R> {
+
+    /**
+     * Generate initial content of the output
+     */
+    protected abstract fun DataSetBuilder<R>.generate(
+        data: DataSet<T>,
+        meta: Meta,
+    )
+
+    /**
+     * Update part of the data set when given [updateKey] is triggered by the source
+     */
+    protected open fun DataSourceBuilder<R>.update(
+        dataSet: DataSet<T>,
+        meta: Meta,
+        updateKey: Name,
+    ) {
+        // By default, recalculate the whole dataset
+        generate(dataSet, meta)
+    }
+
+    @OptIn(DFInternal::class)
+    override fun execute(
+        dataSet: DataSet<T>,
+        meta: Meta,
+    ): DataSet<R> = if (dataSet is DataSource) {
+        DataSource(outputType, dataSet){
+            generate(dataSet, meta)
+
+            launch {
+                dataSet.updates.collect { name ->
+                    update(dataSet, meta, name)
+                }
+            }
+        }
+    } else {
+        DataTree<R>(outputType) {
+            generate(dataSet, meta)
+        }
+    }
+}

dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/Action.kt

@@ -0,0 +1,40 @@
+package space.kscience.dataforge.actions
+
+import space.kscience.dataforge.data.DataSet
+import space.kscience.dataforge.meta.Meta
+import space.kscience.dataforge.misc.DFExperimental
+
+/**
+ * A simple data transformation on a data node. Actions should avoid doing actual dependency evaluation in [execute].
+ */
+public interface Action<in T : Any, out R : Any> {
+
+    /**
+     * Transform the data in the node, producing a new node. By default, it is assumed that all calculations are lazy
+     * so not actual computation is started at this moment.
+     */
+    public fun execute(dataSet: DataSet<T>, meta: Meta = Meta.EMPTY): DataSet<R>
+
+    public companion object
+}
+
+/**
+ * Action composition. The result is terminal if one of its parts is terminal
+ */
+public infix fun <T : Any, I : Any, R : Any> Action<T, I>.then(action: Action<I, R>): Action<T, R> {
+    // TODO introduce composite action and add optimize by adding action to the list
+    return object : Action<T, R> {
+
+        override fun execute(
+            dataSet: DataSet<T>,
+            meta: Meta,
+        ): DataSet<R> = action.execute(this@then.execute(dataSet, meta), meta)
+    }
+}
+
+@DFExperimental
+public operator fun <T : Any, R : Any> Action<T, R>.invoke(
+    dataSet: DataSet<T>,
+    meta: Meta = Meta.EMPTY,
+): DataSet<R> = execute(dataSet, meta)
+

dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/MapAction.kt

@@ -0,0 +1,106 @@
+package space.kscience.dataforge.actions
+
+import space.kscience.dataforge.data.*
+import space.kscience.dataforge.meta.Meta
+import space.kscience.dataforge.meta.MutableMeta
+import space.kscience.dataforge.meta.seal
+import space.kscience.dataforge.meta.toMutableMeta
+import space.kscience.dataforge.misc.DFBuilder
+import space.kscience.dataforge.misc.DFExperimental
+import space.kscience.dataforge.misc.DFInternal
+import space.kscience.dataforge.names.Name
+import kotlin.reflect.KType
+import kotlin.reflect.typeOf
+
+/**
+ * Action environment includes data name, data meta and action configuration meta
+ */
+public data class ActionEnv(
+    val name: Name,
+    val meta: Meta,
+    val actionMeta: Meta,
+)
+
+/**
+ * Action environment
+ */
+@DFBuilder
+public class MapActionBuilder<T, R>(
+    public var name: Name,
+    public var meta: MutableMeta,
+    public val actionMeta: Meta,
+    @PublishedApi internal var outputType: KType,
+) {
+
+    public lateinit var result: suspend ActionEnv.(T) -> R
+
+    /**
+     * Set unsafe [outputType] for the resulting data. Be sure that it is correct.
+     */
+    public fun <R1 : R> result(outputType: KType, f: suspend ActionEnv.(T) -> R1) {
+        this.outputType = outputType
+        result = f;
+    }
+
+    /**
+     * Calculate the result of goal
+     */
+    public inline fun <reified R1 : R> result(noinline f: suspend ActionEnv.(T) -> R1) {
+        outputType = typeOf<R1>()
+        result = f;
+    }
+}
+
+@PublishedApi
+internal class MapAction<in T : Any, R : Any>(
+    outputType: KType,
+    private val block: MapActionBuilder<T, R>.() -> Unit,
+) : AbstractAction<T, R>(outputType) {
+
+    private fun DataSetBuilder<R>.mapOne(name: Name, data: Data<T>, meta: Meta) {
+        // Creating a new environment for action using **old** name, old meta and task meta
+        val env = ActionEnv(name, data.meta, meta)
+
+        //applying transformation from builder
+        val builder = MapActionBuilder<T, R>(
+            name,
+            data.meta.toMutableMeta(), // using data meta
+            meta,
+            outputType
+        ).apply(block)
+
+        //getting new name
+        val newName = builder.name
+
+        //getting new meta
+        val newMeta = builder.meta.seal()
+
+        @OptIn(DFInternal::class)
+        val newData = Data(builder.outputType, newMeta, dependencies = listOf(data)) {
+            builder.result(env, data.await())
+        }
+        //setting the data node
+        data(newName, newData)
+    }
+
+    override fun DataSetBuilder<R>.generate(data: DataSet<T>, meta: Meta) {
+        data.forEach { mapOne(it.name, it.data, meta) }
+    }
+
+    override fun DataSourceBuilder<R>.update(dataSet: DataSet<T>, meta: Meta, updateKey: Name) {
+        remove(updateKey)
+        dataSet[updateKey]?.let { mapOne(updateKey, it, meta) }
+    }
+}
+
+
+/**
+ * A one-to-one mapping action
+ */
+@DFExperimental
+@Suppress("FunctionName")
+public inline fun <T : Any, reified R : Any> Action.Companion.map(
+    noinline builder: MapActionBuilder<T, R>.() -> Unit,
+): Action<T, R> = MapAction(typeOf<R>(), builder)
+
+

dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/ReduceAction.kt

@@ -0,0 +1,117 @@
+package space.kscience.dataforge.actions
+
+import space.kscience.dataforge.data.*
+import space.kscience.dataforge.meta.Meta
+import space.kscience.dataforge.meta.MutableMeta
+import space.kscience.dataforge.misc.DFBuilder
+import space.kscience.dataforge.misc.DFExperimental
+import space.kscience.dataforge.misc.DFInternal
+import space.kscience.dataforge.names.Name
+import space.kscience.dataforge.names.parseAsName
+import kotlin.reflect.KType
+import kotlin.reflect.typeOf
+
+
+public class JoinGroup<T : Any, R : Any>(
+    public var name: String,
+    internal val set: DataSet<T>,
+    @PublishedApi internal var outputType: KType,
+) {
+
+    public var meta: MutableMeta = MutableMeta()
+
+    public lateinit var result: suspend ActionEnv.(Map<Name, ValueWithMeta<T>>) -> R
+
+    internal fun <R1 : R> result(outputType: KType, f: suspend ActionEnv.(Map<Name,  ValueWithMeta<T>>) -> R1) {
+        this.outputType = outputType
+        this.result = f;
+    }
+
+    public inline fun <reified R1 : R> result(noinline f: suspend ActionEnv.(Map<Name,  ValueWithMeta<T>>) -> R1) {
+        outputType = typeOf<R1>()
+        this.result = f;
+    }
+
+}
+
+@DFBuilder
+public class ReduceGroupBuilder<T : Any, R : Any>(
+    public val actionMeta: Meta,
+    private val outputType: KType,
+) {
+    private val groupRules: MutableList<(DataSet<T>) -> List<JoinGroup<T, R>>> = ArrayList();
+
+    /**
+     * introduce grouping by meta value
+     */
+    public fun byValue(tag: String, defaultTag: String = "@default", action: JoinGroup<T, R>.() -> Unit) {
+        groupRules += { node ->
+            GroupRule.byMetaValue(tag, defaultTag).gather(node).map {
+                JoinGroup<T, R>(it.key, it.value, outputType).apply(action)
+            }
+        }
+    }
+
+    public fun group(
+        groupName: String,
+        predicate: (Name, Meta) -> Boolean,
+        action: JoinGroup<T, R>.() -> Unit,
+    ) {
+        groupRules += { source ->
+            listOf(
+                JoinGroup<T, R>(groupName, source.filter(predicate), outputType).apply(action)
+            )
+        }
+    }
+
+    /**
+     * Apply transformation to the whole node
+     */
+    public fun result(resultName: String, f: suspend ActionEnv.(Map<Name,  ValueWithMeta<T>>) -> R) {
+        groupRules += { node ->
+            listOf(JoinGroup<T, R>(resultName, node, outputType).apply { result(outputType, f) })
+        }
+    }
+
+    internal fun buildGroups(input: DataSet<T>): List<JoinGroup<T, R>> =
+        groupRules.flatMap { it.invoke(input) }
+
+}
+
+@PublishedApi
+internal class ReduceAction<T : Any, R : Any>(
+    outputType: KType,
+    private val action: ReduceGroupBuilder<T, R>.() -> Unit,
+) : AbstractAction<T, R>(outputType) {
+    //TODO optimize reduction. Currently, the whole action recalculates on push
+
+    override fun DataSetBuilder<R>.generate(data: DataSet<T>, meta: Meta) {
+        ReduceGroupBuilder<T, R>(meta, outputType).apply(action).buildGroups(data).forEach { group ->
+            val dataFlow: Map<Name, Data<T>> = group.set.asSequence().fold(HashMap()) { acc, value ->
+                acc.apply {
+                    acc[value.name] = value.data
+                }
+            }
+
+            val groupName: String = group.name
+
+            val groupMeta = group.meta
+
+            val env = ActionEnv(groupName.parseAsName(), groupMeta, meta)
+            @OptIn(DFInternal::class) val res: Data<R> = dataFlow.reduceToData(
+                group.outputType,
+                meta = groupMeta
+            ) { group.result.invoke(env, it) }
+
+            data(env.name, res)
+        }
+    }
+}
+
+/**
+ * A one-to-one mapping action
+ */
+@DFExperimental
+public inline fun <reified T : Any, reified R : Any> Action.Companion.reduce(
+    noinline builder: ReduceGroupBuilder<T, R>.() -> Unit,
+): Action<T, R> = ReduceAction(typeOf<R>(), builder)

dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/SplitAction.kt

@@ -0,0 +1,92 @@
+package space.kscience.dataforge.actions
+
+import space.kscience.dataforge.data.*
+import space.kscience.dataforge.meta.Laminate
+import space.kscience.dataforge.meta.Meta
+import space.kscience.dataforge.meta.MutableMeta
+import space.kscience.dataforge.meta.toMutableMeta
+import space.kscience.dataforge.misc.DFExperimental
+import space.kscience.dataforge.names.Name
+import space.kscience.dataforge.names.parseAsName
+import kotlin.collections.set
+import kotlin.reflect.KType
+import kotlin.reflect.typeOf
+
+
+public class SplitBuilder<T : Any, R : Any>(public val name: Name, public val meta: Meta) {
+
+    public class FragmentRule<T : Any, R : Any>(
+        public val name: Name,
+        public var meta: MutableMeta,
+        @PublishedApi internal var outputType: KType,
+    ) {
+        public lateinit var result: suspend (T) -> R
+
+        public inline fun <reified R1 : R> result(noinline f: suspend (T) -> R1) {
+            this.outputType = typeOf<R1>()
+            result = f;
+        }
+    }
+
+    internal val fragments: MutableMap<Name, FragmentRule<T, R>.() -> Unit> = HashMap()
+
+    /**
+     * Add new fragment building rule. If the framgent not defined, result won't be available even if it is present in the map
+     * @param name the name of a fragment
+     * @param rule the rule to transform fragment name and meta using
+     */
+    public fun fragment(name: String, rule: FragmentRule<T, R>.() -> Unit) {
+        fragments[name.parseAsName()] = rule
+    }
+}
+
+/**
+ * Action that splits each incoming element into a number of fragments defined in builder
+ */
+@PublishedApi
+internal class SplitAction<T : Any, R : Any>(
+    outputType: KType,
+    private val action: SplitBuilder<T, R>.() -> Unit,
+) : AbstractAction<T, R>(outputType) {
+
+    private fun DataSetBuilder<R>.splitOne(name: Name, data: Data<T>, meta: Meta) {
+        val laminate = Laminate(data.meta, meta)
+
+        val split = SplitBuilder<T, R>(name, data.meta).apply(action)
+
+
+        // apply individual fragment rules to result
+        split.fragments.forEach { (fragmentName, rule) ->
+            val env = SplitBuilder.FragmentRule<T, R>(
+                fragmentName,
+                laminate.toMutableMeta(),
+                outputType
+            ).apply(rule)
+            //data.map<R>(outputType, meta = env.meta) { env.result(it) }.named(fragmentName)
+
+            data(
+                fragmentName,
+                @Suppress("OPT_IN_USAGE") Data(outputType, meta = env.meta, dependencies = listOf(data)) {
+                    env.result(data.await())
+                }
+            )
+        }
+    }
+
+    override fun DataSetBuilder<R>.generate(data: DataSet<T>, meta: Meta) {
+        data.forEach { splitOne(it.name, it.data, meta) }
+    }
+
+    override fun DataSourceBuilder<R>.update(dataSet: DataSet<T>, meta: Meta, updateKey: Name) {
+        remove(updateKey)
+        dataSet[updateKey]?.let { splitOne(updateKey, it, meta) }
+    }
+}
+
+/**
+ * Action that splits each incoming element into a number of fragments defined in builder
+ */
+@DFExperimental
+public inline fun <T : Any, reified R : Any> Action.Companion.split(
+    noinline builder: SplitBuilder<T, R>.() -> Unit,
+): Action<T, R> = SplitAction(typeOf<R>(), builder)

dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/CoroutineMonitor.kt

@@ -1,12 +1,11 @@
-package hep.dataforge.data
+package space.kscience.dataforge.data
 
-import hep.dataforge.misc.DFExperimental
 import kotlinx.coroutines.CoroutineScope
 import kotlinx.coroutines.Job
+import space.kscience.dataforge.misc.DFExperimental
 import kotlin.coroutines.CoroutineContext
 
 
-
 /**
  * A monitor of goal state that could be accessed only form inside the goal
  */
@@ -49,9 +48,9 @@ public val CoroutineScope.monitor: CoroutineMonitor? get() = coroutineContext.mo
 public val Job.dependencies: Collection<Job> get() = this[Dependencies]?.values ?: emptyList()
 
 @DFExperimental
-public val Job.totalWork: Double get() = dependencies.sumByDouble { totalWork } + (monitor?.totalWork ?: 0.0)
+public val Job.totalWork: Double get() = dependencies.sumOf { totalWork } + (monitor?.totalWork ?: 0.0)
 @DFExperimental
-public val Job.workDone: Double get() = dependencies.sumByDouble { workDone } + (monitor?.workDone ?: 0.0)
+public val Job.workDone: Double get() = dependencies.sumOf { workDone } + (monitor?.workDone ?: 0.0)
 @DFExperimental
 public val Job.status: String get() = monitor?.status ?: ""
 @DFExperimental

dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/Data.kt

@@ -1,11 +1,11 @@
-package hep.dataforge.data
+package space.kscience.dataforge.data
 
-import hep.dataforge.meta.Meta
-import hep.dataforge.meta.MetaRepr
-import hep.dataforge.meta.isEmpty
-import hep.dataforge.misc.DFInternal
-import hep.dataforge.misc.Type
 import kotlinx.coroutines.*
+import space.kscience.dataforge.meta.Meta
+import space.kscience.dataforge.meta.MetaRepr
+import space.kscience.dataforge.meta.isEmpty
+import space.kscience.dataforge.misc.DFInternal
+import space.kscience.dataforge.misc.DfId
 import kotlin.coroutines.CoroutineContext
 import kotlin.coroutines.EmptyCoroutineContext
 import kotlin.reflect.KType
@@ -14,8 +14,8 @@ import kotlin.reflect.typeOf
 /**
  * A data element characterized by its meta
  */
-@Type(Data.TYPE)
-public interface Data<out T : Any> : Goal<T>, MetaRepr {
+@DfId(Data.TYPE)
+public interface Data<out T> : Goal<T>, MetaRepr {
     /**
      * Type marker for the data. The type is known before the calculation takes place so it could be checked.
      */
@@ -49,6 +49,7 @@ public interface Data<out T : Any> : Goal<T>, MetaRepr {
         /**
          * An empty data containing only meta
          */
+        @OptIn(DelicateCoroutinesApi::class)
         public fun empty(meta: Meta): Data<Nothing> = object : Data<Nothing> {
             override val type: KType = TYPE_OF_NOTHING
             override val meta: Meta = meta
@@ -72,7 +73,7 @@ private class LazyData<T : Any>(
     override val type: KType,
     override val meta: Meta = Meta.EMPTY,
     additionalContext: CoroutineContext = EmptyCoroutineContext,
-    dependencies: Collection<Data<*>> = emptyList(),
+    dependencies: Collection<Goal<*>> = emptyList(),
     block: suspend () -> T,
 ) : Data<T>, LazyGoal<T>(additionalContext, dependencies, block)
 
@@ -82,13 +83,17 @@ public class StaticData<T : Any>(
     override val meta: Meta = Meta.EMPTY,
 ) : Data<T>, StaticGoal<T>(value)
 
+@Suppress("FunctionName")
+public inline fun <reified T : Any> Data(value: T, meta: Meta = Meta.EMPTY): StaticData<T> =
+    StaticData(typeOf<T>(), value, meta)
+
 @Suppress("FunctionName")
 @DFInternal
 public fun <T : Any> Data(
     type: KType,
     meta: Meta = Meta.EMPTY,
     context: CoroutineContext = EmptyCoroutineContext,
-    dependencies: Collection<Data<*>> = emptyList(),
+    dependencies: Collection<Goal<*>> = emptyList(),
     block: suspend () -> T,
 ): Data<T> = LazyData(type, meta, context, dependencies, block)
 
@@ -97,6 +102,6 @@ public fun <T : Any> Data(
 public inline fun <reified T : Any> Data(
     meta: Meta = Meta.EMPTY,
     context: CoroutineContext = EmptyCoroutineContext,
-    dependencies: Collection<Data<*>> = emptyList(),
+    dependencies: Collection<Goal<*>> = emptyList(),
     noinline block: suspend () -> T,
 ): Data<T> = Data(typeOf<T>(), meta, context, dependencies, block)

dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSet.kt

@@ -0,0 +1,124 @@
+package space.kscience.dataforge.data
+
+import kotlinx.coroutines.*
+import kotlinx.coroutines.flow.Flow
+import kotlinx.coroutines.flow.emptyFlow
+import kotlinx.coroutines.flow.mapNotNull
+import space.kscience.dataforge.data.Data.Companion.TYPE_OF_NOTHING
+import space.kscience.dataforge.meta.Meta
+import space.kscience.dataforge.names.Name
+import space.kscience.dataforge.names.asName
+import space.kscience.dataforge.names.endsWith
+import space.kscience.dataforge.names.parseAsName
+import kotlin.reflect.KType
+
+public interface DataSet<out T : Any> {
+
+    /**
+     * The minimal common ancestor to all data in the node
+     */
+    public val dataType: KType
+
+    /**
+     * Meta-data associated with this node. If no meta is provided, returns [Meta.EMPTY].
+     */
+    public val meta: Meta
+
+    /**
+     * Traverse this [DataSet] returning named data instances. The order is not guaranteed.
+     */
+    public operator fun iterator(): Iterator<NamedData<T>>
+
+    /**
+     * Get data with given name.
+     */
+    public operator fun get(name: Name): Data<T>?
+
+    public companion object {
+        public val META_KEY: Name = "@meta".asName()
+
+        /**
+         * An empty [DataSet] that suits all types
+         */
+        public val EMPTY: DataSet<Nothing> = object : DataSet<Nothing> {
+            override val dataType: KType = TYPE_OF_NOTHING
+            override val meta: Meta get() = Meta.EMPTY
+
+            override fun iterator(): Iterator<NamedData<Nothing>> = emptySequence<NamedData<Nothing>>().iterator()
+
+            override fun get(name: Name): Data<Nothing>? = null
+        }
+    }
+}
+
+public fun <T : Any> DataSet<T>.asSequence(): Sequence<NamedData<T>> = object : Sequence<NamedData<T>> {
+    override fun iterator(): Iterator<NamedData<T>> = this@asSequence.iterator()
+}
+
+/**
+ * Return a single [Data] in this [DataSet]. Throw error if it is not single.
+ */
+public fun <T : Any> DataSet<T>.single(): NamedData<T> = asSequence().single()
+
+public fun <T : Any> DataSet<T>.asIterable(): Iterable<NamedData<T>> = object : Iterable<NamedData<T>> {
+    override fun iterator(): Iterator<NamedData<T>> = this@asIterable.iterator()
+}
+
+public operator fun <T : Any> DataSet<T>.get(name: String): Data<T>? = get(name.parseAsName())
+
+/**
+ * A [DataSet] with propagated updates.
+ */
+public interface DataSource<out T : Any> : DataSet<T>, CoroutineScope {
+
+    /**
+     * A flow of updated item names. Updates are propagated in a form of [Flow] of names of updated nodes.
+     * Those can include new data items and replacement of existing ones. The replaced items could update existing data content
+     * and replace it completely, so they should be pulled again.
+     *
+     */
+    public val updates: Flow<Name>
+
+    /**
+     * Stop generating updates from this [DataSource]
+     */
+    public fun close() {
+        coroutineContext[Job]?.cancel()
+    }
+}
+
+public val <T : Any> DataSet<T>.updates: Flow<Name> get() = if (this is DataSource) updates else emptyFlow()
+//
+///**
+// * Flow all data nodes with names starting with [branchName]
+// */
+//public fun <T : Any> DataSet<T>.children(branchName: Name): Sequence<NamedData<T>> =
+//    this@children.asSequence().filter {
+//        it.name.startsWith(branchName)
+//    }
+
+/**
+ * Start computation for all goals in data node and return a job for the whole node
+ */
+public fun <T : Any> DataSet<T>.startAll(coroutineScope: CoroutineScope): Job = coroutineScope.launch {
+    asIterable().map {
+        it.launch(this@launch)
+    }.joinAll()
+}
+
+public suspend fun <T : Any> DataSet<T>.computeAndJoinAll(): Unit = coroutineScope { startAll(this).join() }
+
+public fun DataSet<*>.toMeta(): Meta = Meta {
+    forEach {
+        if (it.name.endsWith(DataSet.META_KEY)) {
+            set(it.name, it.meta)
+        } else {
+            it.name put {
+                "type" put it.type.toString()
+                "meta" put it.meta
+            }
+        }
+    }
+}
+
+public val <T : Any> DataSet<T>.updatesWithData: Flow<NamedData<T>> get() = updates.mapNotNull { get(it)?.named(it) }

dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSetBuilder.kt

@@ -0,0 +1,165 @@
+package space.kscience.dataforge.data
+
+import space.kscience.dataforge.meta.Meta
+import space.kscience.dataforge.meta.MutableMeta
+import space.kscience.dataforge.misc.DFExperimental
+import space.kscience.dataforge.names.Name
+import space.kscience.dataforge.names.isEmpty
+import space.kscience.dataforge.names.plus
+import kotlin.reflect.KType
+
+public interface DataSetBuilder<in T : Any> {
+    public val dataType: KType
+
+    /**
+     * Remove all data items starting with [name]
+     */
+    public fun remove(name: Name)
+
+    public fun data(name: Name, data: Data<T>?)
+
+    /**
+     * Set a current state of given [dataSet] into a branch [name]. Does not propagate updates
+     */
+    public fun node(name: Name, dataSet: DataSet<T>) {
+        //remove previous items
+        if (name != Name.EMPTY) {
+            remove(name)
+        }
+
+        //Set new items
+        dataSet.forEach {
+            data(name + it.name, it.data)
+        }
+    }
+
+    /**
+     * Set meta for the given node
+     */
+    public fun meta(name: Name, meta: Meta)
+
+}
+
+/**
+ * Define meta in this [DataSet]
+ */
+public fun <T : Any> DataSetBuilder<T>.meta(value: Meta): Unit = meta(Name.EMPTY, value)
+
+/**
+ * Define meta in this [DataSet]
+ */
+public fun <T : Any> DataSetBuilder<T>.meta(mutableMeta: MutableMeta.() -> Unit): Unit = meta(Meta(mutableMeta))
+
+@PublishedApi
+internal class SubSetBuilder<in T : Any>(
+    private val parent: DataSetBuilder<T>,
+    private val branch: Name,
+) : DataSetBuilder<T> {
+    override val dataType: KType get() = parent.dataType
+
+    override fun remove(name: Name) {
+        parent.remove(branch + name)
+    }
+
+    override fun data(name: Name, data: Data<T>?) {
+        parent.data(branch + name, data)
+    }
+
+    override fun node(name: Name, dataSet: DataSet<T>) {
+        parent.node(branch + name, dataSet)
+    }
+
+    override fun meta(name: Name, meta: Meta) {
+        parent.meta(branch + name, meta)
+    }
+}
+
+public inline fun <T : Any> DataSetBuilder<T>.node(
+    name: Name,
+    crossinline block: DataSetBuilder<T>.() -> Unit,
+) {
+    if (name.isEmpty()) block() else SubSetBuilder(this, name).block()
+}
+
+
+public fun <T : Any> DataSetBuilder<T>.data(name: String, value: Data<T>) {
+    data(Name.parse(name), value)
+}
+
+public fun <T : Any> DataSetBuilder<T>.node(name: String, set: DataSet<T>) {
+    node(Name.parse(name), set)
+}
+
+public inline fun <T : Any> DataSetBuilder<T>.node(
+    name: String,
+    crossinline block: DataSetBuilder<T>.() -> Unit,
+): Unit = node(Name.parse(name), block)
+
+public fun <T : Any> DataSetBuilder<T>.set(value: NamedData<T>) {
+    data(value.name, value.data)
+}
+
+/**
+ * Produce lazy [Data] and emit it into the [DataSetBuilder]
+ */
+public inline fun <reified T : Any> DataSetBuilder<T>.produce(
+    name: String,
+    meta: Meta = Meta.EMPTY,
+    noinline producer: suspend () -> T,
+) {
+    val data = Data(meta, block = producer)
+    data(name, data)
+}
+
+public inline fun <reified T : Any> DataSetBuilder<T>.produce(
+    name: Name,
+    meta: Meta = Meta.EMPTY,
+    noinline producer: suspend () -> T,
+) {
+    val data = Data(meta, block = producer)
+    data(name, data)
+}
+
+/**
+ * Emit a static data with the fixed value
+ */
+public inline fun <reified T : Any> DataSetBuilder<T>.static(
+    name: String,
+    data: T,
+    meta: Meta = Meta.EMPTY,
+): Unit = data(name, Data.static(data, meta))
+
+public inline fun <reified T : Any> DataSetBuilder<T>.static(
+    name: Name,
+    data: T,
+    meta: Meta = Meta.EMPTY,
+): Unit = data(name, Data.static(data, meta))
+
+public inline fun <reified T : Any> DataSetBuilder<T>.static(
+    name: String,
+    data: T,
+    mutableMeta: MutableMeta.() -> Unit,
+): Unit = data(Name.parse(name), Data.static(data, Meta(mutableMeta)))
+
+/**
+ * Update data with given node data and meta with node meta.
+ */
+@DFExperimental
+public fun <T : Any> DataSetBuilder<T>.populateFrom(tree: DataSet<T>): Unit {
+    tree.forEach {
+        //TODO check if the place is occupied
+        data(it.name, it.data)
+    }
+}
+
+//public fun <T : Any> DataSetBuilder<T>.populateFrom(flow: Flow<NamedData<T>>) {
+//    flow.collect {
+//        data(it.name, it.data)
+//    }
+//}
+
+public fun <T : Any> DataSetBuilder<T>.populateFrom(sequence: Sequence<NamedData<T>>) {
+    sequence.forEach {
+        data(it.name, it.data)
+    }
+}

dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataTree.kt

@@ -0,0 +1,119 @@
+package space.kscience.dataforge.data
+
+import space.kscience.dataforge.meta.Meta
+import space.kscience.dataforge.misc.DFInternal
+import space.kscience.dataforge.misc.DfId
+import space.kscience.dataforge.names.*
+import kotlin.collections.component1
+import kotlin.collections.component2
+import kotlin.reflect.KType
+import kotlin.reflect.typeOf
+
+public sealed class DataTreeItem<out T : Any> {
+
+    public abstract val meta: Meta
+
+    public class Node<out T : Any>(public val tree: DataTree<T>) : DataTreeItem<T>() {
+        override val meta: Meta get() = tree.meta
+    }
+
+    public class Leaf<out T : Any>(public val data: Data<T>) : DataTreeItem<T>() {
+        override val meta: Meta get() = data.meta
+    }
+}
+
+public val <T : Any> DataTreeItem<T>.type: KType
+    get() = when (this) {
+        is DataTreeItem.Node -> tree.dataType
+        is DataTreeItem.Leaf -> data.type
+    }
+
+/**
+ * A tree-like [DataSet] grouped into the node. All data inside the node must inherit its type
+ */
+@DfId(DataTree.TYPE)
+public interface DataTree<out T : Any> : DataSet<T> {
+
+    /**
+     * Top-level children items of this [DataTree]
+     */
+    public val items: Map<NameToken, DataTreeItem<T>>
+
+    override val meta: Meta get() = items[META_ITEM_NAME_TOKEN]?.meta ?: Meta.EMPTY
+
+    override fun iterator(): Iterator<NamedData<T>> = iterator {
+        items.forEach { (token, childItem: DataTreeItem<T>) ->
+            if (!token.body.startsWith("@")) {
+                when (childItem) {
+                    is DataTreeItem.Leaf -> yield(childItem.data.named(token.asName()))
+                    is DataTreeItem.Node -> yieldAll(childItem.tree.asSequence().map { it.named(token + it.name) })
+                }
+            }
+        }
+    }
+
+    override fun get(name: Name): Data<T>? = when (name.length) {
+        0 -> null
+        1 -> items[name.firstOrNull()!!].data
+        else -> items[name.firstOrNull()!!].tree?.get(name.cutFirst())
+    }
+
+    public companion object {
+        public const val TYPE: String = "dataTree"
+
+        /**
+         * A name token used to designate tree node meta
+         */
+        public val META_ITEM_NAME_TOKEN: NameToken = NameToken("@meta")
+
+        @DFInternal
+        public fun <T : Any> emptyWithType(type: KType, meta: Meta = Meta.EMPTY): DataTree<T> = object : DataTree<T> {
+            override val items: Map<NameToken, DataTreeItem<T>> get() = emptyMap()
+            override val dataType: KType get() = type
+            override val meta: Meta get() = meta
+        }
+
+        @OptIn(DFInternal::class)
+        public inline fun <reified T : Any> empty(meta: Meta = Meta.EMPTY): DataTree<T> =
+            emptyWithType<T>(typeOf<T>(), meta)
+    }
+}
+
+public fun <T : Any> DataTree<T>.listChildren(prefix: Name): List<Name> =
+    getItem(prefix).tree?.items?.keys?.map { prefix + it } ?: emptyList()
+
+/**
+ * Get a [DataTreeItem] with given [name] or null if the item does not exist
+ */
+public tailrec fun <T : Any> DataTree<T>.getItem(name: Name): DataTreeItem<T>? = when (name.length) {
+    0 -> DataTreeItem.Node(this)
+    1 -> items[name.firstOrNull()]
+    else -> items[name.firstOrNull()!!].tree?.getItem(name.cutFirst())
+}
+
+public val <T : Any> DataTreeItem<T>?.tree: DataTree<T>? get() = (this as? DataTreeItem.Node<T>)?.tree
+public val <T : Any> DataTreeItem<T>?.data: Data<T>? get() = (this as? DataTreeItem.Leaf<T>)?.data
+
+/**
+ * A [Sequence] of all children including nodes
+ */
+public fun <T : Any> DataTree<T>.traverseItems(): Sequence<Pair<Name, DataTreeItem<T>>> = sequence {
+    items.forEach { (head, item) ->
+        yield(head.asName() to item)
+        if (item is DataTreeItem.Node) {
+            val subSequence = item.tree.traverseItems()
+                .map { (name, data) -> (head.asName() + name) to data }
+            yieldAll(subSequence)
+        }
+    }
+}
+
+/**
+ * Get a branch of this [DataTree] with a given [branchName].
+ * The difference from similar method for [DataSet] is that internal logic is more simple and the return value is a [DataTree]
+ */
+@OptIn(DFInternal::class)
+public fun <T : Any> DataTree<T>.branch(branchName: Name): DataTree<T> =
+    getItem(branchName)?.tree ?: DataTree.emptyWithType(dataType)
+
+public fun <T : Any> DataTree<T>.branch(branchName: String): DataTree<T> = branch(branchName.parseAsName())

dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataTreeBuilder.kt

@@ -0,0 +1,127 @@
+package space.kscience.dataforge.data
+
+import kotlinx.coroutines.CoroutineScope
+import kotlinx.coroutines.Job
+import kotlinx.coroutines.flow.MutableSharedFlow
+import kotlinx.coroutines.launch
+import space.kscience.dataforge.meta.Meta
+import space.kscience.dataforge.misc.DFInternal
+import space.kscience.dataforge.misc.ThreadSafe
+import space.kscience.dataforge.names.*
+import kotlin.collections.set
+import kotlin.coroutines.CoroutineContext
+import kotlin.coroutines.coroutineContext
+import kotlin.reflect.KType
+import kotlin.reflect.typeOf
+
+public interface DataSourceBuilder<T : Any> : DataSetBuilder<T>, DataSource<T> {
+    override val updates: MutableSharedFlow<Name>
+}
+
+/**
+ * A mutable [DataTree] that propagates updates
+ */
+public class DataTreeBuilder<T : Any>(
+    override val dataType: KType,
+    coroutineContext: CoroutineContext,
+) : DataTree<T>, DataSourceBuilder<T> {
+
+    override val coroutineContext: CoroutineContext =
+        coroutineContext + Job(coroutineContext[Job]) + GoalExecutionRestriction()
+
+    private val treeItems = HashMap<NameToken, DataTreeItem<T>>()
+
+    override val items: Map<NameToken, DataTreeItem<T>>
+        get() = treeItems.filter { !it.key.body.startsWith("@") }
+
+    override val updates: MutableSharedFlow<Name> = MutableSharedFlow<Name>()
+
+    @ThreadSafe
+    private fun remove(token: NameToken) {
+        if (treeItems.remove(token) != null) {
+            launch {
+                updates.emit(token.asName())
+            }
+        }
+    }
+
+    override fun remove(name: Name) {
+        if (name.isEmpty()) error("Can't remove the root node")
+        (getItem(name.cutLast()).tree as? DataTreeBuilder)?.remove(name.lastOrNull()!!)
+    }
+
+    @ThreadSafe
+    private fun set(token: NameToken, data: Data<T>) {
+        treeItems[token] = DataTreeItem.Leaf(data)
+    }
+
+    @ThreadSafe
+    private fun set(token: NameToken, node: DataTree<T>) {
+        treeItems[token] = DataTreeItem.Node(node)
+    }
+
+    private fun getOrCreateNode(token: NameToken): DataTreeBuilder<T> =
+        (treeItems[token] as? DataTreeItem.Node<T>)?.tree as? DataTreeBuilder<T>
+            ?: DataTreeBuilder<T>(dataType, coroutineContext).also { set(token, it) }
+
+    private fun getOrCreateNode(name: Name): DataTreeBuilder<T> = when (name.length) {
+        0 -> this
+        1 -> getOrCreateNode(name.firstOrNull()!!)
+        else -> getOrCreateNode(name.firstOrNull()!!).getOrCreateNode(name.cutFirst())
+    }
+
+    override fun data(name: Name, data: Data<T>?) {
+        if (data == null) {
+            remove(name)
+        } else {
+            when (name.length) {
+                0 -> error("Can't add data with empty name")
+                1 -> set(name.firstOrNull()!!, data)
+                2 -> getOrCreateNode(name.cutLast()).set(name.lastOrNull()!!, data)
+            }
+        }
+        launch {
+            updates.emit(name)
+        }
+    }
+
+    override fun meta(name: Name, meta: Meta) {
+        val item = getItem(name)
+        if (item is DataTreeItem.Leaf) error("TODO: Can't change meta of existing leaf item.")
+        data(name + DataTree.META_ITEM_NAME_TOKEN, Data.empty(meta))
+    }
+}
+
+/**
+ * Create a dynamic [DataSource]. Initial data is placed synchronously.
+ */
+@DFInternal
+@Suppress("FunctionName")
+public fun <T : Any> DataSource(
+    type: KType,
+    parent: CoroutineScope,
+    block: DataSourceBuilder<T>.() -> Unit,
+): DataTreeBuilder<T> = DataTreeBuilder<T>(type, parent.coroutineContext).apply(block)
+
+@Suppress("OPT_IN_USAGE", "FunctionName")
+public inline fun <reified T : Any> DataSource(
+    parent: CoroutineScope,
+    crossinline block: DataSourceBuilder<T>.() -> Unit,
+): DataTreeBuilder<T> = DataSource(typeOf<T>(), parent) { block() }
+
+@Suppress("FunctionName")
+public suspend inline fun <reified T : Any> DataSource(
+    crossinline block: DataSourceBuilder<T>.() -> Unit = {},
+): DataTreeBuilder<T> = DataTreeBuilder<T>(typeOf<T>(), coroutineContext).apply { block() }
+
+public inline fun <reified T : Any> DataSourceBuilder<T>.emit(
+    name: Name,
+    parent: CoroutineScope,
+    noinline block: DataSourceBuilder<T>.() -> Unit,
+): Unit = node(name, DataSource(parent, block))
+
+public inline fun <reified T : Any> DataSourceBuilder<T>.emit(
+    name: String,
+    parent: CoroutineScope,
+    noinline block: DataSourceBuilder<T>.() -> Unit,
+): Unit = node(Name.parse(name), DataSource(parent, block))

dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/Goal.kt

@@ -1,7 +1,7 @@
-package hep.dataforge.data
+package space.kscience.dataforge.data
 
-import hep.dataforge.misc.DFExperimental
 import kotlinx.coroutines.*
+import space.kscience.dataforge.misc.DFExperimental
 import kotlin.coroutines.CoroutineContext
 import kotlin.coroutines.EmptyCoroutineContext
 
@@ -67,7 +67,7 @@ public open class LazyGoal<T>(
      * If [GoalExecutionRestriction] is present in the [coroutineScope] context, the call could produce a error a warning
      * depending on the settings.
      */
-    @DFExperimental
+    @OptIn(DFExperimental::class)
     override fun async(coroutineScope: CoroutineScope): Deferred<T> {
         val log = coroutineScope.coroutineContext[GoalLogger]
         // Check if context restricts goal computation

dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/GoalExecutionRestriction.kt

@@ -1,13 +1,27 @@
-package hep.dataforge.data
+package space.kscience.dataforge.data
 
 import kotlin.coroutines.CoroutineContext
 
 public enum class GoalExecutionRestrictionPolicy {
+    /**
+     * Allow eager execution
+     */
     NONE,
+
+    /**
+     * Give warning on eager execution
+     */
     WARNING,
+
+    /**
+     * Throw error on eager execution
+     */
     ERROR
 }
 
+/**
+ * A special coroutine context key that allows or disallows goal execution during configuration time (eager execution).
+ */
 public class GoalExecutionRestriction(
     public val policy: GoalExecutionRestrictionPolicy = GoalExecutionRestrictionPolicy.ERROR,
 ) : CoroutineContext.Element {

dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/GoalLogger.kt

@@ -1,7 +1,10 @@
-package hep.dataforge.data
+package space.kscience.dataforge.data
 
 import kotlin.coroutines.CoroutineContext
 
+/**
+ * Coroutine context element that provides logging capabilities
+ */
 public interface GoalLogger : CoroutineContext.Element {
     override val key: CoroutineContext.Key<*> get() = GoalLogger
 

dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/GroupRule.kt

@@ -0,0 +1,78 @@
+/* 
+ * Copyright 2015 Alexander Nozik.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package space.kscience.dataforge.data
+
+import kotlinx.coroutines.launch
+import space.kscience.dataforge.meta.get
+import space.kscience.dataforge.meta.string
+import space.kscience.dataforge.misc.DFInternal
+
+public interface GroupRule {
+    public fun <T : Any> gather(set: DataSet<T>): Map<String, DataSet<T>>
+
+    public companion object {
+        /**
+         * Create grouping rule that creates groups for different values of value
+         * field with name [key]
+         *
+         * @param key
+         * @param defaultTagValue
+         * @return
+         */
+        @OptIn(DFInternal::class)
+        public fun byMetaValue(
+            key: String,
+            defaultTagValue: String,
+        ): GroupRule = object : GroupRule {
+
+            override fun <T : Any> gather(
+                set: DataSet<T>,
+            ): Map<String, DataSet<T>> {
+                val map = HashMap<String, DataSet<T>>()
+
+                if (set is DataSource) {
+                    set.forEach { data ->
+                        val tagValue: String = data.meta[key]?.string ?: defaultTagValue
+                        (map.getOrPut(tagValue) { DataTreeBuilder(set.dataType, set.coroutineContext) } as DataTreeBuilder<T>)
+                            .data(data.name, data.data)
+
+                        set.launch {
+                            set.updates.collect { name ->
+                                val dataUpdate = set[name]
+
+                                val updateTagValue = dataUpdate?.meta?.get(key)?.string ?: defaultTagValue
+                                map.getOrPut(updateTagValue) {
+                                    DataSource(set.dataType, this) {
+                                        data(name, dataUpdate)
+                                    }
+                                }
+                            }
+                        }
+                    }
+                } else {
+                    set.forEach { data ->
+                        val tagValue: String = data.meta[key]?.string ?: defaultTagValue
+                        (map.getOrPut(tagValue) { StaticDataTree(set.dataType) } as StaticDataTree<T>)
+                            .data(data.name, data.data)
+                    }
+                }
+
+
+                return map
+            }
+        }
+    }
+}

dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/NamedData.kt

@@ -1,14 +1,17 @@
-package hep.dataforge.data
+package space.kscience.dataforge.data
 
-import hep.dataforge.meta.isEmpty
-import hep.dataforge.misc.Named
-import hep.dataforge.names.Name
+import space.kscience.dataforge.meta.isEmpty
+import space.kscience.dataforge.misc.Named
+import space.kscience.dataforge.names.Name
 
 public interface NamedData<out T : Any> : Named, Data<T> {
     override val name: Name
     public val data: Data<T>
 }
 
+public operator fun NamedData<*>.component1(): Name = name
+public operator fun <T: Any> NamedData<T>.component2(): Data<T> = data
+
 private class NamedDataImpl<out T : Any>(
     override val name: Name,
     override val data: Data<T>,

dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/StaticDataTree.kt

@@ -0,0 +1,82 @@
+package space.kscience.dataforge.data
+
+import space.kscience.dataforge.meta.Meta
+import space.kscience.dataforge.misc.DFExperimental
+import space.kscience.dataforge.names.*
+import kotlin.reflect.KType
+import kotlin.reflect.typeOf
+
+@PublishedApi
+internal class StaticDataTree<T : Any>(
+    override val dataType: KType,
+) : DataSetBuilder<T>, DataTree<T> {
+
+    private val _items: MutableMap<NameToken, DataTreeItem<T>> = HashMap()
+
+    override val items: Map<NameToken, DataTreeItem<T>>
+        get() = _items.filter { !it.key.body.startsWith("@") }
+
+    override fun remove(name: Name) {
+        when (name.length) {
+            0 -> error("Can't remove root tree node")
+            1 -> _items.remove(name.firstOrNull()!!)
+            else -> (_items[name.firstOrNull()!!].tree as? StaticDataTree<T>)?.remove(name.cutFirst())
+        }
+    }
+
+    private fun getOrCreateNode(name: Name): StaticDataTree<T> = when (name.length) {
+        0 -> this
+        1 -> {
+            val itemName = name.firstOrNull()!!
+            (_items[itemName].tree as? StaticDataTree<T>) ?: StaticDataTree<T>(dataType).also {
+                _items[itemName] = DataTreeItem.Node(it)
+            }
+        }
+        else -> getOrCreateNode(name.cutLast()).getOrCreateNode(name.lastOrNull()!!.asName())
+    }
+
+    private fun set(name: Name, item: DataTreeItem<T>?) {
+        if (name.isEmpty()) error("Can't set top level tree node")
+        if (item == null) {
+            remove(name)
+        } else {
+            getOrCreateNode(name.cutLast())._items[name.lastOrNull()!!] = item
+        }
+    }
+
+    override fun data(name: Name, data: Data<T>?) {
+        set(name, data?.let { DataTreeItem.Leaf(it) })
+    }
+
+    override fun node(name: Name, dataSet: DataSet<T>) {
+        if (dataSet is StaticDataTree) {
+            set(name, DataTreeItem.Node(dataSet))
+        } else {
+            dataSet.forEach {
+                data(name + it.name, it.data)
+            }
+        }
+    }
+
+    override fun meta(name: Name, meta: Meta) {
+        val item = getItem(name)
+        if (item is DataTreeItem.Leaf) TODO("Can't change meta of existing leaf item.")
+        data(name + DataTree.META_ITEM_NAME_TOKEN, Data.empty(meta))
+    }
+}
+
+@Suppress("FunctionName")
+public inline fun <T : Any> DataTree(
+    dataType: KType,
+    block: DataSetBuilder<T>.() -> Unit,
+): DataTree<T> = StaticDataTree<T>(dataType).apply { block() }
+
+@Suppress("FunctionName")
+public inline fun <reified T : Any> DataTree(
+    noinline block: DataSetBuilder<T>.() -> Unit,
+): DataTree<T> = DataTree(typeOf<T>(), block)
+
+@OptIn(DFExperimental::class)
+public fun <T : Any> DataSet<T>.seal(): DataTree<T> = DataTree(dataType) {
+    populateFrom(this@seal)
+}

dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataFilter.kt

@@ -0,0 +1,105 @@
+package space.kscience.dataforge.data
+
+import kotlinx.coroutines.flow.Flow
+import kotlinx.coroutines.flow.filter
+import kotlinx.coroutines.flow.map
+import kotlinx.coroutines.flow.mapNotNull
+import space.kscience.dataforge.meta.Meta
+import space.kscience.dataforge.misc.DFExperimental
+import space.kscience.dataforge.names.*
+import kotlin.coroutines.CoroutineContext
+import kotlin.coroutines.EmptyCoroutineContext
+import kotlin.reflect.KType
+
+
+/**
+ * A stateless filtered [DataSet]
+ */
+public fun <T : Any> DataSet<T>.filter(
+    predicate: (Name, Meta) -> Boolean,
+): DataSource<T> = object : DataSource<T> {
+
+    override val dataType: KType get() = this@filter.dataType
+
+    override val coroutineContext: CoroutineContext
+        get() = (this@filter as? DataSource)?.coroutineContext ?: EmptyCoroutineContext
+
+
+    override val meta: Meta get() = this@filter.meta
+
+    override fun iterator(): Iterator<NamedData<T>> = iterator {
+        for (d in this@filter) {
+            if (predicate(d.name, d.meta)) {
+                yield(d)
+            }
+        }
+    }
+
+    override fun get(name: Name): Data<T>? = this@filter.get(name)?.takeIf {
+        predicate(name, it.meta)
+    }
+
+    override val updates: Flow<Name> = this@filter.updates.filter flowFilter@{ name ->
+        val theData = this@filter[name] ?: return@flowFilter false
+        predicate(name, theData.meta)
+    }
+}
+
+/**
+ * Generate a wrapper data set with a given name prefix appended to all names
+ */
+public fun <T : Any> DataSet<T>.withNamePrefix(prefix: Name): DataSet<T> = if (prefix.isEmpty()) {
+    this
+} else object : DataSource<T> {
+
+    override val dataType: KType get() = this@withNamePrefix.dataType
+
+    override val coroutineContext: CoroutineContext
+        get() = (this@withNamePrefix as? DataSource)?.coroutineContext ?: EmptyCoroutineContext
+
+    override val meta: Meta get() = this@withNamePrefix.meta
+
+
+    override fun iterator(): Iterator<NamedData<T>> = iterator {
+        for (d in this@withNamePrefix) {
+            yield(d.data.named(prefix + d.name))
+        }
+    }
+
+    override fun get(name: Name): Data<T>? =
+        name.removeFirstOrNull(name)?.let { this@withNamePrefix.get(it) }
+
+    override val updates: Flow<Name> get() = this@withNamePrefix.updates.map { prefix + it }
+}
+
+/**
+ * Get a subset of data starting with a given [branchName]
+ */
+public fun <T : Any> DataSet<T>.branch(branchName: Name): DataSet<T> = if (branchName.isEmpty()) {
+    this
+} else object : DataSource<T> {
+    override val dataType: KType get() = this@branch.dataType
+
+    override val coroutineContext: CoroutineContext
+        get() = (this@branch as? DataSource)?.coroutineContext ?: EmptyCoroutineContext
+
+    override val meta: Meta get() = this@branch.meta
+
+    override fun iterator(): Iterator<NamedData<T>> = iterator {
+        for (d in this@branch) {
+            d.name.removeFirstOrNull(branchName)?.let { name ->
+                yield(d.data.named(name))
+            }
+        }
+    }
+
+    override fun get(name: Name): Data<T>? = this@branch.get(branchName + name)
+
+    override val updates: Flow<Name> get() = this@branch.updates.mapNotNull { it.removeFirstOrNull(branchName) }
+}
+
+public fun <T : Any> DataSet<T>.branch(branchName: String): DataSet<T> = this@branch.branch(branchName.parseAsName())
+
+@DFExperimental
+public suspend fun <T : Any> DataSet<T>.rootData(): Data<T>? = get(Name.EMPTY)
+

dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataTransform.kt

@@ -0,0 +1,221 @@
+package space.kscience.dataforge.data
+
+import space.kscience.dataforge.meta.Meta
+import space.kscience.dataforge.meta.MutableMeta
+import space.kscience.dataforge.meta.seal
+import space.kscience.dataforge.meta.toMutableMeta
+import space.kscience.dataforge.misc.DFInternal
+import space.kscience.dataforge.names.Name
+import kotlin.coroutines.CoroutineContext
+import kotlin.coroutines.EmptyCoroutineContext
+import kotlin.reflect.KType
+import kotlin.reflect.typeOf
+
+public data class ValueWithMeta<T>(val meta: Meta, val value: T)
+
+public suspend fun <T : Any> Data<T>.awaitWithMeta(): ValueWithMeta<T> = ValueWithMeta(meta, await())
+
+public data class NamedValueWithMeta<T>(val name: Name, val meta: Meta, val value: T)
+
+public suspend fun <T : Any> NamedData<T>.awaitWithMeta(): NamedValueWithMeta<T> =
+    NamedValueWithMeta(name, meta, await())
+
+
+/**
+ * Lazily transform this data to another data. By convention [block] should not use external data (be pure).
+ * @param coroutineContext additional [CoroutineContext] elements used for data computation.
+ * @param meta for the resulting data. By default equals input data.
+ * @param block the transformation itself
+ */
+public inline fun <T : Any, reified R : Any> Data<T>.map(
+    coroutineContext: CoroutineContext = EmptyCoroutineContext,
+    meta: Meta = this.meta,
+    crossinline block: suspend (T) -> R,
+): Data<R> = Data(meta, coroutineContext, listOf(this)) {
+    block(await())
+}
+
+/**
+ * Combine this data with the other data using [block]. See [Data::map] for other details
+ */
+public inline fun <T1 : Any, T2 : Any, reified R : Any> Data<T1>.combine(
+    other: Data<T2>,
+    coroutineContext: CoroutineContext = EmptyCoroutineContext,
+    meta: Meta = this.meta,
+    crossinline block: suspend (left: T1, right: T2) -> R,
+): Data<R> = Data(meta, coroutineContext, listOf(this, other)) {
+    block(await(), other.await())
+}
+
+
+//data collection operations
+
+/**
+ * Lazily reduce a collection of [Data] to a single data.
+ */
+public inline fun <T : Any, reified R : Any> Collection<Data<T>>.reduceToData(
+    coroutineContext: CoroutineContext = EmptyCoroutineContext,
+    meta: Meta = Meta.EMPTY,
+    crossinline block: suspend (List<ValueWithMeta<T>>) -> R,
+): Data<R> = Data(
+    meta,
+    coroutineContext,
+    this
+) {
+    block(map { it.awaitWithMeta() })
+}
+
+@DFInternal
+public fun <K, T : Any, R : Any> Map<K, Data<T>>.reduceToData(
+    outputType: KType,
+    coroutineContext: CoroutineContext = EmptyCoroutineContext,
+    meta: Meta = Meta.EMPTY,
+    block: suspend (Map<K, ValueWithMeta<T>>) -> R,
+): Data<R> = Data(
+    outputType,
+    meta,
+    coroutineContext,
+    this.values
+) {
+    block(mapValues { it.value.awaitWithMeta() })
+}
+
+/**
+ * Lazily reduce a [Map] of [Data] with any static key.
+ * @param K type of the map key
+ * @param T type of the input goal
+ * @param R type of the result goal
+ */
+public inline fun <K, T : Any, reified R : Any> Map<K, Data<T>>.reduceToData(
+    coroutineContext: CoroutineContext = EmptyCoroutineContext,
+    meta: Meta = Meta.EMPTY,
+    crossinline block: suspend (Map<K, ValueWithMeta<T>>) -> R,
+): Data<R> = Data(
+    meta,
+    coroutineContext,
+    this.values
+) {
+    block(mapValues { it.value.awaitWithMeta() })
+}
+
+//Iterable operations
+
+@DFInternal
+public inline fun <T : Any, R : Any> Iterable<Data<T>>.reduceToData(
+    outputType: KType,
+    coroutineContext: CoroutineContext = EmptyCoroutineContext,
+    meta: Meta = Meta.EMPTY,
+    crossinline transformation: suspend (Collection<ValueWithMeta<T>>) -> R,
+): Data<R> = Data(
+    outputType,
+    meta,
+    coroutineContext,
+    toList()
+) {
+    transformation(map { it.awaitWithMeta() })
+}
+
+@OptIn(DFInternal::class)
+public inline fun <T : Any, reified R : Any> Iterable<Data<T>>.reduceToData(
+    coroutineContext: CoroutineContext = EmptyCoroutineContext,
+    meta: Meta = Meta.EMPTY,
+    crossinline transformation: suspend (Collection<ValueWithMeta<T>>) -> R,
+): Data<R> = reduceToData(typeOf<R>(), coroutineContext, meta) {
+    transformation(it)
+}
+
+public inline fun <T : Any, reified R : Any> Iterable<Data<T>>.foldToData(
+    initial: R,
+    coroutineContext: CoroutineContext = EmptyCoroutineContext,
+    meta: Meta = Meta.EMPTY,
+    crossinline block: suspend (result: R, data: ValueWithMeta<T>) -> R,
+): Data<R> = reduceToData(
+    coroutineContext, meta
+) {
+    it.fold(initial) { acc, t -> block(acc, t) }
+}
+
+/**
+ * Transform an [Iterable] of [NamedData] to a single [Data].
+ */
+@DFInternal
+public inline fun <T : Any, R : Any> Iterable<NamedData<T>>.reduceNamedToData(
+    outputType: KType,
+    coroutineContext: CoroutineContext = EmptyCoroutineContext,
+    meta: Meta = Meta.EMPTY,
+    crossinline transformation: suspend (Collection<NamedValueWithMeta<T>>) -> R,
+): Data<R> = Data(
+    outputType,
+    meta,
+    coroutineContext,
+    toList()
+) {
+    transformation(map { it.awaitWithMeta() })
+}
+
+@OptIn(DFInternal::class)
+public inline fun <T : Any, reified R : Any> Iterable<NamedData<T>>.reduceNamedToData(
+    coroutineContext: CoroutineContext = EmptyCoroutineContext,
+    meta: Meta = Meta.EMPTY,
+    crossinline transformation: suspend (Collection<NamedValueWithMeta<T>>) -> R,
+): Data<R> = reduceNamedToData(typeOf<R>(), coroutineContext, meta) {
+    transformation(it)
+}
+
+/**
+ * Fold a [Iterable] of named data into a single [Data]
+ */
+public inline fun <T : Any, reified R : Any> Iterable<NamedData<T>>.foldNamedToData(
+    initial: R,
+    coroutineContext: CoroutineContext = EmptyCoroutineContext,
+    meta: Meta = Meta.EMPTY,
+    crossinline block: suspend (result: R, data: NamedValueWithMeta<T>) -> R,
+): Data<R> = reduceNamedToData(
+    coroutineContext, meta
+) {
+    it.fold(initial) { acc, t -> block(acc, t) }
+}
+
+//DataSet operations
+
+@DFInternal
+public suspend fun <T : Any, R : Any> DataSet<T>.map(
+    outputType: KType,
+    coroutineContext: CoroutineContext = EmptyCoroutineContext,
+    metaTransform: MutableMeta.() -> Unit = {},
+    block: suspend (NamedValueWithMeta<T>) -> R,
+): DataTree<R> = DataTree<R>(outputType) {
+    forEach {
+        val newMeta = it.meta.toMutableMeta().apply(metaTransform).seal()
+        val d = Data(outputType, newMeta, coroutineContext, listOf(it)) {
+            block(it.awaitWithMeta())
+        }
+        data(it.name, d)
+    }
+}
+
+@OptIn(DFInternal::class)
+public suspend inline fun <T : Any, reified R : Any> DataSet<T>.map(
+    coroutineContext: CoroutineContext = EmptyCoroutineContext,
+    noinline metaTransform: MutableMeta.() -> Unit = {},
+    noinline block: suspend (NamedValueWithMeta<T>) -> R,
+): DataTree<R> = map(typeOf<R>(), coroutineContext, metaTransform, block)
+
+public inline fun <T : Any> DataSet<T>.forEach(block: (NamedData<T>) -> Unit) {
+    for (d in this) {
+        block(d)
+    }
+}
+
+public inline fun <T : Any, reified R : Any> DataSet<T>.reduceToData(
+    coroutineContext: CoroutineContext = EmptyCoroutineContext,
+    meta: Meta = Meta.EMPTY,
+    crossinline transformation: suspend (Iterable<NamedValueWithMeta<T>>) -> R,
+): Data<R> = asIterable().reduceNamedToData(coroutineContext, meta, transformation)
+
+public inline fun <T : Any, reified R : Any> DataSet<T>.foldToData(
+    initial: R,
+    coroutineContext: CoroutineContext = EmptyCoroutineContext,
+    meta: Meta = Meta.EMPTY,
+    crossinline block: suspend (result: R, data: NamedValueWithMeta<T>) -> R,
+): Data<R> = asIterable().foldNamedToData(initial, coroutineContext, meta, block)

dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/select.kt

@@ -1,64 +0,0 @@
-package hep.dataforge.data
-
-import hep.dataforge.misc.DFExperimental
-import hep.dataforge.names.Name
-import hep.dataforge.names.matches
-import hep.dataforge.names.toName
-import kotlinx.coroutines.flow.Flow
-import kotlinx.coroutines.flow.filter
-import kotlinx.coroutines.flow.map
-import kotlin.reflect.KType
-import kotlin.reflect.full.isSubtypeOf
-import kotlin.reflect.typeOf
-
-
-/**
- * Cast the node to given type if the cast is possible or return null
- */
-@Suppress("UNCHECKED_CAST")
-private fun <R : Any> Data<*>.castOrNull(type: KType): Data<R>? =
-    if (!this.type.isSubtypeOf(type)) null else object : Data<R> by (this as Data<R>) {
-        override val type: KType = type
-    }
-
-/**
- * Select all data matching given type and filters. Does not modify paths
- */
-@OptIn(DFExperimental::class)
-@PublishedApi
-internal fun <R : Any> DataSet<*>.select(
-    type: KType,
-    namePattern: Name? = null,
-): ActiveDataSet<R> = object : ActiveDataSet<R> {
-    override val dataType = type
-
-
-    override fun flow(): Flow<NamedData<R>> = this@select.flow().filter { datum ->
-        datum.type.isSubtypeOf(type) && (namePattern == null || datum.name.matches(namePattern))
-    }.map {
-        @Suppress("UNCHECKED_CAST")
-        it as NamedData<R>
-    }
-
-    override suspend fun getData(name: Name): Data<R>? = this@select.getData(name)?.castOrNull(type)
-
-    override val updates: Flow<Name> = this@select.updates.filter {
-        val datum = this@select.getData(it)
-        datum?.type?.isSubtypeOf(type) ?: false
-    }
-
-}
-
-/**
- * Select a single datum of the appropriate type
- */
-public inline fun <reified R : Any> DataSet<*>.select(namePattern: Name? = null): DataSet<R> =
-    select(typeOf<R>(), namePattern)
-
-public suspend fun <R : Any> DataSet<*>.selectOne(type: KType, name: Name): NamedData<R>? =
-    getData(name)?.castOrNull<R>(type)?.named(name)
-
-public suspend inline fun <reified R : Any> DataSet<*>.selectOne(name: Name): NamedData<R>? = selectOne(typeOf<R>(), name)
-
-public suspend inline fun <reified R : Any> DataSet<*>.selectOne(name: String): NamedData<R>? =
-    selectOne(typeOf<R>(), name.toName())

dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/actionInContext.kt

@@ -0,0 +1,2 @@
+package space.kscience.dataforge.data
+

dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/dataFilterJvm.kt

@@ -0,0 +1,85 @@
+package space.kscience.dataforge.data
+
+import kotlinx.coroutines.flow.Flow
+import kotlinx.coroutines.flow.filter
+import space.kscience.dataforge.meta.Meta
+import space.kscience.dataforge.misc.DFExperimental
+import space.kscience.dataforge.names.Name
+import kotlin.coroutines.CoroutineContext
+import kotlin.coroutines.EmptyCoroutineContext
+import kotlin.reflect.KType
+import kotlin.reflect.full.isSubtypeOf
+import kotlin.reflect.typeOf
+
+
+/**
+ * Cast the node to given type if the cast is possible or return null
+ */
+@Suppress("UNCHECKED_CAST")
+private fun <R : Any> Data<*>.castOrNull(type: KType): Data<R>? =
+    if (!this.type.isSubtypeOf(type)) {
+        null
+    } else {
+        object : Data<R> by (this as Data<R>) {
+            override val type: KType = type
+        }
+    }
+
+/**
+ * Select all data matching given type and filters. Does not modify paths
+ *
+ * @param predicate addition filtering condition based on item name and meta. By default, accepts all
+ */
+@OptIn(DFExperimental::class)
+public fun <R : Any> DataSet<*>.filterByType(
+    type: KType,
+    predicate: (name: Name, meta: Meta) -> Boolean = { _, _ -> true },
+): DataSource<R> = object : DataSource<R> {
+    override val dataType = type
+
+    override val coroutineContext: CoroutineContext
+        get() = (this@filterByType as? DataSource)?.coroutineContext ?: EmptyCoroutineContext
+
+    override val meta: Meta get() = this@filterByType.meta
+
+    private fun checkDatum(name: Name, datum: Data<*>): Boolean = datum.type.isSubtypeOf(type)
+            && predicate(name, datum.meta)
+
+    override fun iterator(): Iterator<NamedData<R>> = iterator {
+        for(d in this@filterByType){
+            if(checkDatum(d.name,d.data)){
+                @Suppress("UNCHECKED_CAST")
+                yield(d as NamedData<R>)
+            }
+        }
+    }
+
+    override fun get(name: Name): Data<R>? = this@filterByType[name]?.let { datum ->
+        if (checkDatum(name, datum)) datum.castOrNull(type) else null
+    }
+
+    override val updates: Flow<Name> = this@filterByType.updates.filter { name ->
+        get(name)?.let { datum ->
+            checkDatum(name, datum)
+        } ?: false
+    }
+}
+
+/**
+ * Select a single datum of the appropriate type
+ */
+public inline fun <reified R : Any> DataSet<*>.filterByType(
+    noinline predicate: (name: Name, meta: Meta) -> Boolean = { _, _ -> true },
+): DataSet<R> = filterByType(typeOf<R>(), predicate)
+
+/**
+ * Select a single datum if it is present and of given [type]
+ */
+public fun <R : Any> DataSet<*>.getByType(type: KType, name: Name): NamedData<R>? =
+    get(name)?.castOrNull<R>(type)?.named(name)
+
+public inline fun <reified R : Any> DataSet<*>.getByType(name: Name): NamedData<R>? =
+    this@getByType.getByType(typeOf<R>(), name)
+
+public inline fun <reified R : Any> DataSet<*>.getByType(name: String): NamedData<R>? =
+    this@getByType.getByType(typeOf<R>(), Name.parse(name))

dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/dataSetBuilderInContext.kt

@@ -0,0 +1,40 @@
+package space.kscience.dataforge.data
+
+import kotlinx.coroutines.CoroutineScope
+import kotlinx.coroutines.Job
+import kotlinx.coroutines.launch
+import space.kscience.dataforge.names.Name
+import space.kscience.dataforge.names.plus
+
+
+/**
+ * Append data to node
+ */
+context(DataSetBuilder<T>) public infix fun <T : Any> String.put(data: Data<T>): Unit =
+    data(Name.parse(this), data)
+
+/**
+ * Append node
+ */
+context(DataSetBuilder<T>) public infix fun <T : Any> String.put(dataSet: DataSet<T>): Unit =
+    node(Name.parse(this), dataSet)
+
+/**
+ * Build and append node
+ */
+context(DataSetBuilder<T>) public infix fun <T : Any> String.put(
+    block: DataSetBuilder<T>.() -> Unit,
+): Unit = node(Name.parse(this), block)
+
+/**
+ * Copy given data set and mirror its changes to this [DataTreeBuilder] in [this@setAndObserve]. Returns an update [Job]
+ */
+context(DataSetBuilder<T>) public fun <T : Any> CoroutineScope.setAndWatch(
+    name: Name,
+    dataSet: DataSet<T>,
+): Job = launch {
+    node(name, dataSet)
+    dataSet.updates.collect { nameInBranch ->
+        data(name + nameInBranch, dataSet.get(nameInBranch))
+    }
+}

dataforge-data/src/jvmTest/kotlin/hep/dataforge/data/ActionsTest.kt

@@ -1,42 +0,0 @@
-package hep.dataforge.data
-
-import hep.dataforge.actions.Action
-import hep.dataforge.actions.map
-import kotlinx.coroutines.runBlocking
-import org.junit.jupiter.api.Test
-import kotlin.test.assertEquals
-
-@Suppress("EXPERIMENTAL_API_USAGE")
-class ActionsTest {
-    val data: DataTree<Int> = runBlocking {
-        DataTree {
-            repeat(10) {
-                static(it.toString(), it)
-            }
-        }
-    }
-
-    @Test
-    fun testStaticMapAction() {
-        val plusOne = Action.map<Int, Int> {
-            result { it + 1 }
-        }
-        runBlocking {
-            val result = plusOne.execute(data)
-            assertEquals(2, result.getData("1")?.await())
-        }
-    }
-
-    @Test
-    fun testDynamicMapAction() {
-        val plusOne = Action.map<Int, Int> {
-            result { it + 1 }
-        }
-        val datum = runBlocking {
-            val result = plusOne.execute(data, scope = this)
-            result.getData("1")?.await()
-        }
-        assertEquals(2, datum)
-    }
-
-}

dataforge-data/src/jvmTest/kotlin/space/kscience/dataforge/data/ActionsTest.kt

@@ -0,0 +1,50 @@
+package space.kscience.dataforge.data
+
+import kotlinx.coroutines.ExperimentalCoroutinesApi
+import kotlinx.coroutines.delay
+import kotlinx.coroutines.test.runTest
+import org.junit.jupiter.api.Test
+import space.kscience.dataforge.actions.Action
+import space.kscience.dataforge.actions.invoke
+import space.kscience.dataforge.actions.map
+import space.kscience.dataforge.misc.DFExperimental
+import kotlin.test.assertEquals
+
+@OptIn(DFExperimental::class, ExperimentalCoroutinesApi::class)
+internal class ActionsTest {
+    @Test
+    fun testStaticMapAction() = runTest {
+        val data: DataTree<Int> = DataTree {
+            repeat(10) {
+                static(it.toString(), it)
+            }
+        }
+
+        val plusOne = Action.map<Int, Int> {
+            result { it + 1 }
+        }
+        val result = plusOne(data)
+        assertEquals(2, result["1"]?.await())
+    }
+
+    @Test
+    fun testDynamicMapAction() = runTest {
+        val data: DataSourceBuilder<Int> = DataSource()
+
+        val plusOne = Action.map<Int, Int> {
+            result { it + 1 }
+        }
+
+        val result = plusOne(data)
+
+        repeat(10) {
+            data.static(it.toString(), it)
+        }
+
+        delay(20)
+
+        assertEquals(2, result["1"]?.await())
+        data.close()
+    }
+
+}

dataforge-data/src/jvmTest/kotlin/space/kscience/dataforge/data/DataTreeBuilderTest.kt

@@ -1,8 +1,8 @@
-package hep.dataforge.data
+package space.kscience.dataforge.data
 
-import hep.dataforge.names.toName
 import kotlinx.coroutines.*
-import kotlinx.coroutines.flow.collect
+import space.kscience.dataforge.misc.DFExperimental
+import space.kscience.dataforge.names.asName
 import kotlin.test.Test
 import kotlin.test.assertEquals
 
@@ -19,13 +19,14 @@ internal class DataTreeBuilderTest {
             static("c.f", "c.f")
         }
         runBlocking {
-            assertEquals("a", node.getData("primary.a")?.await())
-            assertEquals("b", node.getData("primary.b")?.await())
-            assertEquals("c.d", node.getData("c.d")?.await())
-            assertEquals("c.f", node.getData("c.f")?.await())
+            assertEquals("a", node["primary.a"]?.await())
+            assertEquals("b", node["primary.b"]?.await())
+            assertEquals("c.d", node["c.d"]?.await())
+            assertEquals("c.f", node["c.f"]?.await())
         }
     }
 
+    @OptIn(DFExperimental::class)
     @Test
     fun testDataUpdate() = runBlocking {
         val updateData: DataTree<Any> = DataTree {
@@ -41,12 +42,12 @@ internal class DataTreeBuilderTest {
                 static("b", "b")
             }
             static("root", "root")
-            populate(updateData)
+            populateFrom(updateData)
         }
 
         runBlocking {
-            assertEquals("a", node.getData("update.a")?.await())
-            assertEquals("a", node.getData("primary.a")?.await())
+            assertEquals("a", node["update.a"]?.await())
+            assertEquals("a", node["primary.a"]?.await())
         }
     }
 
@@ -55,7 +56,7 @@ internal class DataTreeBuilderTest {
         try {
             lateinit var updateJob: Job
             supervisorScope {
-                val subNode = ActiveDataTree<Int> {
+                val subNode = DataSource<Int> {
                     updateJob = launch {
                         repeat(10) {
                             delay(10)
@@ -69,8 +70,8 @@ internal class DataTreeBuilderTest {
                         println(it)
                     }
                 }
-                val rootNode = ActiveDataTree<Int> {
-                    setAndObserve("sub".toName(), subNode)
+                val rootNode = DataSource<Int> {
+                    setAndWatch("sub".asName(), subNode)
                 }
 
                 launch {
@@ -79,11 +80,11 @@ internal class DataTreeBuilderTest {
                     }
                 }
                 updateJob.join()
-                assertEquals(9, rootNode.getData("sub.value")?.await())
+                assertEquals(9, rootNode["sub.value"]?.await())
                 cancel()
             }
         } catch (t: Throwable) {
-            if (t !is CancellationException) throw  t
+            if (t !is CancellationException) throw t
         }
 
     }

dataforge-io/README.md

@@ -0,0 +1,23 @@
+# Module dataforge-io
+
+IO module
+
+## Usage
+
+## Artifact:
+
+The Maven coordinates of this project are `space.kscience:dataforge-io:0.7.0`.
+
+**Gradle Kotlin DSL:**
+```kotlin
+repositories {
+    maven("https://repo.kotlin.link")
+    //uncomment to access development builds
+    //maven("https://maven.pkg.jetbrains.space/spc/p/sci/dev")
+    mavenCentral()
+}
+
+dependencies {
+    implementation("space.kscience:dataforge-io:0.7.0")
+}
+```

dataforge-io/build.gradle.kts

@@ -1,29 +1,26 @@
 plugins {
-    id("ru.mipt.npm.mpp")
-    id("ru.mipt.npm.native")
+    id("space.kscience.gradle.mpp")
 }
 
 description = "IO module"
 
+val ioVersion = "0.2.1"
+
 kscience {
-    useSerialization(sourceSet = ru.mipt.npm.gradle.DependencySourceSet.TEST) {
+    jvm()
+    js()
+    native()
+    useSerialization()
+    useSerialization(sourceSet = space.kscience.gradle.DependencySourceSet.TEST) {
         cbor()
     }
-}
-
-val ioVersion by rootProject.extra("0.2.0-npm-dev-11")
-
-kotlin {
-    sourceSets {
-        commonMain {
-            dependencies {
-                api(project(":dataforge-context"))
-                api("org.jetbrains.kotlinx:kotlinx-io:$ioVersion")
-            }
-        }
+    dependencies {
+        api(projects.dataforgeContext)
+        api("org.jetbrains.kotlinx:kotlinx-io-core:$ioVersion")
+        api("org.jetbrains.kotlinx:kotlinx-io-bytestring:$ioVersion")
     }
 }
 
 readme{
-    maturity = ru.mipt.npm.gradle.Maturity.PROTOTYPE
+    maturity = space.kscience.gradle.Maturity.EXPERIMENTAL
 }

dataforge-io/dataforge-io-yaml/README.md

@@ -0,0 +1,23 @@
+# Module dataforge-io-yaml
+
+YAML meta IO
+
+## Usage
+
+## Artifact:
+
+The Maven coordinates of this project are `space.kscience:dataforge-io-yaml:0.7.0`.
+
+**Gradle Kotlin DSL:**
+```kotlin
+repositories {
+    maven("https://repo.kotlin.link")
+    //uncomment to access development builds
+    //maven("https://maven.pkg.jetbrains.space/spc/p/sci/dev")
+    mavenCentral()
+}
+
+dependencies {
+    implementation("space.kscience:dataforge-io-yaml:0.7.0")
+}
+```

dataforge-io/dataforge-io-yaml/build.gradle.kts

@@ -1,32 +1,23 @@
 plugins {
-    id("ru.mipt.npm.mpp")
-//    id("ru.mipt.npm.native")
+    id("space.kscience.gradle.mpp")
 }
 
 description = "YAML meta IO"
 
 kscience {
-    useSerialization{
-        yamlKt("0.9.0-dev-1")
+    jvm()
+    js()
+    native()
+    dependencies {
+        api(projects.dataforgeIo)
     }
-}
-
-repositories{
-    maven("https://dl.bintray.com/mamoe/yamlkt")
-}
-
-kotlin {
-    sourceSets {
-        commonMain{
-            dependencies {
-                api(project(":dataforge-io"))
-            }
-        }
+    useSerialization{
+        yamlKt()
     }
 }
 
 readme{
-    maturity = ru.mipt.npm.gradle.Maturity.PROTOTYPE
+    maturity = space.kscience.gradle.Maturity.PROTOTYPE
     description ="""
         YAML meta converters and Front Matter envelope format
     """.trimIndent()

dataforge-io/dataforge-io-yaml/src/commonMain/kotlin/hep/dataforge/io/yaml/FrontMatterEnvelopeFormat.kt

@@ -1,124 +0,0 @@
-package hep.dataforge.io.yaml
-
-import hep.dataforge.context.Context
-import hep.dataforge.io.*
-import hep.dataforge.io.IOFormat.Companion.META_KEY
-import hep.dataforge.io.IOFormat.Companion.NAME_KEY
-import hep.dataforge.meta.Meta
-import hep.dataforge.misc.DFExperimental
-import kotlinx.io.*
-import kotlinx.io.text.readUtf8Line
-import kotlinx.io.text.writeUtf8String
-
-@DFExperimental
-public class FrontMatterEnvelopeFormat(
-    private val io: IOPlugin,
-    private val meta: Meta = Meta.EMPTY,
-) : EnvelopeFormat {
-
-    override fun readPartial(input: Input): PartialEnvelope {
-        var line: String
-        var offset = 0u
-        do {
-            line = input.readUtf8Line() //?: error("Input does not contain front matter separator")
-            offset += line.encodeToByteArray().size.toUInt()
-        } while (!line.startsWith(SEPARATOR))
-
-        val readMetaFormat =
-            metaTypeRegex.matchEntire(line)?.groupValues?.first()
-                ?.let { io.resolveMetaFormat(it) } ?: YamlMetaFormat
-
-        //TODO replace by preview
-        val meta = Binary {
-            do {
-                line = input.readUtf8Line()
-                writeUtf8String(line + "\r\n")
-                offset += line.encodeToByteArray().size.toUInt()
-            } while (!line.startsWith(SEPARATOR))
-        }.read {
-            readMetaFormat.readMeta(input)
-
-        }
-        return PartialEnvelope(meta, offset, null)
-    }
-
-    override fun readObject(input: Input): Envelope {
-        var line: String
-        do {
-            line = input.readUtf8Line() //?: error("Input does not contain front matter separator")
-        } while (!line.startsWith(SEPARATOR))
-
-        val readMetaFormat =
-            metaTypeRegex.matchEntire(line)?.groupValues?.first()
-                ?.let { io.resolveMetaFormat(it) } ?: YamlMetaFormat
-
-        val meta = Binary {
-            do {
-                writeUtf8String(input.readUtf8Line() + "\r\n")
-            } while (!line.startsWith(SEPARATOR))
-        }.read {
-            readMetaFormat.readMeta(input)
-        }
-        val bytes = input.readByteArray()
-        val data = bytes.asBinary()
-        return SimpleEnvelope(meta, data)
-    }
-
-    override fun writeEnvelope(
-        output: Output,
-        envelope: Envelope,
-        metaFormatFactory: MetaFormatFactory,
-        formatMeta: Meta,
-    ) {
-        val metaFormat = metaFormatFactory(formatMeta, this@FrontMatterEnvelopeFormat.io.context)
-        output.writeRawString("${hep.dataforge.io.yaml.FrontMatterEnvelopeFormat.Companion.SEPARATOR}\r\n")
-        metaFormat.run { this.writeObject(output, envelope.meta) }
-        output.writeRawString("${hep.dataforge.io.yaml.FrontMatterEnvelopeFormat.Companion.SEPARATOR}\r\n")
-        //Printing data
-        envelope.data?.let { data ->
-            output.writeBinary(data)
-        }
-    }
-
-    override fun toMeta(): Meta = Meta {
-        NAME_KEY put name.toString()
-        META_KEY put meta
-    }
-
-    public companion object : EnvelopeFormatFactory {
-        public const val SEPARATOR: String = "---"
-
-        private val metaTypeRegex = "---(\\w*)\\s*".toRegex()
-
-        override fun invoke(meta: Meta, context: Context): EnvelopeFormat {
-            return FrontMatterEnvelopeFormat(context.io, meta)
-        }
-
-        override fun peekFormat(io: IOPlugin, input: Input): EnvelopeFormat? {
-            return input.preview {
-                val line = readUtf8Line()
-                return@preview if (line.startsWith("---")) {
-                    invoke()
-                } else {
-                    null
-                }
-            }
-        }
-
-        private val default by lazy { invoke() }
-
-        override fun readPartial(input: Input): PartialEnvelope =
-            default.readPartial(input)
-
-        override fun writeEnvelope(
-            output: Output,
-            envelope: Envelope,
-            metaFormatFactory: MetaFormatFactory,
-            formatMeta: Meta,
-        ): Unit = default.writeEnvelope(output, envelope, metaFormatFactory, formatMeta)
-
-
-        override fun readObject(input: Input): Envelope = default.readObject(input)
-
-    }
-}

dataforge-io/dataforge-io-yaml/src/commonMain/kotlin/hep/dataforge/io/yaml/YamlMetaFormat.kt

@@ -1,122 +0,0 @@
-package hep.dataforge.io.yaml
-
-import hep.dataforge.context.Context
-import hep.dataforge.io.IOFormat.Companion.META_KEY
-import hep.dataforge.io.IOFormat.Companion.NAME_KEY
-import hep.dataforge.io.MetaFormat
-import hep.dataforge.io.MetaFormatFactory
-import hep.dataforge.meta.*
-import hep.dataforge.meta.descriptors.ItemDescriptor
-import hep.dataforge.meta.descriptors.NodeDescriptor
-import hep.dataforge.misc.DFExperimental
-import hep.dataforge.names.NameToken
-import hep.dataforge.names.withIndex
-import hep.dataforge.values.ListValue
-import hep.dataforge.values.Null
-import hep.dataforge.values.parseValue
-import kotlinx.io.Input
-import kotlinx.io.Output
-import kotlinx.io.text.readUtf8String
-import kotlinx.io.text.writeUtf8String
-import net.mamoe.yamlkt.*
-
-public fun Meta.toYaml(): YamlMap {
-    val map: Map<String, Any?> = items.entries.associate { (key, item) ->
-        key.toString() to when (item) {
-            is MetaItemValue -> {
-                item.value.value
-            }
-            is MetaItemNode -> {
-                item.node.toYaml()
-            }
-        }
-    }
-    return YamlMap(map)
-}
-
-private class YamlMeta(private val yamlMap: YamlMap, private val descriptor: NodeDescriptor? = null) : MetaBase() {
-
-    private fun buildItems(): Map<NameToken, MetaItem> {
-        val map = LinkedHashMap<NameToken, MetaItem>()
-
-        yamlMap.content.entries.forEach { (key, value) ->
-            val stringKey = key.toString()
-            val itemDescriptor = descriptor?.items?.get(stringKey)
-            val token = NameToken(stringKey)
-            when (value) {
-                YamlNull -> Null.asMetaItem()
-                is YamlLiteral -> map[token] = value.content.parseValue().asMetaItem()
-                is YamlMap -> map[token] = value.toMeta().asMetaItem()
-                is YamlList -> if (value.all { it is YamlLiteral }) {
-                    val listValue = ListValue(
-                        value.map {
-                            //We already checked that all values are primitives
-                            (it as YamlLiteral).content.parseValue()
-                        }
-                    )
-                    map[token] = MetaItemValue(listValue)
-                } else value.forEachIndexed { index, yamlElement ->
-                    val indexKey = (itemDescriptor as? NodeDescriptor)?.indexKey ?: ItemDescriptor.DEFAULT_INDEX_KEY
-                    val indexValue: String = (yamlElement as? YamlMap)?.getStringOrNull(indexKey)
-                        ?: index.toString() //In case index is non-string, the backward transformation will be broken.
-
-                    val tokenWithIndex = token.withIndex(indexValue)
-                    map[tokenWithIndex] = yamlElement.toMetaItem(itemDescriptor)
-                }
-            }
-        }
-        return map
-    }
-
-    override val items: Map<NameToken, MetaItem> get() = buildItems()
-}
-
-public fun YamlElement.toMetaItem(descriptor: ItemDescriptor? = null): MetaItem = when (this) {
-    YamlNull -> Null.asMetaItem()
-    is YamlLiteral -> content.parseValue().asMetaItem()
-    is YamlMap -> toMeta().asMetaItem()
-    //We can't return multiple items therefore we create top level node
-    is YamlList -> YamlMap(mapOf("@yamlArray" to this)).toMetaItem(descriptor)
-}
-
-public fun YamlMap.toMeta(): Meta = YamlMeta(this)
-
-
-/**
- * Represent meta as Yaml
- */
-@DFExperimental
-public class YamlMetaFormat(private val meta: Meta) : MetaFormat {
-
-    override fun writeMeta(output: Output, meta: Meta, descriptor: NodeDescriptor?) {
-        val yaml = meta.toYaml()
-        val string = Yaml.encodeToString(yaml)
-        output.writeUtf8String(string)
-    }
-
-    override fun readMeta(input: Input, descriptor: NodeDescriptor?): Meta {
-        val yaml = Yaml.decodeYamlMapFromString(input.readUtf8String())
-        return yaml.toMeta()
-    }
-
-    override fun toMeta(): Meta = Meta {
-        NAME_KEY put FrontMatterEnvelopeFormat.name.toString()
-        META_KEY put meta
-    }
-
-    public companion object : MetaFormatFactory {
-        override fun invoke(meta: Meta, context: Context): MetaFormat = YamlMetaFormat(meta)
-
-        override val shortName: String = "yaml"
-
-        override val key: Short = 0x594d //YM
-
-        private val default = YamlMetaFormat()
-
-        override fun writeMeta(output: Output, meta: Meta, descriptor: NodeDescriptor?): Unit =
-            default.writeMeta(output, meta, descriptor)
-
-        override fun readMeta(input: Input, descriptor: NodeDescriptor?): Meta =
-            default.readMeta(input, descriptor)
-    }
-}

dataforge-io/dataforge-io-yaml/src/commonMain/kotlin/space/kscience/dataforge/io/yaml/FrontMatterEnvelopeFormat.kt

@@ -0,0 +1,97 @@
+package space.kscience.dataforge.io.yaml
+
+import kotlinx.io.Sink
+import kotlinx.io.Source
+import kotlinx.io.bytestring.ByteString
+import kotlinx.io.bytestring.encodeToByteString
+import kotlinx.io.readByteString
+import kotlinx.io.writeString
+import space.kscience.dataforge.context.Context
+import space.kscience.dataforge.context.Global
+import space.kscience.dataforge.io.*
+import space.kscience.dataforge.meta.Meta
+import space.kscience.dataforge.names.Name
+import space.kscience.dataforge.names.plus
+
+public class FrontMatterEnvelopeFormat(
+    private val io: IOPlugin,
+    private val meta: Meta = Meta.EMPTY,
+    private val metaFormatFactory: MetaFormatFactory = YamlMetaFormat,
+) : EnvelopeFormat {
+
+    override fun readFrom(binary: Binary): Envelope = binary.read {
+        var offset = 0
+
+        offset += discardWithSeparator(
+            SEPARATOR,
+            atMost = 1024,
+        )
+
+        val line = ByteArray {
+            offset += readWithSeparatorTo(this, "\n".encodeToByteString())
+        }.decodeToString()
+
+        val readMetaFormat = line.trim().takeIf { it.isNotBlank() }?.let { io.resolveMetaFormat(it) } ?: YamlMetaFormat
+
+        val packet = ByteArray {
+            offset += readWithSeparatorTo(this, SEPARATOR)
+        }
+
+        offset += discardLine()
+
+        val meta = readMetaFormat.readFrom(packet.asBinary())
+        Envelope(meta, binary.view(offset))
+    }
+
+    override fun readFrom(source: Source): Envelope = readFrom(source.readBinary())
+
+    override fun writeTo(
+        sink: Sink,
+        obj: Envelope,
+    ) {
+        val metaFormat = metaFormatFactory.build(io.context, meta)
+        val formatSuffix = if (metaFormat is YamlMetaFormat) "" else metaFormatFactory.shortName
+        sink.writeString("$SEPARATOR${formatSuffix}\r\n")
+        metaFormat.run { metaFormat.writeTo(sink, obj.meta) }
+        sink.writeString("$SEPARATOR\r\n")
+        //Printing data
+        obj.data?.let { data ->
+            sink.writeBinary(data)
+        }
+    }
+
+    public companion object : EnvelopeFormatFactory {
+        public val SEPARATOR: ByteString = "---".encodeToByteString()
+
+        private val metaTypeRegex = "---(\\w*)\\s*".toRegex()
+
+        override val name: Name = EnvelopeFormatFactory.ENVELOPE_FACTORY_NAME + "frontMatter"
+
+        override fun build(context: Context, meta: Meta): EnvelopeFormat {
+            return FrontMatterEnvelopeFormat(context.io, meta)
+        }
+
+        override fun peekFormat(io: IOPlugin, binary: Binary): EnvelopeFormat? = binary.read {
+            //read raw string to avoid UTF issues
+            val line = readByteString(3)
+            return@read if (line == "---".encodeToByteString()) {
+                default
+            } else {
+                null
+            }
+        }
+
+        private val default by lazy { build(Global, Meta.EMPTY) }
+
+        override fun readFrom(binary: Binary): Envelope = default.readFrom(binary)
+
+        override fun writeTo(
+            sink: Sink,
+            obj: Envelope,
+        ): Unit = default.writeTo(sink, obj)
+
+
+        override fun readFrom(source: Source): Envelope = default.readFrom(source)
+
+    }
+}

dataforge-io/dataforge-io-yaml/src/commonMain/kotlin/space/kscience/dataforge/io/yaml/YamlMetaFormat.kt

@@ -0,0 +1,118 @@
+package space.kscience.dataforge.io.yaml
+
+import kotlinx.io.Sink
+import kotlinx.io.Source
+import kotlinx.io.readString
+import kotlinx.io.writeString
+import net.mamoe.yamlkt.*
+import space.kscience.dataforge.context.Context
+import space.kscience.dataforge.io.MetaFormat
+import space.kscience.dataforge.io.MetaFormatFactory
+import space.kscience.dataforge.meta.*
+import space.kscience.dataforge.meta.descriptors.MetaDescriptor
+import space.kscience.dataforge.meta.descriptors.get
+import space.kscience.dataforge.names.NameToken
+import space.kscience.dataforge.names.withIndex
+import kotlin.collections.component1
+import kotlin.collections.component2
+import kotlin.collections.set
+
+public fun Meta.toYaml(): YamlMap {
+    val map: Map<String, Any?> = items.entries.associate { (key, item) ->
+        key.toString() to if (item.isLeaf) {
+            item.value?.value
+        } else {
+            item.toYaml()
+        }
+    }
+
+    return YamlMap(map)
+}
+
+private class YamlMeta(private val yamlMap: YamlMap, private val descriptor: MetaDescriptor? = null) : Meta {
+
+    override val value: Value?
+        get() = yamlMap.getStringOrNull(null)?.let { Value.parse(it) }
+
+    private fun buildItems(): Map<NameToken, Meta> {
+        val map = LinkedHashMap<NameToken, Meta>()
+
+        yamlMap.content.entries.forEach { (key, value) ->
+            val stringKey = key.toString()
+            val itemDescriptor = descriptor?.get(stringKey)
+            val token = NameToken(stringKey)
+            when (value) {
+                YamlNull -> Meta(Null)
+                is YamlLiteral -> map[token] = Meta(Value.parse(value.content))
+                is YamlMap -> map[token] = value.toMeta()
+                is YamlList -> if (value.all { it is YamlLiteral }) {
+                    val listValue = ListValue(
+                        value.map {
+                            //We already checked that all values are primitives
+                            Value.parse((it as YamlLiteral).content)
+                        }
+                    )
+                    map[token] = Meta(listValue)
+                } else value.forEachIndexed { index, yamlElement ->
+                    val indexKey = itemDescriptor?.indexKey
+                    val indexValue: String = (yamlElement as? YamlMap)?.getStringOrNull(indexKey)
+                        ?: index.toString() //In case index is non-string, the backward transformation will be broken.
+
+                    val tokenWithIndex = token.withIndex(indexValue)
+                    map[tokenWithIndex] = yamlElement.toMeta(itemDescriptor)
+                }
+            }
+        }
+        return map
+    }
+
+    override val items: Map<NameToken, Meta> get() = buildItems()
+
+    override fun toString(): String = Meta.toString(this)
+    override fun equals(other: Any?): Boolean = Meta.equals(this, other as? Meta)
+    override fun hashCode(): Int = Meta.hashCode(this)
+}
+
+public fun YamlElement.toMeta(descriptor: MetaDescriptor? = null): Meta = when (this) {
+    YamlNull -> Meta(Null)
+    is YamlLiteral -> Meta(Value.parse(content))
+    is YamlMap -> toMeta()
+    //We can't return multiple items therefore we create top level node
+    is YamlList -> YamlMap(mapOf("@yamlArray" to this)).toMeta(descriptor)
+}
+
+public fun YamlMap.toMeta(): Meta = YamlMeta(this)
+
+
+/**
+ * Represent meta as Yaml
+ */
+public class YamlMetaFormat(private val meta: Meta) : MetaFormat {
+
+    override fun writeMeta(sink: Sink, meta: Meta, descriptor: MetaDescriptor?) {
+        val yaml: YamlMap = meta.toYaml()
+        val string = Yaml.encodeToString(YamlMap.serializer(), yaml)
+        sink.writeString(string)
+    }
+
+    override fun readMeta(source: Source, descriptor: MetaDescriptor?): Meta {
+        val yaml = Yaml.decodeYamlMapFromString(source.readString())
+        return yaml.toMeta()
+    }
+
+    public companion object : MetaFormatFactory {
+        override fun build(context: Context, meta: Meta): MetaFormat = YamlMetaFormat(meta)
+
+        override val shortName: String = "yaml"
+
+        override val key: Short = 0x594d //YM
+
+        private val default = YamlMetaFormat(Meta.EMPTY)
+
+        override fun writeMeta(sink: Sink, meta: Meta, descriptor: MetaDescriptor?): Unit =
+            default.writeMeta(sink, meta, descriptor)
+
+        override fun readMeta(source: Source, descriptor: MetaDescriptor?): Meta =
+            default.readMeta(source, descriptor)
+    }
+}

dataforge-io/dataforge-io-yaml/src/commonMain/kotlin/space/kscience/dataforge/io/yaml/YamlPlugin.kt

@@ -0,0 +1,30 @@
+package space.kscience.dataforge.io.yaml
+
+import space.kscience.dataforge.context.AbstractPlugin
+import space.kscience.dataforge.context.Context
+import space.kscience.dataforge.context.PluginFactory
+import space.kscience.dataforge.context.PluginTag
+import space.kscience.dataforge.io.EnvelopeFormatFactory
+import space.kscience.dataforge.io.IOPlugin
+import space.kscience.dataforge.io.MetaFormatFactory
+import space.kscience.dataforge.meta.Meta
+import space.kscience.dataforge.names.Name
+import space.kscience.dataforge.names.asName
+
+public class YamlPlugin(meta: Meta) : AbstractPlugin(meta) {
+    public val io: IOPlugin by require(IOPlugin)
+
+    override val tag: PluginTag get() = Companion.tag
+
+    override fun content(target: String): Map<Name, Any> = when (target) {
+        MetaFormatFactory.META_FORMAT_TYPE -> mapOf("yaml".asName() to YamlMetaFormat)
+        EnvelopeFormatFactory.ENVELOPE_FORMAT_TYPE -> mapOf(FrontMatterEnvelopeFormat.name to FrontMatterEnvelopeFormat)
+        else -> super.content(target)
+    }
+
+    public companion object : PluginFactory<YamlPlugin> {
+        override val tag: PluginTag = PluginTag("io.yaml", group = PluginTag.DATAFORGE_GROUP)
+
+        override fun build(context: Context, meta: Meta): YamlPlugin = YamlPlugin(meta)
+    }
+}

dataforge-io/dataforge-io-yaml/src/commonTest/kotlin/space/kscience/dataforge/io/yaml/FrontMatterEnvelopeFormatTest.kt

@@ -0,0 +1,37 @@
+@file:OptIn(DFExperimental::class)
+
+package space.kscience.dataforge.io.yaml
+
+import space.kscience.dataforge.context.Context
+import space.kscience.dataforge.io.io
+import space.kscience.dataforge.io.readEnvelope
+import space.kscience.dataforge.io.toByteArray
+import space.kscience.dataforge.meta.get
+import space.kscience.dataforge.meta.string
+import space.kscience.dataforge.misc.DFExperimental
+import kotlin.test.Test
+import kotlin.test.assertEquals
+
+internal class FrontMatterEnvelopeFormatTest {
+
+    val context = Context {
+        plugin(YamlPlugin)
+    }
+
+    @Test
+    fun frontMatter(){
+        val text = """
+            ---
+            content_type: magprog
+            magprog_section: contacts
+            section_title: Контакты
+            language: ru
+            ---
+            Some text here
+        """.trimIndent()
+
+        val envelope = context.io.readEnvelope(text)
+        assertEquals("Some text here", envelope.data!!.toByteArray().decodeToString().trim())
+        assertEquals("magprog", envelope.meta["content_type"].string)
+    }
+}

dataforge-io/dataforge-io-yaml/src/commonTest/kotlin/space/kscience/dataforge/io/yaml/YamlMetaFormatTest.kt

@@ -1,11 +1,11 @@
-package hep.dataforge.io.yaml
+package space.kscience.dataforge.io.yaml
 
-import hep.dataforge.io.parse
-import hep.dataforge.io.toString
-import hep.dataforge.meta.Meta
-import hep.dataforge.meta.get
-import hep.dataforge.meta.seal
-import hep.dataforge.misc.DFExperimental
+import space.kscience.dataforge.io.parse
+import space.kscience.dataforge.io.toString
+import space.kscience.dataforge.meta.Meta
+import space.kscience.dataforge.meta.get
+import space.kscience.dataforge.meta.seal
+import space.kscience.dataforge.misc.DFExperimental
 import kotlin.test.Test
 import kotlin.test.assertEquals