diff --git a/build.gradle.kts b/build.gradle.kts index 97c9b3f8..8c9f0b6e 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -4,7 +4,7 @@ plugins { allprojects { group = "space.kscience" - version = "0.5.2-dev-3" + version = "0.5.2-dev-4" repositories{ mavenCentral() } diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/MapAction.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/MapAction.kt index 0e6d6f0f..962d4b4d 100644 --- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/MapAction.kt +++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/MapAction.kt @@ -76,7 +76,7 @@ internal class MapAction( return newData.named(newName) } - val flow = dataSet.flow().map(::mapOne) + val flow = dataSet.flowData().map(::mapOne) return ActiveDataTree(outputType) { populate(flow) diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/ReduceAction.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/ReduceAction.kt index 7186504d..39f26dd6 100644 --- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/ReduceAction.kt +++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/ReduceAction.kt @@ -84,7 +84,7 @@ internal class ReduceAction( override fun CoroutineScope.transform(set: DataSet, meta: Meta, key: Name): Flow> = flow { ReduceGroupBuilder(inputType, this@transform, meta).apply(action).buildGroups(set).forEach { group -> - val dataFlow: Map> = group.set.flow().fold(HashMap()) { acc, value -> + val dataFlow: Map> = group.set.flowData().fold(HashMap()) { acc, value -> acc.apply { acc[value.name] = value.data } diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/SplitAction.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/SplitAction.kt index e8ed7961..51a47c4d 100644 --- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/SplitAction.kt +++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/SplitAction.kt @@ -72,7 +72,7 @@ internal class SplitAction( } return ActiveDataTree(outputType) { - populate(dataSet.flow().flatMapConcat(transform = ::splitOne)) + populate(dataSet.flowData().flatMapConcat(transform = ::splitOne)) scope?.launch { dataSet.updates.collect { name -> //clear old nodes diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSet.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSet.kt index d4ce7990..64f1f521 100644 --- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSet.kt +++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSet.kt @@ -17,10 +17,8 @@ public interface DataSet { /** * Traverse this provider or its child. The order is not guaranteed. - * [root] points to a root name for traversal. If it is empty, traverse this source, if it points to a [Data], - * return flow, that contains single [Data], if it points to a node with children, return children. */ - public fun flow(): Flow> + public fun flowData(): Flow> /** * Get data with given name. @@ -31,7 +29,7 @@ public interface DataSet { * Get a snapshot of names of top level children of given node. Empty if node does not exist or is a leaf. */ public suspend fun listTop(prefix: Name = Name.EMPTY): List = - flow().map { it.name }.filter { it.startsWith(prefix) && (it.length == prefix.length + 1) }.toList() + flowData().map { it.name }.filter { it.startsWith(prefix) && (it.length == prefix.length + 1) }.toList() // By default, traverses the whole tree. Could be optimized in descendants public companion object { @@ -45,7 +43,7 @@ public interface DataSet { //private val nothing: Nothing get() = error("this is nothing") - override fun flow(): Flow> = emptyFlow() + override fun flowData(): Flow> = emptyFlow() override suspend fun getData(name: Name): Data? = null } @@ -67,7 +65,7 @@ public val DataSet.updates: Flow get() = if (this is ActiveDa /** * Flow all data nodes with names starting with [branchName] */ -public fun DataSet.flowChildren(branchName: Name): Flow> = this@flowChildren.flow().filter { +public fun DataSet.flowChildren(branchName: Name): Flow> = this@flowChildren.flowData().filter { it.name.startsWith(branchName) } @@ -75,7 +73,7 @@ public fun DataSet.flowChildren(branchName: Name): Flow DataSet.startAll(coroutineScope: CoroutineScope): Job = coroutineScope.launch { - flow().map { + flowData().map { it.launch(this@launch) }.toList().joinAll() } @@ -83,7 +81,7 @@ public fun DataSet.startAll(coroutineScope: CoroutineScope): Job = public suspend fun DataSet.join(): Unit = coroutineScope { startAll(this).join() } public suspend fun DataSet<*>.toMeta(): Meta = Meta { - flow().collect { + flowData().collect { if (it.name.endsWith(DataSet.META_KEY)) { set(it.name, it.meta) } else { diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSetBuilder.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSetBuilder.kt index d966d8ae..ea8e3e38 100644 --- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSetBuilder.kt +++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSetBuilder.kt @@ -30,7 +30,7 @@ public interface DataSetBuilder { } //Set new items - dataSet.flow().collect { + dataSet.flowData().collect { emit(name + it.name, it.data) } } @@ -139,7 +139,7 @@ public suspend inline fun DataSetBuilder.static( */ @DFExperimental public suspend fun DataSetBuilder.populate(tree: DataSet): Unit = coroutineScope { - tree.flow().collect { + tree.flowData().collect { //TODO check if the place is occupied emit(it.name, it.data) } diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataTree.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataTree.kt index 87397f3e..ea9d67e2 100644 --- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataTree.kt +++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataTree.kt @@ -32,12 +32,12 @@ public interface DataTree : DataSet { */ public suspend fun items(): Map> - override fun flow(): Flow> = flow { + override fun flowData(): Flow> = flow { items().forEach { (token, childItem: DataTreeItem) -> if(!token.body.startsWith("@")) { when (childItem) { is DataTreeItem.Leaf -> emit(childItem.data.named(token.asName())) - is DataTreeItem.Node -> emitAll(childItem.tree.flow().map { it.named(token + it.name) }) + is DataTreeItem.Node -> emitAll(childItem.tree.flowData().map { it.named(token + it.name) }) } } } diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/GroupRule.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/GroupRule.kt index a9050b3c..d26cbfb1 100644 --- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/GroupRule.kt +++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/GroupRule.kt @@ -44,7 +44,7 @@ public interface GroupRule { ): Map> { val map = HashMap>() - set.flow().collect { data -> + set.flowData().collect { data -> val tagValue = data.meta[key]?.string ?: defaultTagValue map.getOrPut(tagValue) { ActiveDataTree(set.dataType) }.emit(data.name, data.data) } diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/StaticDataTree.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/StaticDataTree.kt index f57af184..e68c16e2 100644 --- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/StaticDataTree.kt +++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/StaticDataTree.kt @@ -53,7 +53,7 @@ internal class StaticDataTree( set(name, DataTreeItem.Node(dataSet)) } else { coroutineScope { - dataSet.flow().collect { + dataSet.flowData().collect { emit(name + it.name, it.data) } } diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataFilter.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataFilter.kt index 469594fe..a526cdd7 100644 --- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataFilter.kt +++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataFilter.kt @@ -20,8 +20,8 @@ public fun DataSet.filter( ): ActiveDataSet = object : ActiveDataSet { override val dataType: KType get() = this@filter.dataType - override fun flow(): Flow> = - this@filter.flow().filter { predicate(it.name, it.data) } + override fun flowData(): Flow> = + this@filter.flowData().filter { predicate(it.name, it.data) } override suspend fun getData(name: Name): Data? = this@filter.getData(name)?.takeIf { predicate(name, it) @@ -40,7 +40,7 @@ public fun DataSet.withNamePrefix(prefix: Name): DataSet = if (p else object : ActiveDataSet { override val dataType: KType get() = this@withNamePrefix.dataType - override fun flow(): Flow> = this@withNamePrefix.flow().map { it.data.named(prefix + it.name) } + override fun flowData(): Flow> = this@withNamePrefix.flowData().map { it.data.named(prefix + it.name) } override suspend fun getData(name: Name): Data? = name.removeHeadOrNull(name)?.let { this@withNamePrefix.getData(it) } @@ -56,7 +56,7 @@ public fun DataSet.branch(branchName: Name): DataSet = if (branc } else object : ActiveDataSet { override val dataType: KType get() = this@branch.dataType - override fun flow(): Flow> = this@branch.flow().mapNotNull { + override fun flowData(): Flow> = this@branch.flowData().mapNotNull { it.name.removeHeadOrNull(branchName)?.let { name -> it.data.named(name) } diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataTransform.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataTransform.kt index f4c4a710..14b90729 100644 --- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataTransform.kt +++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataTransform.kt @@ -144,7 +144,7 @@ public suspend fun DataSet.map( block: suspend (T) -> R, ): DataTree = DataTree(outputType) { populate( - flow().map { + flowData().map { val newMeta = it.meta.toMutableMeta().apply(metaTransform).seal() Data(outputType, newMeta, coroutineContext, listOf(it)) { block(it.await()) @@ -162,7 +162,7 @@ public suspend inline fun DataSet.map( public suspend fun DataSet.forEach(block: suspend (NamedData) -> Unit) { contract { callsInPlace(block, InvocationKind.EXACTLY_ONCE) } - flow().collect { + flowData().collect { block(it) } } @@ -171,11 +171,11 @@ public suspend inline fun DataSet.reduceToData( coroutineContext: CoroutineContext = EmptyCoroutineContext, meta: Meta = Meta.EMPTY, noinline transformation: suspend (Flow>) -> R, -): Data = flow().reduceToData(coroutineContext, meta, transformation) +): Data = flowData().reduceToData(coroutineContext, meta, transformation) public suspend inline fun DataSet.foldToData( initial: R, coroutineContext: CoroutineContext = EmptyCoroutineContext, meta: Meta = Meta.EMPTY, noinline block: suspend (result: R, data: NamedData) -> R, -): Data = flow().foldToData(initial, coroutineContext, meta, block) \ No newline at end of file +): Data = flowData().foldToData(initial, coroutineContext, meta, block) \ No newline at end of file diff --git a/dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/select.kt b/dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/select.kt index 5b09468d..5e65cee7 100644 --- a/dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/select.kt +++ b/dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/select.kt @@ -43,7 +43,7 @@ public fun DataSet<*>.select( && (namePattern == null || name.matches(namePattern)) && filter(name, datum.meta) - override fun flow(): Flow> = this@select.flow().filter { + override fun flowData(): Flow> = this@select.flowData().filter { checkDatum(it.name, it.data) }.map { @Suppress("UNCHECKED_CAST") diff --git a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/Binary.kt b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/Binary.kt index 0a375ab1..54353a15 100644 --- a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/Binary.kt +++ b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/Binary.kt @@ -54,6 +54,7 @@ public fun Binary.toByteArray(): ByteArray = if (this is ByteArrayBinary) { } } +//TODO optimize for file-based Inputs public fun Input.readBinary(size: Int): Binary { val array = readBytes(size) return ByteArrayBinary(array) diff --git a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/IOPlugin.kt b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/IOPlugin.kt index daf5ba41..fafa3ce3 100644 --- a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/IOPlugin.kt +++ b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/IOPlugin.kt @@ -29,7 +29,6 @@ public class IOPlugin(meta: Meta) : AbstractPlugin(meta) { } } - public val metaFormatFactories: Collection by lazy { context.gather(META_FORMAT_TYPE).values } diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MutableMeta.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MutableMeta.kt index 0c2c653e..e1661e53 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MutableMeta.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MutableMeta.kt @@ -155,7 +155,7 @@ public fun MutableMeta.getOrCreate(key: String): MutableMeta = getOrCreate(Name. public interface MutableTypedMeta> : TypedMeta, MutableMeta { /** - * Zero-copy attach or replace existing node. Node is used with any additional state, listeners, etc. + * Zero-copy (if possible) attach or replace existing node. Node is used with any additional state, listeners, etc. * In some cases it is possible to have the same node as a child to several others */ @DFExperimental diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/Scheme.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/Scheme.kt index ffc29152..fa829d29 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/Scheme.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/Scheme.kt @@ -32,7 +32,7 @@ public open class Scheme : Described, MetaRepr, MutableMetaProvider, Configurabl internal fun wrap( newMeta: MutableMeta, - preserveDefault: Boolean = false + preserveDefault: Boolean = false, ) { if (preserveDefault) { defaultMeta = targetMeta.seal() @@ -120,7 +120,11 @@ public open class Scheme : Described, MetaRepr, MutableMetaProvider, Configurabl @DFExperimental override fun attach(name: Name, node: ObservableMutableMeta) { - TODO("Not yet implemented") + //TODO implement zero-copy attachment + setMeta(name, meta) + node.onChange(this) { changeName -> + setMeta(name + changeName, this[changeName]) + } } } @@ -139,6 +143,12 @@ public fun T.retarget(provider: MutableMeta): T = apply { */ public inline operator fun T.invoke(block: T.() -> Unit): T = apply(block) +/** + * Create a copy of given [Scheme] + */ +public inline fun T.copy(spec: SchemeSpec, block: T.() -> Unit = {}): T = + spec.read(meta.copy()).apply(block) + /** * A specification for simplified generation of wrappers */ diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/names/Name.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/names/Name.kt index 7bf8d14e..debe5e17 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/names/Name.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/names/Name.kt @@ -201,4 +201,6 @@ public fun Name.removeHeadOrNull(head: Name): Name? = if (startsWith(head)) { Name(tokens.subList(head.length, length)) } else { null -} \ No newline at end of file +} + +public fun String.parseAsName(): Name = Name.parse(this) \ No newline at end of file diff --git a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/TaskResult.kt b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/TaskResult.kt index 5a65f219..6a12e3bc 100644 --- a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/TaskResult.kt +++ b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/TaskResult.kt @@ -25,7 +25,7 @@ public interface TaskResult : DataSet { */ public val taskMeta: Meta - override fun flow(): Flow> + override fun flowData(): Flow> override suspend fun getData(name: Name): TaskData? } @@ -36,7 +36,7 @@ private class TaskResultImpl( override val taskMeta: Meta, ) : TaskResult, DataSet by dataSet { - override fun flow(): Flow> = dataSet.flow().map { + override fun flowData(): Flow> = dataSet.flowData().map { workspace.wrapData(it, it.name, taskName, taskMeta) } diff --git a/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/workspaceJvm.kt b/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/workspaceJvm.kt index 61e1dbec..ffa397d3 100644 --- a/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/workspaceJvm.kt +++ b/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/workspaceJvm.kt @@ -11,11 +11,11 @@ public fun WorkspaceBuilder.data(builder: suspend DataSetBuilder.() -> Unit buildData(builder) } -public inline fun TaskResultBuilder<*>.selectData(namePattern: Name? = null): DataSelector = object : DataSelector { +public inline fun TaskResultBuilder<*>.data(namePattern: Name? = null): DataSelector = object : DataSelector { override suspend fun select(workspace: Workspace, meta: Meta): DataSet = workspace.data.select(namePattern) } -public suspend inline fun TaskResultBuilder<*>.from( +public suspend inline fun TaskResultBuilder<*>.fromTask( task: Name, taskMeta: Meta = Meta.EMPTY, ): DataSet = workspace.produce(task, taskMeta).select() \ No newline at end of file diff --git a/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/DataPropagationTest.kt b/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/DataPropagationTest.kt index effc43e0..f88c6f59 100644 --- a/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/DataPropagationTest.kt +++ b/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/DataPropagationTest.kt @@ -16,7 +16,7 @@ class DataPropagationTestPlugin : WorkspacePlugin() { val allData by task { val selectedData = workspace.data.select() - val result: Data = selectedData.flow().foldToData(0) { result, data -> + val result: Data = selectedData.flowData().foldToData(0) { result, data -> result + data.await() } emit("result", result) @@ -58,7 +58,7 @@ class DataPropagationTest { fun testAllData() { runBlocking { val node = testWorkspace.produce("Test.allData") - assertEquals(4950, node.flow().single().await()) + assertEquals(4950, node.flowData().single().await()) } } @@ -66,7 +66,7 @@ class DataPropagationTest { fun testSingleData() { runBlocking { val node = testWorkspace.produce("Test.singleData") - assertEquals(12, node.flow().single().await()) + assertEquals(12, node.flowData().single().await()) } } } \ No newline at end of file diff --git a/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/SimpleWorkspaceTest.kt b/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/SimpleWorkspaceTest.kt index edbe3a4b..a1ce6ae1 100644 --- a/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/SimpleWorkspaceTest.kt +++ b/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/SimpleWorkspaceTest.kt @@ -71,7 +71,7 @@ class SimpleWorkspaceTest { } val square by task { - pipeFrom(selectData()) { arg, name, meta -> + pipeFrom(data()) { arg, name, meta -> if (meta["testFlag"].boolean == true) { println("flag") } @@ -89,7 +89,7 @@ class SimpleWorkspaceTest { } val linear by task { - pipeFrom(selectData()) { arg, name, _ -> + pipeFrom(data()) { arg, name, _ -> workspace.logger.info { "Starting linear on $name" } arg * 2 + 1 } @@ -162,7 +162,7 @@ class SimpleWorkspaceTest { fun testWorkspace() { runBlocking { val node = workspace.runBlocking("sum") - val res = node.flow().single() + val res = node.flowData().single() assertEquals(328350, res.await()) } } @@ -172,7 +172,7 @@ class SimpleWorkspaceTest { fun testMetaPropagation() { runBlocking { val node = workspace.produce("sum") { "testFlag" put true } - val res = node.flow().single().await() + val res = node.flowData().single().await() } } @@ -195,7 +195,7 @@ class SimpleWorkspaceTest { fun testFilter() { runBlocking { val node = workspace.produce("filterOne") - assertEquals(12, node.flow().first().await()) + assertEquals(12, node.flowData().first().await()) } } } \ No newline at end of file diff --git a/settings.gradle.kts b/settings.gradle.kts index fcf4a710..2e5ee74f 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -5,7 +5,7 @@ pluginManagement { gradlePluginPortal() } - val toolsVersion = "0.10.5" + val toolsVersion = "0.10.7" plugins { id("ru.mipt.npm.gradle.project") version toolsVersion