From b916a038f70a9a4c20753513924d2677ee1c9896 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Sun, 10 Jan 2021 18:18:34 +0300 Subject: [PATCH] WIP full data refactor --- .../kotlin/hep/dataforge/data/Data.kt | 2 +- .../kotlin/hep/dataforge/data/DataTree.kt | 8 +++++--- .../hep/dataforge/data/MutableDataTree.kt | 3 --- .../hep/dataforge/data/dataSetOperations.kt | 12 +++++------ .../hep/dataforge/data/DataTreeBuilderTest.kt | 4 ++-- .../hep/dataforge/workspace/TaskBuilder.kt | 2 +- .../dataforge/workspace/WorkspaceBuilder.kt | 20 +++++-------------- .../hep/dataforge/workspace/tasksJVM.kt | 9 --------- .../workspace/DataPropagationTest.kt | 2 +- .../hep/dataforge/workspace/FileDataTest.kt | 12 +++++++---- .../workspace/SimpleWorkspaceTest.kt | 17 ++++++---------- 11 files changed, 35 insertions(+), 56 deletions(-) delete mode 100644 dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/tasksJVM.kt diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/Data.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/Data.kt index 259c69e3..954df54d 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/Data.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/Data.kt @@ -44,7 +44,7 @@ public interface Data : Goal, MetaRepr { /** * An empty data containing only meta */ - public fun empty(meta: Meta): Data = object : Data { + public fun empty(meta: Meta): Data = object : Data { override val type: KClass = Nothing::class override val meta: Meta = meta override val dependencies: Collection> = emptyList() diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataTree.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataTree.kt index 5f6e198d..939066a9 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataTree.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataTree.kt @@ -67,6 +67,8 @@ public interface DataTree : DataSet { } } +public suspend fun DataSet.getData(name: String): Data? = getData(name.toName()) + /** * Get a [DataTreeItem] with given [name] or null if the item does not exist */ @@ -97,10 +99,10 @@ public fun DataTree.itemFlow(): Flow>> = * Get a branch of this [DataTree] with a given [branchName]. * The difference from similar method for [DataSet] is that internal logic is more simple and the return value is a [DataTree] */ -public operator fun DataTree.get(branchName: Name): DataTree = object : DataTree { - override val dataType: KClass get() = this@get.dataType +public fun DataTree.branch(branchName: Name): DataTree = object : DataTree { + override val dataType: KClass get() = this@branch.dataType - override val updates: Flow = this@get.updates.mapNotNull { it.removeHeadOrNull(branchName) } + override val updates: Flow = this@branch.updates.mapNotNull { it.removeHeadOrNull(branchName) } override suspend fun items(): Map> = getItem(branchName).tree?.items() ?: emptyMap() } diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/MutableDataTree.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/MutableDataTree.kt index d46419c7..863dda43 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/MutableDataTree.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/MutableDataTree.kt @@ -154,6 +154,3 @@ public suspend fun DataSet.toMutableTree( set(it, getData(it)) }.launchIn(scope) } - -public fun MutableDataTree.get(branchName: Name): MutableDataTree = - (this as DataTree).get(branchName) as MutableDataTree diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/dataSetOperations.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/dataSetOperations.kt index 24846c30..9db5f963 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/dataSetOperations.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/dataSetOperations.kt @@ -51,22 +51,22 @@ else object : DataSet { /** * Get a subset of data starting with a given [branchName] */ -public operator fun DataSet.get(branchName: Name): DataSet = if (branchName.isEmpty()) this +public fun DataSet.branch(branchName: Name): DataSet = if (branchName.isEmpty()) this else object : DataSet { - override val dataType: KClass get() = this@get.dataType + override val dataType: KClass get() = this@branch.dataType - override fun flow(): Flow> = this@get.flow().mapNotNull { + override fun flow(): Flow> = this@branch.flow().mapNotNull { it.name.removeHeadOrNull(branchName)?.let { name -> it.data.named(name) } } - override suspend fun getData(name: Name): Data? = this@get.getData(branchName + name) + override suspend fun getData(name: Name): Data? = this@branch.getData(branchName + name) - override val updates: Flow get() = this@get.updates.mapNotNull { it.removeHeadOrNull(branchName) } + override val updates: Flow get() = this@branch.updates.mapNotNull { it.removeHeadOrNull(branchName) } } -public operator fun DataSet.get(branchName: String): DataSet = this@get.get(branchName.toName()) +public fun DataSet.branch(branchName: String): DataSet = this@branch.branch(branchName.toName()) @DFExperimental public suspend fun DataSet.rootData(): Data? = getData(Name.EMPTY) diff --git a/dataforge-data/src/jvmTest/kotlin/hep/dataforge/data/DataTreeBuilderTest.kt b/dataforge-data/src/jvmTest/kotlin/hep/dataforge/data/DataTreeBuilderTest.kt index 5fbeebe5..f93d6ee7 100644 --- a/dataforge-data/src/jvmTest/kotlin/hep/dataforge/data/DataTreeBuilderTest.kt +++ b/dataforge-data/src/jvmTest/kotlin/hep/dataforge/data/DataTreeBuilderTest.kt @@ -27,8 +27,8 @@ internal class DataTreeBuilderTest{ } - assertTrue { node["update.a"] != null } - assertTrue { node["primary.a"] != null } + assertTrue { node.branch("update.a") != null } + assertTrue { node.branch("primary.a") != null } } } \ No newline at end of file diff --git a/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/TaskBuilder.kt b/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/TaskBuilder.kt index a34ca581..f41cfa49 100644 --- a/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/TaskBuilder.kt +++ b/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/TaskBuilder.kt @@ -57,7 +57,7 @@ public class TaskBuilder(private val name: Name, public val type: KClas ) { dataTransforms += { context, model, data -> val env = TaskEnv(Name.EMPTY, model.meta, context, data) - val startData = data.get(from) + val startData = data.branch(from) env.block(startData).withNamePrefix(to) } } diff --git a/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/WorkspaceBuilder.kt b/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/WorkspaceBuilder.kt index b78c76ab..83be2f63 100644 --- a/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/WorkspaceBuilder.kt +++ b/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/WorkspaceBuilder.kt @@ -3,10 +3,8 @@ package hep.dataforge.workspace import hep.dataforge.context.Context import hep.dataforge.context.ContextBuilder import hep.dataforge.context.Global -import hep.dataforge.data.DataTree import hep.dataforge.data.MutableDataTree import hep.dataforge.meta.* -import hep.dataforge.names.Name import hep.dataforge.names.toName import kotlin.reflect.KClass @@ -28,19 +26,11 @@ public fun WorkspaceBuilder.context(name: String = "WORKSPACE", block: ContextBu context = ContextBuilder(parentContext, name).apply(block).build() } -public inline fun WorkspaceBuilder.data( - name: Name = Name.EMPTY, - noinline block: MutableDataTree.() -> Unit, -): DataTree { - TODO() - //data.branch(name).apply(block) -} - -@JvmName("rawData") -public fun WorkspaceBuilder.data( - name: Name = Name.EMPTY, +public inline fun WorkspaceBuilder.data( block: MutableDataTree.() -> Unit, -): DataTree = data(name, block) +): Unit{ + data.apply(block) +} public fun WorkspaceBuilder.target(name: String, block: MetaBuilder.() -> Unit) { @@ -80,7 +70,7 @@ public fun WorkspaceBuilder.task( */ public class SimpleWorkspaceBuilder(override val parentContext: Context) : WorkspaceBuilder { override var context: Context = parentContext - override var data: MutableDataTree = MutableDataTree(Any::class,context) + override var data: MutableDataTree = MutableDataTree(Any::class, context) override var tasks: MutableSet> = HashSet() override var targets: MutableMap = HashMap() diff --git a/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/tasksJVM.kt b/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/tasksJVM.kt deleted file mode 100644 index 6d4b29bd..00000000 --- a/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/tasksJVM.kt +++ /dev/null @@ -1,9 +0,0 @@ -package hep.dataforge.workspace - -import hep.dataforge.data.DataSet -import hep.dataforge.meta.MetaBuilder -import kotlinx.coroutines.runBlocking - -public fun Workspace.runBlocking(task: String, block: MetaBuilder.() -> Unit = {}): DataSet = runBlocking{ - run(task, block) -} \ No newline at end of file diff --git a/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/DataPropagationTest.kt b/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/DataPropagationTest.kt index 9a531731..323b44de 100644 --- a/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/DataPropagationTest.kt +++ b/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/DataPropagationTest.kt @@ -13,7 +13,7 @@ import kotlin.reflect.KClass import kotlin.test.Test import kotlin.test.assertEquals -public fun DataSet.first(): NamedData? = runBlocking { flow().firstOrNull() } +fun DataSet.first(): NamedData? = runBlocking { flow().firstOrNull() } class DataPropagationTestPlugin : WorkspacePlugin() { override val tag: PluginTag = Companion.tag diff --git a/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/FileDataTest.kt b/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/FileDataTest.kt index a2a029d5..b5a7c445 100644 --- a/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/FileDataTest.kt +++ b/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/FileDataTest.kt @@ -67,8 +67,10 @@ class FileDataTest { } println(dir.toUri().toString()) val reconstructed = readDataDirectory(dir,StringFormatResolver) - assertEquals(dataNode["dir.a"].data?.meta, reconstructed["dir.a"].data?.meta) - assertEquals(dataNode["b"]?.data?.value(), reconstructed["b"]?.data?.value()) + runBlocking { + assertEquals(dataNode.getData("dir.a")?.meta, reconstructed.getData("dir.a")?.meta) + assertEquals(dataNode.getData("b")?.value(), reconstructed.getData("b")?.value()) + } } } @@ -83,8 +85,10 @@ class FileDataTest { } println(zip.toUri().toString()) val reconstructed = readDataDirectory(zip, StringFormatResolver) - assertEquals(dataNode["dir.a"].data?.meta, reconstructed["dir.a"].data?.meta) - assertEquals(dataNode["b"]?.data?.value(), reconstructed["b"]?.data?.value()) + runBlocking { + assertEquals(dataNode.getData("dir.a")?.meta, reconstructed.getData("dir.a")?.meta) + assertEquals(dataNode.getData("b")?.value(), reconstructed.getData("b")?.value()) + } } } } \ No newline at end of file diff --git a/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/SimpleWorkspaceTest.kt b/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/SimpleWorkspaceTest.kt index 680b0016..c0379594 100644 --- a/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/SimpleWorkspaceTest.kt +++ b/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/SimpleWorkspaceTest.kt @@ -6,6 +6,7 @@ import hep.dataforge.meta.* import hep.dataforge.names.plus import kotlinx.coroutines.flow.collect import kotlinx.coroutines.runBlocking +import org.junit.jupiter.api.Timeout import kotlin.reflect.KClass import kotlin.test.Test import kotlin.test.assertEquals @@ -21,15 +22,8 @@ public inline fun P.toFactory(): PluginFactory

= object override val type: KClass = P::class } -public fun DataTree<*>.toMeta(): Meta = Meta { - "type" put (dataType.simpleName ?: "undefined") - "items" put { - runBlocking { - flow().collect { - it.name.toString() put it.data.meta - } - } - } +public fun Workspace.runBlocking(task: String, block: MetaBuilder.() -> Unit = {}): DataSet = runBlocking{ + run(task, block) } @@ -89,8 +83,8 @@ class SimpleWorkspaceTest { val linearDep = dependsOn(linear, placement = DataPlacement.into("linear")) } transform { data -> - val squareNode = data["square"].filterIsInstance() //squareDep() - val linearNode = data["linear"].filterIsInstance() //linearDep() + val squareNode = data.branch("square").filterIsInstance() //squareDep() + val linearNode = data.branch("linear").filterIsInstance() //linearDep() DataTree.dynamic(context) { squareNode.flow().collect { val newData: Data = Data { @@ -163,6 +157,7 @@ class SimpleWorkspaceTest { } @Test + @Timeout(400) fun testMetaPropagation() { val node = workspace.runBlocking("sum") { "testFlag" put true } val res = node.first()?.value()