From 65633bbd0d915d104fe2afeed1c8539dbd556c7a Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Wed, 7 Aug 2019 21:14:20 +0300 Subject: [PATCH] Build passes, but a lot of tests fails. Need more work. --- .../hep/dataforge/data/{tasks.kt => goals.kt} | 16 +++++++++------- .../commonMain/kotlin/hep/dataforge/meta/Meta.kt | 7 ++++++- .../kotlin/hep/dataforge/workspace/TaskModel.kt | 3 ++- .../kotlin/hep/dataforge/workspace/Workspace.kt | 5 +++-- .../hep/dataforge/workspace/WorkspaceBuilder.kt | 14 ++++++-------- .../hep/dataforge/workspace/TaskBuilder.kt | 2 +- .../kotlin/hep/dataforge/workspace/fileData.kt | 4 ++-- .../dataforge/workspace/SimpleWorkspaceTest.kt | 4 ++-- 8 files changed, 31 insertions(+), 24 deletions(-) rename dataforge-data/src/commonMain/kotlin/hep/dataforge/data/{tasks.kt => goals.kt} (79%) diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/tasks.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/goals.kt similarity index 79% rename from dataforge-data/src/commonMain/kotlin/hep/dataforge/data/tasks.kt rename to dataforge-data/src/commonMain/kotlin/hep/dataforge/data/goals.kt index c46a02bc..f4f200f0 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/tasks.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/goals.kt @@ -9,11 +9,14 @@ import kotlin.coroutines.EmptyCoroutineContext * * **Important:** Unlike regular deferred, the [Deferred] is started lazily, so the actual calculation is called only when result is requested. */ -fun CoroutineScope.task( - context: CoroutineContext, +fun goal( + context: CoroutineContext = EmptyCoroutineContext, dependencies: Collection = emptyList(), block: suspend CoroutineScope.() -> T -): Deferred = async(context + CoroutineMonitor() + Dependencies(dependencies), start = CoroutineStart.LAZY) { +): Deferred = CoroutineScope(context).async( + CoroutineMonitor() + Dependencies(dependencies), + start = CoroutineStart.LAZY +) { dependencies.forEach { job -> job.start() job.invokeOnCompletion { error -> @@ -29,7 +32,7 @@ fun CoroutineScope.task( fun Deferred.pipe( context: CoroutineContext = EmptyCoroutineContext, block: suspend CoroutineScope.(T) -> R -): Deferred = CoroutineScope(this + context).task(context, listOf(this)) { +): Deferred = goal(this + context,listOf(this)) { block(await()) } @@ -38,10 +41,9 @@ fun Deferred.pipe( * @param scope the scope for resulting goal. By default use first goal in list */ fun Collection>.join( - scope: CoroutineScope, context: CoroutineContext = EmptyCoroutineContext, block: suspend CoroutineScope.(Collection) -> R -): Deferred = scope.task(context, this) { +): Deferred = goal(context, this) { block(map { it.await() }) } @@ -54,7 +56,7 @@ fun Collection>.join( fun Map>.join( context: CoroutineContext = EmptyCoroutineContext, block: suspend CoroutineScope.(Map) -> R -): Deferred = CoroutineScope(values.first() + context).task(context, this.values) { +): Deferred = goal(context, this.values) { block(mapValues { it.value.await() }) } diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Meta.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Meta.kt index 1f918da3..9e0dedf9 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Meta.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Meta.kt @@ -186,7 +186,12 @@ operator fun > MetaNode?.get(key: NameToken): MetaItem? = abstract class MetaBase: Meta{ override fun equals(other: Any?): Boolean = if(other is Meta) { - items == other.items + this.items == other.items +// val items = items +// val otherItems = other.items +// (items.keys == otherItems.keys) && items.keys.all { +// items[it] == otherItems[it] +// } } else { false } diff --git a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskModel.kt b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskModel.kt index 534f88f0..b6c4b2b6 100644 --- a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskModel.kt +++ b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskModel.kt @@ -8,6 +8,7 @@ package hep.dataforge.workspace import hep.dataforge.data.DataFilter import hep.dataforge.data.DataTree import hep.dataforge.data.DataTreeBuilder +import hep.dataforge.data.dataSequence import hep.dataforge.meta.* import hep.dataforge.names.EmptyName import hep.dataforge.names.Name @@ -51,7 +52,7 @@ data class TaskModel( */ fun TaskModel.buildInput(workspace: Workspace): DataTree { return DataTreeBuilder(Any::class).apply { - dependencies.asSequence().flatMap { it.apply(workspace).data }.forEach { (name, data) -> + dependencies.asSequence().flatMap { it.apply(workspace).dataSequence() }.forEach { (name, data) -> //TODO add concise error on replacement this[name] = data } diff --git a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/Workspace.kt b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/Workspace.kt index 169e1cc0..f5f0f3a6 100644 --- a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/Workspace.kt +++ b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/Workspace.kt @@ -3,6 +3,7 @@ package hep.dataforge.workspace import hep.dataforge.context.ContextAware import hep.dataforge.data.Data import hep.dataforge.data.DataNode +import hep.dataforge.data.dataSequence import hep.dataforge.meta.Meta import hep.dataforge.meta.MetaBuilder import hep.dataforge.meta.buildMeta @@ -33,8 +34,8 @@ interface Workspace : ContextAware, Provider { return when (target) { "target", Meta.TYPE -> targets.mapKeys { it.key.toName() } Task.TYPE -> tasks - Data.TYPE -> data.data.toMap() - DataNode.TYPE -> data.nodes.toMap() + Data.TYPE -> data.dataSequence().toMap() + //DataNode.TYPE -> data.nodes.toMap() else -> emptyMap() } } diff --git a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/WorkspaceBuilder.kt b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/WorkspaceBuilder.kt index b8df2fef..999ee50c 100644 --- a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/WorkspaceBuilder.kt +++ b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/WorkspaceBuilder.kt @@ -8,8 +8,6 @@ import hep.dataforge.data.DataTreeBuilder import hep.dataforge.meta.* import hep.dataforge.names.Name import hep.dataforge.names.toName -import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.GlobalScope @TaskBuildScope interface WorkspaceBuilder { @@ -36,14 +34,14 @@ fun WorkspaceBuilder.data(name: Name, data: Data) { fun WorkspaceBuilder.data(name: String, data: Data) = data(name.toName(), data) -fun WorkspaceBuilder.static(name: Name, data: Any, scope: CoroutineScope = GlobalScope, meta: Meta = EmptyMeta) = - data(name, Data.static(scope, data, meta)) +fun WorkspaceBuilder.static(name: Name, data: Any, meta: Meta = EmptyMeta) = + data(name, Data.static(data, meta)) -fun WorkspaceBuilder.static(name: Name, data: Any, scope: CoroutineScope = GlobalScope, block: MetaBuilder.() -> Unit = {}) = - data(name, Data.static(scope, data, buildMeta(block))) +fun WorkspaceBuilder.static(name: Name, data: Any, block: MetaBuilder.() -> Unit = {}) = + data(name, Data.static(data, buildMeta(block))) -fun WorkspaceBuilder.static(name: String, data: Any, scope: CoroutineScope = GlobalScope, block: MetaBuilder.() -> Unit = {}) = - data(name, Data.static(scope, data, buildMeta(block))) +fun WorkspaceBuilder.static(name: String, data: Any, block: MetaBuilder.() -> Unit = {}) = + data(name, Data.static(data, buildMeta(block))) fun WorkspaceBuilder.data(name: Name, node: DataNode) { this.data[name] = node diff --git a/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/TaskBuilder.kt b/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/TaskBuilder.kt index d2e1b864..175c4d87 100644 --- a/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/TaskBuilder.kt +++ b/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/TaskBuilder.kt @@ -27,7 +27,7 @@ class TaskBuilder(val name: String) { val localData = if (from.isEmpty()) { node } else { - node.getNode(from.toName()) ?: return null + node[from.toName()].node ?: return null } return transform(workspace.context, model, localData) } diff --git a/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/fileData.kt b/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/fileData.kt index fa8f6ddf..62fc97bd 100644 --- a/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/fileData.kt +++ b/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/fileData.kt @@ -2,7 +2,7 @@ package hep.dataforge.workspace import hep.dataforge.context.Context import hep.dataforge.data.Data -import hep.dataforge.data.task +import hep.dataforge.data.goal import hep.dataforge.descriptors.NodeDescriptor import hep.dataforge.io.IOFormat import hep.dataforge.io.JsonMetaFormat @@ -58,7 +58,7 @@ suspend fun Context.readData( } else { null } - val goal = task { + val goal = goal { withContext(Dispatchers.IO) { format.run { Files.newByteChannel(path, StandardOpenOption.READ) diff --git a/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/SimpleWorkspaceTest.kt b/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/SimpleWorkspaceTest.kt index d47f972f..51471525 100644 --- a/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/SimpleWorkspaceTest.kt +++ b/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/SimpleWorkspaceTest.kt @@ -92,13 +92,13 @@ class SimpleWorkspaceTest { fun testWorkspace() { val node = workspace.run("sum") val res = node.first() - assertEquals(328350, res.get()) + assertEquals(328350, res?.get()) } @Test fun testMetaPropagation() { val node = workspace.run("sum") { "testFlag" to true } - val res = node.first().get() + val res = node.first()?.get() } @Test