From b07d281a83c9bbd3731e30c0801405e3812f64d9 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Wed, 29 Sep 2021 11:08:55 +0300 Subject: [PATCH 1/4] Kotlin 1.6 --- build.gradle.kts | 6 +++--- .../kotlin/space/kscience/dataforge/data/Goal.kt | 2 -- .../space/kscience/dataforge/meta/Scheme.kt | 1 - .../space/kscience/dataforge/values/Value.kt | 4 +--- .../dataforge/meta/MetaSerializationTest.kt | 16 ++++++++++++++++ gradle/wrapper/gradle-wrapper.properties | 2 +- settings.gradle.kts | 2 +- 7 files changed, 22 insertions(+), 11 deletions(-) create mode 100644 dataforge-meta/src/commonTest/kotlin/space/kscience/dataforge/meta/MetaSerializationTest.kt diff --git a/build.gradle.kts b/build.gradle.kts index d49db3c4..fe2e7f58 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -5,6 +5,9 @@ plugins { allprojects { group = "space.kscience" version = "0.5.1" + repositories{ + mavenCentral() + } } subprojects { @@ -22,8 +25,5 @@ ksciencePublish { } apiValidation { - if(project.version.toString().contains("dev")) { - validationDisabled = true - } nonPublicMarkers.add("space.kscience.dataforge.misc.DFExperimental") } \ No newline at end of file diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/Goal.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/Goal.kt index e41c124a..b7ddea96 100644 --- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/Goal.kt +++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/Goal.kt @@ -1,7 +1,6 @@ package space.kscience.dataforge.data import kotlinx.coroutines.* -import space.kscience.dataforge.misc.DFExperimental import kotlin.coroutines.CoroutineContext import kotlin.coroutines.EmptyCoroutineContext @@ -67,7 +66,6 @@ public open class LazyGoal( * If [GoalExecutionRestriction] is present in the [coroutineScope] context, the call could produce a error a warning * depending on the settings. */ - @DFExperimental override fun async(coroutineScope: CoroutineScope): Deferred { val log = coroutineScope.coroutineContext[GoalLogger] // Check if context restricts goal computation diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/Scheme.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/Scheme.kt index 2bc0e234..ffc29152 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/Scheme.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/Scheme.kt @@ -123,7 +123,6 @@ public open class Scheme : Described, MetaRepr, MutableMetaProvider, Configurabl TODO("Not yet implemented") } - } } diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/values/Value.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/values/Value.kt index e2a83b9c..707c86f6 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/values/Value.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/values/Value.kt @@ -166,9 +166,7 @@ public class EnumValue>(override val value: E) : Value { override fun toString(): String = value.toString() - override fun equals(other: Any?): Boolean { - return string == (other as? Value)?.string - } + override fun equals(other: Any?): Boolean = string == (other as? Value)?.string override fun hashCode(): Int = value.hashCode() } diff --git a/dataforge-meta/src/commonTest/kotlin/space/kscience/dataforge/meta/MetaSerializationTest.kt b/dataforge-meta/src/commonTest/kotlin/space/kscience/dataforge/meta/MetaSerializationTest.kt new file mode 100644 index 00000000..10e76f90 --- /dev/null +++ b/dataforge-meta/src/commonTest/kotlin/space/kscience/dataforge/meta/MetaSerializationTest.kt @@ -0,0 +1,16 @@ +package space.kscience.dataforge.meta + +import kotlinx.serialization.json.Json +import space.kscience.dataforge.values.string +import kotlin.test.Test +import kotlin.test.assertEquals + +class MetaSerializationTest { + + @Test + fun singleValueDeserialization(){ + val string = "ddd" + val meta = Json.decodeFromString(MetaSerializer, string) + assertEquals(string, meta.value?.string) + } +} \ No newline at end of file diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index f371643e..ffed3a25 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.0-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-7.2-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/settings.gradle.kts b/settings.gradle.kts index 6d8e56a0..26f6fcc4 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -5,7 +5,7 @@ pluginManagement { gradlePluginPortal() } - val toolsVersion = "0.10.2" + val toolsVersion = "0.10.4" plugins { id("ru.mipt.npm.gradle.project") version toolsVersion From 00d964eef3bbf3ef00df9c1463f646bc90cf4ab7 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Fri, 8 Oct 2021 09:54:11 +0300 Subject: [PATCH 2/4] Workspace and task updates --- .../dataforge/workspace/SimpleWorkspace.kt | 2 +- .../kscience/dataforge/workspace/Task.kt | 10 +++++--- .../kscience/dataforge/workspace/TaskData.kt | 2 +- .../dataforge/workspace/TaskResult.kt | 11 +++++---- .../kscience/dataforge/workspace/Workspace.kt | 5 ++++ .../dataforge/workspace/WorkspaceBuilder.kt | 13 +++++++++- .../dataforge/workspace/taskBuilders.kt | 9 +++++++ .../dataforge/workspace/taskBuilders.kt | 24 ------------------- .../workspace/workspaceExtensions.kt | 11 ++++++++- 9 files changed, 52 insertions(+), 35 deletions(-) create mode 100644 dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/taskBuilders.kt delete mode 100644 dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/taskBuilders.kt diff --git a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/SimpleWorkspace.kt b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/SimpleWorkspace.kt index 7e881a8f..e800d8b3 100644 --- a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/SimpleWorkspace.kt +++ b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/SimpleWorkspace.kt @@ -17,7 +17,7 @@ public class SimpleWorkspace( private val externalTasks: Map>, ) : Workspace { - override val data: TaskResult<*> = internalize(data, Name.EMPTY, Meta.EMPTY) + override val data: TaskResult<*> = wrapResult(data, Name.EMPTY, Meta.EMPTY) override val tasks: Map> get() = context.gather>(Task.TYPE) + externalTasks diff --git a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/Task.kt b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/Task.kt index 965f859e..dcf63db2 100644 --- a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/Task.kt +++ b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/Task.kt @@ -28,7 +28,7 @@ public interface Task : Described { public suspend fun execute(workspace: Workspace, taskName: Name, taskMeta: Meta): TaskResult public companion object { - public const val TYPE: String = "workspace.stage" + public const val TYPE: String = "workspace.task" } } @@ -42,6 +42,10 @@ public class TaskResultBuilder( /** * Create a [Task] that composes a result using [builder]. Only data from the workspace could be used. * Data dependency cycles are not allowed. + * + * @param resultType the type boundary for data produced by this task + * @param descriptor of meta accepted by this task + * @param builder for resulting data set */ @Suppress("FunctionName") @DFInternal @@ -60,9 +64,9 @@ public fun Task( ): TaskResult = withContext(GoalExecutionRestriction() + workspace.goalLogger) { //TODO use safe builder and check for external data on add and detects cycles val dataset = DataTree(resultType) { - TaskResultBuilder(workspace,taskName, taskMeta, this).apply { builder() } + TaskResultBuilder(workspace, taskName, taskMeta, this).apply { builder() } } - workspace.internalize(dataset, taskName, taskMeta) + workspace.wrapResult(dataset, taskName, taskMeta) } } diff --git a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/TaskData.kt b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/TaskData.kt index ff501600..6d14616d 100644 --- a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/TaskData.kt +++ b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/TaskData.kt @@ -42,6 +42,6 @@ private class TaskDataImpl( // } } -internal fun Workspace.internalize(data: Data, name: Name, stage: Name, stageMeta: Meta): TaskData = +internal fun Workspace.wrapResult(data: Data, name: Name, stage: Name, stageMeta: Meta): TaskData = TaskDataImpl(this, data, name, stage, stageMeta) diff --git a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/TaskResult.kt b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/TaskResult.kt index d0097e55..61e4c650 100644 --- a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/TaskResult.kt +++ b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/TaskResult.kt @@ -37,13 +37,16 @@ private class TaskResultImpl( ) : TaskResult, DataSet by dataSet { override fun flow(): Flow> = dataSet.flow().map { - workspace.internalize(it, it.name, taskName, taskMeta) + workspace.wrapResult(it, it.name, taskName, taskMeta) } override suspend fun getData(name: Name): TaskData? = dataSet.getData(name)?.let { - workspace.internalize(it, name, taskName, taskMeta) + workspace.wrapResult(it, name, taskName, taskMeta) } } -internal fun Workspace.internalize(dataSet: DataSet, stage: Name, stageMeta: Meta): TaskResult = - TaskResultImpl(this, dataSet, stage, stageMeta) \ No newline at end of file +/** + * Wrap data into [TaskResult] + */ +public fun Workspace.wrapResult(dataSet: DataSet, taskName: Name, taskMeta: Meta): TaskResult = + TaskResultImpl(this, dataSet, taskName, taskMeta) \ No newline at end of file diff --git a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/Workspace.kt b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/Workspace.kt index e5458258..bc2eb7dd 100644 --- a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/Workspace.kt +++ b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/Workspace.kt @@ -1,6 +1,7 @@ package space.kscience.dataforge.workspace import space.kscience.dataforge.context.ContextAware +import space.kscience.dataforge.data.DataSet import space.kscience.dataforge.meta.Meta import space.kscience.dataforge.meta.MutableMeta import space.kscience.dataforge.misc.Type @@ -8,6 +9,10 @@ import space.kscience.dataforge.names.Name import space.kscience.dataforge.provider.Provider +public interface DataSelector{ + public suspend fun select(workspace: Workspace, meta: Meta): DataSet +} + @Type(Workspace.TYPE) public interface Workspace : ContextAware, Provider { /** diff --git a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/WorkspaceBuilder.kt b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/WorkspaceBuilder.kt index 423a1807..72a35da7 100644 --- a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/WorkspaceBuilder.kt +++ b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/WorkspaceBuilder.kt @@ -17,7 +17,18 @@ import space.kscience.dataforge.names.Name import kotlin.properties.PropertyDelegateProvider import kotlin.properties.ReadOnlyProperty -public data class TaskReference(public val taskName: Name, public val task: Task) +public data class TaskReference(public val taskName: Name, public val task: Task): DataSelector{ + + @Suppress("UNCHECKED_CAST") + override suspend fun select(workspace: Workspace, meta: Meta): DataSet { + if (workspace.tasks[taskName] == task) { + return workspace.produce(taskName, meta) as TaskResult + } else { + error("Task $taskName does not belong to the workspace") + } + } + +} public interface TaskContainer { public fun registerTask(taskName: Name, task: Task<*>) diff --git a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/taskBuilders.kt b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/taskBuilders.kt new file mode 100644 index 00000000..bcfeae32 --- /dev/null +++ b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/taskBuilders.kt @@ -0,0 +1,9 @@ +package space.kscience.dataforge.workspace + +import space.kscience.dataforge.data.DataSet +import space.kscience.dataforge.meta.Meta + +public suspend fun TaskResultBuilder<*>.from( + selector: DataSelector, + meta: Meta = Meta.EMPTY, +): DataSet = selector.select(workspace, meta) diff --git a/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/taskBuilders.kt b/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/taskBuilders.kt deleted file mode 100644 index d5ab5313..00000000 --- a/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/taskBuilders.kt +++ /dev/null @@ -1,24 +0,0 @@ -package space.kscience.dataforge.workspace - -import space.kscience.dataforge.data.DataSet -import space.kscience.dataforge.data.select -import space.kscience.dataforge.meta.Meta -import space.kscience.dataforge.names.Name - -public suspend inline fun TaskResultBuilder.from( - task: Name, - taskMeta: Meta = Meta.EMPTY, -): DataSet = workspace.produce(task, taskMeta).select() - - -@Suppress("UNCHECKED_CAST") -public suspend fun TaskResultBuilder<*>.from( - reference: TaskReference, - taskMeta: Meta = Meta.EMPTY, -): DataSet { - if (workspace.tasks[reference.taskName] == reference.task) { - return workspace.produce(reference.taskName, taskMeta) as TaskResult - } else { - throw error("Task ${reference.taskName} does not belong to the workspace") - } -} diff --git a/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/workspaceExtensions.kt b/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/workspaceExtensions.kt index 543b0da1..2690a8f6 100644 --- a/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/workspaceExtensions.kt +++ b/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/workspaceExtensions.kt @@ -1,8 +1,17 @@ package space.kscience.dataforge.workspace import kotlinx.coroutines.runBlocking +import space.kscience.dataforge.data.DataSet import space.kscience.dataforge.data.DataSetBuilder +import space.kscience.dataforge.data.select +import space.kscience.dataforge.meta.Meta +import space.kscience.dataforge.names.Name public fun WorkspaceBuilder.data(builder: suspend DataSetBuilder.() -> Unit): Unit = runBlocking { buildData(builder) -} \ No newline at end of file +} + +public suspend inline fun TaskResultBuilder<*>.from( + task: Name, + taskMeta: Meta = Meta.EMPTY, +): DataSet = workspace.produce(task, taskMeta).select() \ No newline at end of file From aded38254e647d7cb9ffad6acea653060007ce46 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Sat, 9 Oct 2021 10:49:38 +0300 Subject: [PATCH 3/4] Workspace and task updates --- .../space/kscience/dataforge/data/select.kt | 10 +++-- .../kscience/dataforge/workspace/TaskData.kt | 4 +- .../dataforge/workspace/TaskResult.kt | 4 +- .../dataforge/workspace/taskBuilders.kt | 40 ++++++++++++++++++- ...workspaceExtensions.kt => workspaceJvm.kt} | 4 ++ .../workspace/SimpleWorkspaceTest.kt | 27 +++++++++---- 6 files changed, 71 insertions(+), 18 deletions(-) rename dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/{workspaceExtensions.kt => workspaceJvm.kt} (63%) diff --git a/dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/select.kt b/dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/select.kt index 7d311159..2cbbc1c1 100644 --- a/dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/select.kt +++ b/dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/select.kt @@ -16,8 +16,12 @@ import kotlin.reflect.typeOf */ @Suppress("UNCHECKED_CAST") private fun Data<*>.castOrNull(type: KType): Data? = - if (!this.type.isSubtypeOf(type)) null else object : Data by (this as Data) { - override val type: KType = type + if (!this.type.isSubtypeOf(type)) { + null + } else { + object : Data by (this as Data) { + override val type: KType = type + } } /** @@ -31,7 +35,6 @@ internal fun DataSet<*>.select( ): ActiveDataSet = object : ActiveDataSet { override val dataType = type - override fun flow(): Flow> = this@select.flow().filter { datum -> datum.type.isSubtypeOf(type) && (namePattern == null || datum.name.matches(namePattern)) }.map { @@ -45,7 +48,6 @@ internal fun DataSet<*>.select( val datum = this@select.getData(it) datum?.type?.isSubtypeOf(type) ?: false } - } /** diff --git a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/TaskData.kt b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/TaskData.kt index 6d14616d..bd2bdb48 100644 --- a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/TaskData.kt +++ b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/TaskData.kt @@ -42,6 +42,6 @@ private class TaskDataImpl( // } } -internal fun Workspace.wrapResult(data: Data, name: Name, stage: Name, stageMeta: Meta): TaskData = - TaskDataImpl(this, data, name, stage, stageMeta) +public fun Workspace.wrapData(data: Data, name: Name, taskName: Name, stageMeta: Meta): TaskData = + TaskDataImpl(this, data, name, taskName, stageMeta) diff --git a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/TaskResult.kt b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/TaskResult.kt index 61e4c650..5a65f219 100644 --- a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/TaskResult.kt +++ b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/TaskResult.kt @@ -37,11 +37,11 @@ private class TaskResultImpl( ) : TaskResult, DataSet by dataSet { override fun flow(): Flow> = dataSet.flow().map { - workspace.wrapResult(it, it.name, taskName, taskMeta) + workspace.wrapData(it, it.name, taskName, taskMeta) } override suspend fun getData(name: Name): TaskData? = dataSet.getData(name)?.let { - workspace.wrapResult(it, name, taskName, taskMeta) + workspace.wrapData(it, name, taskName, taskMeta) } } diff --git a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/taskBuilders.kt b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/taskBuilders.kt index bcfeae32..4744d415 100644 --- a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/taskBuilders.kt +++ b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/taskBuilders.kt @@ -1,9 +1,45 @@ package space.kscience.dataforge.workspace import space.kscience.dataforge.data.DataSet +import space.kscience.dataforge.data.forEach +import space.kscience.dataforge.data.map import space.kscience.dataforge.meta.Meta +import space.kscience.dataforge.meta.toMutableMeta +import space.kscience.dataforge.misc.DFExperimental +import space.kscience.dataforge.names.Name +/** + * Select data using given [selector] + */ public suspend fun TaskResultBuilder<*>.from( selector: DataSelector, - meta: Meta = Meta.EMPTY, -): DataSet = selector.select(workspace, meta) +): DataSet = selector.select(workspace, taskMeta) + +public val TaskResultBuilder<*>.allData: DataSelector<*> + get() = object : DataSelector { + override suspend fun select(workspace: Workspace, meta: Meta): DataSet = workspace.data + } + +/** + * Perform a lazy mapping task using given [selector] and [action]. The meta of resulting + * TODO move selector to receiver with multi-receivers + */ +@DFExperimental +public suspend inline fun TaskResultBuilder.pipeFrom( + selector: DataSelector, + crossinline action: suspend (arg: T, name: Name, meta: Meta) -> R +) { + from(selector).forEach { data -> + val meta = data.meta.toMutableMeta().apply { + taskName put taskMeta + } + + val res = data.map(workspace.context.coroutineContext, meta) { + action(it, data.name, meta) + } + + emit(data.name, res) + } +} + + diff --git a/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/workspaceExtensions.kt b/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/workspaceJvm.kt similarity index 63% rename from dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/workspaceExtensions.kt rename to dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/workspaceJvm.kt index 2690a8f6..61e1dbec 100644 --- a/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/workspaceExtensions.kt +++ b/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/workspaceJvm.kt @@ -11,6 +11,10 @@ public fun WorkspaceBuilder.data(builder: suspend DataSetBuilder.() -> Unit buildData(builder) } +public inline fun TaskResultBuilder<*>.selectData(namePattern: Name? = null): DataSelector = object : DataSelector { + override suspend fun select(workspace: Workspace, meta: Meta): DataSet = workspace.data.select(namePattern) +} + public suspend inline fun TaskResultBuilder<*>.from( task: Name, taskMeta: Meta = Meta.EMPTY, diff --git a/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/SimpleWorkspaceTest.kt b/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/SimpleWorkspaceTest.kt index e5657444..edbe3a4b 100644 --- a/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/SimpleWorkspaceTest.kt +++ b/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/SimpleWorkspaceTest.kt @@ -71,21 +71,32 @@ class SimpleWorkspaceTest { } val square by task { - workspace.data.select().forEach { data -> - if (data.meta["testFlag"].boolean == true) { + pipeFrom(selectData()) { arg, name, meta -> + if (meta["testFlag"].boolean == true) { println("flag") } - val value = data.await() - workspace.logger.info { "Starting square on $value" } - emit(data.name, data.map { it * it }) + workspace.logger.info { "Starting square on $name" } + arg * arg } +// workspace.data.select().forEach { data -> +// if (data.meta["testFlag"].boolean == true) { +// println("flag") +// } +// val value = data.await() +// workspace.logger.info { "Starting square on $value" } +// emit(data.name, data.map { it * it }) +// } } val linear by task { - workspace.data.select().forEach { data -> - workspace.logger.info { "Starting linear on $data" } - emit(data.name, data.data.map { it * 2 + 1 }) + pipeFrom(selectData()) { arg, name, _ -> + workspace.logger.info { "Starting linear on $name" } + arg * 2 + 1 } +// workspace.data.select().forEach { data -> +// workspace.logger.info { "Starting linear on $data" } +// emit(data.name, data.data.map { it * 2 + 1 }) +// } } val fullSquare by task { From 3f54eee578c9f85d36a713b0e603d78d0fa52fd5 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Sun, 10 Oct 2021 14:32:32 +0300 Subject: [PATCH 4/4] Workaround for https://youtrack.jetbrains.com/issue/KT-48988. Smart building of child contexts --- build.gradle.kts | 2 +- .../kscience/dataforge/context/Context.kt | 17 ++++----- .../dataforge/context/ContextBuilder.kt | 14 ++++---- .../kscience/dataforge/context/Global.kt | 6 ++-- .../kscience/dataforge/context/LogManager.kt | 2 +- .../dataforge/context/PluginManager.kt | 3 +- .../kscience/dataforge/context/ContextTest.kt | 3 +- .../kscience/dataforge/context/loggingJs.kt | 2 +- .../kscience/dataforge/context/loggingJvm.kt | 2 +- .../dataforge/context/loggingNative.kt | 2 +- .../space/kscience/dataforge/data/DataSet.kt | 4 +-- .../space/kscience/dataforge/data/select.kt | 36 +++++++++++++------ .../space/kscience/dataforge/io/IOPlugin.kt | 18 +++++----- .../dataforge/io/TaggedEnvelopeFormat.kt | 2 +- .../dataforge/io/TaglessEnvelopeFormat.kt | 6 ++-- .../dataforge/scripting/BuildersKtTest.kt | 4 +-- .../dataforge/workspace/WorkspaceBuilder.kt | 3 +- gradle.properties | 4 +-- 18 files changed, 72 insertions(+), 58 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index fe2e7f58..42505cb9 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -4,7 +4,7 @@ plugins { allprojects { group = "space.kscience" - version = "0.5.1" + version = "0.5.2-dev-1" repositories{ mavenCentral() } diff --git a/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/context/Context.kt b/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/context/Context.kt index 97efc76a..427d8611 100644 --- a/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/context/Context.kt +++ b/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/context/Context.kt @@ -4,6 +4,7 @@ import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Job import kotlinx.coroutines.SupervisorJob import space.kscience.dataforge.meta.* +import space.kscience.dataforge.misc.DFExperimental import space.kscience.dataforge.misc.Named import space.kscience.dataforge.names.Name import space.kscience.dataforge.provider.Provider @@ -71,16 +72,16 @@ public open class Context internal constructor( private val childrenContexts = HashMap() /** - * Build and register a child context + * Get and validate existing context or build and register a new child context. + * @param name the relative (tail) name of the new context. If null, uses context hash code as a marker. */ + @OptIn(DFExperimental::class) @Synchronized - public fun buildContext(name: String? = null, block: ContextBuilder.() -> Unit = {}): Context { - val newContext = ContextBuilder(this) - .apply { name?.let { name(it) } } - .apply(block) - .build() - childrenContexts[newContext.name] = newContext - return newContext + public fun buildContext(name: Name? = null, block: ContextBuilder.() -> Unit = {}): Context { + val existing = name?.let { childrenContexts[name] } + return existing?.modify(block)?: ContextBuilder(this, name).apply(block).build().also { + childrenContexts[it.name] = it + } } /** diff --git a/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/context/ContextBuilder.kt b/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/context/ContextBuilder.kt index 0de5fb7a..80329176 100644 --- a/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/context/ContextBuilder.kt +++ b/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/context/ContextBuilder.kt @@ -20,7 +20,7 @@ import kotlin.collections.set @DFBuilder public class ContextBuilder internal constructor( private val parent: Context, - public var name: Name? = null, + public val name: Name? = null, meta: Meta = Meta.EMPTY, ) { internal val factories = HashMap, Meta>() @@ -30,10 +30,6 @@ public class ContextBuilder internal constructor( meta.action() } - public fun name(string: String) { - this.name = Name.parse(string) - } - @OptIn(DFExperimental::class) private fun findPluginFactory(tag: PluginTag): PluginFactory<*> = parent.gatherInSequence>(PluginFactory.TYPE).values @@ -95,19 +91,21 @@ public class ContextBuilder internal constructor( } /** - * Check if current context contains all plugins required by the builder and return it it does or forks to a new context + * Check if current context contains all plugins required by the builder and return it does or forks to a new context * if it does not. */ -public fun Context.withEnv(block: ContextBuilder.() -> Unit): Context { +@DFExperimental +public fun Context.modify(block: ContextBuilder.() -> Unit): Context { fun Context.contains(factory: PluginFactory<*>, meta: Meta): Boolean { val loaded = plugins[factory.tag] ?: return false return loaded.meta == meta } - val builder = ContextBuilder(this, name + "env", properties).apply(block) + val builder = ContextBuilder(this, name + "mod", properties).apply(block) val requiresFork = builder.factories.any { (factory, meta) -> !contains(factory, meta) } || ((properties as Meta) == builder.meta) + return if (requiresFork) builder.build() else this } \ No newline at end of file diff --git a/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/context/Global.kt b/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/context/Global.kt index 9ace5800..05bce836 100644 --- a/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/context/Global.kt +++ b/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/context/Global.kt @@ -3,11 +3,13 @@ package space.kscience.dataforge.context import kotlinx.coroutines.CoroutineName import kotlinx.coroutines.Job import space.kscience.dataforge.meta.Meta +import space.kscience.dataforge.names.Name +import space.kscience.dataforge.names.Name.Companion.parse import space.kscience.dataforge.names.asName import kotlin.coroutines.CoroutineContext import kotlin.native.concurrent.ThreadLocal -internal expect val globalLoggerFactory: PluginFactory +internal expect fun getGlobalLoggerFactory(): PluginFactory /** * A global root context. Closing [Global] terminates the framework. @@ -20,4 +22,4 @@ private object GlobalContext : Context("GLOBAL".asName(), null, emptySet(), Meta public val Global: Context get() = GlobalContext public fun Context(name: String? = null, block: ContextBuilder.() -> Unit = {}): Context = - Global.buildContext(name, block) \ No newline at end of file + Global.buildContext(name?.let(Name::parse), block) \ No newline at end of file diff --git a/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/context/LogManager.kt b/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/context/LogManager.kt index d3efed6f..84a2423d 100644 --- a/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/context/LogManager.kt +++ b/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/context/LogManager.kt @@ -75,7 +75,7 @@ public class DefaultLogManager : AbstractPlugin(), LogManager { */ public val Context.logger: LogManager get() = plugins.find(inherit = true) { it is LogManager } as? LogManager - ?: globalLoggerFactory(context = Global).apply { attach(Global) } + ?: getGlobalLoggerFactory()(context = Global).apply { attach(Global) } /** * The named proxy logger for a context member diff --git a/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/context/PluginManager.kt b/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/context/PluginManager.kt index 81d455c7..d8d7b602 100644 --- a/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/context/PluginManager.kt +++ b/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/context/PluginManager.kt @@ -1,6 +1,7 @@ package space.kscience.dataforge.context import space.kscience.dataforge.meta.Meta +import space.kscience.dataforge.names.plus import kotlin.reflect.KClass @@ -84,7 +85,7 @@ public inline fun Context.fetch(factory: PluginFactory, val existing = plugins[factory] return if (existing != null && existing.meta == meta) existing else { - buildContext { + buildContext(name = this@fetch.name + factory.tag.name) { plugin(factory, meta) }.plugins[factory]!! } diff --git a/dataforge-context/src/commonTest/kotlin/space/kscience/dataforge/context/ContextTest.kt b/dataforge-context/src/commonTest/kotlin/space/kscience/dataforge/context/ContextTest.kt index 8e39eb3f..c00921cd 100644 --- a/dataforge-context/src/commonTest/kotlin/space/kscience/dataforge/context/ContextTest.kt +++ b/dataforge-context/src/commonTest/kotlin/space/kscience/dataforge/context/ContextTest.kt @@ -20,8 +20,7 @@ class ContextTest { @Test fun testPluginManager() { - val context = Global.buildContext { - name("test") + val context = Context("test") { plugin(DummyPlugin()) } val members = context.gather("test") diff --git a/dataforge-context/src/jsMain/kotlin/space/kscience/dataforge/context/loggingJs.kt b/dataforge-context/src/jsMain/kotlin/space/kscience/dataforge/context/loggingJs.kt index 406929ca..0aae274a 100644 --- a/dataforge-context/src/jsMain/kotlin/space/kscience/dataforge/context/loggingJs.kt +++ b/dataforge-context/src/jsMain/kotlin/space/kscience/dataforge/context/loggingJs.kt @@ -29,4 +29,4 @@ public class ConsoleLogManager : AbstractPlugin(), LogManager { } } -internal actual val globalLoggerFactory: PluginFactory = ConsoleLogManager +internal actual fun getGlobalLoggerFactory(): PluginFactory = ConsoleLogManager diff --git a/dataforge-context/src/jvmMain/kotlin/space/kscience/dataforge/context/loggingJvm.kt b/dataforge-context/src/jvmMain/kotlin/space/kscience/dataforge/context/loggingJvm.kt index 05763e00..445e04c9 100644 --- a/dataforge-context/src/jvmMain/kotlin/space/kscience/dataforge/context/loggingJvm.kt +++ b/dataforge-context/src/jvmMain/kotlin/space/kscience/dataforge/context/loggingJvm.kt @@ -31,4 +31,4 @@ public class SlfLogManager : AbstractPlugin(), LogManager { } } -internal actual val globalLoggerFactory: PluginFactory = SlfLogManager +internal actual fun getGlobalLoggerFactory(): PluginFactory = SlfLogManager diff --git a/dataforge-context/src/nativeMain/kotlin/space/kscience/dataforge/context/loggingNative.kt b/dataforge-context/src/nativeMain/kotlin/space/kscience/dataforge/context/loggingNative.kt index 62512132..ff4155f0 100644 --- a/dataforge-context/src/nativeMain/kotlin/space/kscience/dataforge/context/loggingNative.kt +++ b/dataforge-context/src/nativeMain/kotlin/space/kscience/dataforge/context/loggingNative.kt @@ -1,4 +1,4 @@ package space.kscience.dataforge.context -internal actual val globalLoggerFactory: PluginFactory = DefaultLogManager +internal actual fun getGlobalLoggerFactory(): PluginFactory = DefaultLogManager diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSet.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSet.kt index 305f50f8..d4ce7990 100644 --- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSet.kt +++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSet.kt @@ -32,7 +32,7 @@ public interface DataSet { */ public suspend fun listTop(prefix: Name = Name.EMPTY): List = flow().map { it.name }.filter { it.startsWith(prefix) && (it.length == prefix.length + 1) }.toList() - // By default traverses the whole tree. Could be optimized in descendants + // By default, traverses the whole tree. Could be optimized in descendants public companion object { public val META_KEY: Name = "@meta".asName() @@ -43,7 +43,7 @@ public interface DataSet { public val EMPTY: DataSet = object : DataSet { override val dataType: KType = TYPE_OF_NOTHING - private val nothing: Nothing get() = error("this is nothing") + //private val nothing: Nothing get() = error("this is nothing") override fun flow(): Flow> = emptyFlow() diff --git a/dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/select.kt b/dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/select.kt index 2cbbc1c1..5b09468d 100644 --- a/dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/select.kt +++ b/dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/select.kt @@ -3,6 +3,7 @@ package space.kscience.dataforge.data import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.filter import kotlinx.coroutines.flow.map +import space.kscience.dataforge.meta.Meta import space.kscience.dataforge.misc.DFExperimental import space.kscience.dataforge.names.Name import space.kscience.dataforge.names.matches @@ -26,40 +27,55 @@ private fun Data<*>.castOrNull(type: KType): Data? = /** * Select all data matching given type and filters. Does not modify paths + * + * @param namePattern a name match patter according to [Name.matches] + * @param filter addition filtering condition based on item name and meta. By default, accepts all */ @OptIn(DFExperimental::class) -@PublishedApi -internal fun DataSet<*>.select( +public fun DataSet<*>.select( type: KType, namePattern: Name? = null, + filter: (name: Name, meta: Meta) -> Boolean = { _, _ -> true } ): ActiveDataSet = object : ActiveDataSet { override val dataType = type - override fun flow(): Flow> = this@select.flow().filter { datum -> - datum.type.isSubtypeOf(type) && (namePattern == null || datum.name.matches(namePattern)) + private fun checkDatum(name: Name, datum: Data<*>): Boolean = datum.type.isSubtypeOf(type) + && (namePattern == null || name.matches(namePattern)) + && filter(name, datum.meta) + + override fun flow(): Flow> = this@select.flow().filter { + checkDatum(it.name, it.data) }.map { @Suppress("UNCHECKED_CAST") it as NamedData } - override suspend fun getData(name: Name): Data? = this@select.getData(name)?.castOrNull(type) + override suspend fun getData(name: Name): Data? = this@select.getData(name)?.let { datum -> + if (checkDatum(name, datum)) datum.castOrNull(type) else null + } override val updates: Flow = this@select.updates.filter { - val datum = this@select.getData(it) - datum?.type?.isSubtypeOf(type) ?: false + val datum = this@select.getData(it) ?: return@filter false + checkDatum(it, datum) } } /** * Select a single datum of the appropriate type */ -public inline fun DataSet<*>.select(namePattern: Name? = null): DataSet = - select(typeOf(), namePattern) +public inline fun DataSet<*>.select( + namePattern: Name? = null, + noinline filter: (name: Name, meta: Meta) -> Boolean = { _, _ -> true } +): DataSet = select(typeOf(), namePattern, filter) +/** + * Select a single datum if it is present and of given [type] + */ public suspend fun DataSet<*>.selectOne(type: KType, name: Name): NamedData? = getData(name)?.castOrNull(type)?.named(name) -public suspend inline fun DataSet<*>.selectOne(name: Name): NamedData? = selectOne(typeOf(), name) +public suspend inline fun DataSet<*>.selectOne(name: Name): NamedData? = + selectOne(typeOf(), name) public suspend inline fun DataSet<*>.selectOne(name: String): NamedData? = selectOne(typeOf(), Name.parse(name)) \ No newline at end of file diff --git a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/IOPlugin.kt b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/IOPlugin.kt index d7ed301f..66281f19 100644 --- a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/IOPlugin.kt +++ b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/IOPlugin.kt @@ -10,8 +10,7 @@ import space.kscience.dataforge.meta.Meta import space.kscience.dataforge.meta.get import space.kscience.dataforge.meta.string import space.kscience.dataforge.names.Name - -import kotlin.native.concurrent.ThreadLocal +import space.kscience.dataforge.names.asName import kotlin.reflect.KClass public class IOPlugin(meta: Meta) : AbstractPlugin(meta) { @@ -75,10 +74,11 @@ public class IOPlugin(meta: Meta) : AbstractPlugin(meta) { } } -@ThreadLocal -internal val ioContext = Global.withEnv { - name("IO") - plugin(IOPlugin) -} - -public val Context.io: IOPlugin get() = (if (this == Global) ioContext else this).fetch(IOPlugin) \ No newline at end of file +public val Context.io: IOPlugin + get() = if (this == Global) { + Global.buildContext("IO".asName()) { + plugin(IOPlugin) + }.fetch(IOPlugin) + } else { + fetch(IOPlugin) + } \ No newline at end of file diff --git a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/TaggedEnvelopeFormat.kt b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/TaggedEnvelopeFormat.kt index 8347118c..96c62db2 100644 --- a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/TaggedEnvelopeFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/TaggedEnvelopeFormat.kt @@ -161,7 +161,7 @@ public class TaggedEnvelopeFormat( } } - private val default by lazy { invoke(context = ioContext) } + private val default by lazy { invoke() } override fun readPartial(input: Input): PartialEnvelope = default.run { readPartial(input) } diff --git a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/TaglessEnvelopeFormat.kt b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/TaglessEnvelopeFormat.kt index e1a55607..bc8031d7 100644 --- a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/TaglessEnvelopeFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/TaglessEnvelopeFormat.kt @@ -193,11 +193,9 @@ public class TaglessEnvelopeFormat( override val name: Name = TAGLESS_ENVELOPE_TYPE.asName() - override fun invoke(meta: Meta, context: Context): EnvelopeFormat { - return TaglessEnvelopeFormat(context.io, meta) - } + override fun invoke(meta: Meta, context: Context): EnvelopeFormat = TaglessEnvelopeFormat(context.io, meta) - private val default by lazy { invoke(context = ioContext) } + private val default by lazy { invoke() } override fun readPartial(input: Input): PartialEnvelope = default.run { readPartial(input) } diff --git a/dataforge-scripting/src/jvmTest/kotlin/space/kscience/dataforge/scripting/BuildersKtTest.kt b/dataforge-scripting/src/jvmTest/kotlin/space/kscience/dataforge/scripting/BuildersKtTest.kt index 36ad16fb..345faf14 100644 --- a/dataforge-scripting/src/jvmTest/kotlin/space/kscience/dataforge/scripting/BuildersKtTest.kt +++ b/dataforge-scripting/src/jvmTest/kotlin/space/kscience/dataforge/scripting/BuildersKtTest.kt @@ -15,7 +15,7 @@ class BuildersKtTest { Workspace(Global){ println("I am working") - context { name("test") } + //context { name("test") } target("testTarget") { "a" put 12 @@ -28,8 +28,6 @@ class BuildersKtTest { val script = """ println("I am working") - context{ name("test") } - target("testTarget"){ "a" put 12 } diff --git a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/WorkspaceBuilder.kt b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/WorkspaceBuilder.kt index 72a35da7..2d5715de 100644 --- a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/WorkspaceBuilder.kt +++ b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/WorkspaceBuilder.kt @@ -14,6 +14,7 @@ import space.kscience.dataforge.meta.descriptors.MetaDescriptorBuilder import space.kscience.dataforge.misc.DFBuilder import space.kscience.dataforge.misc.DFExperimental import space.kscience.dataforge.names.Name +import space.kscience.dataforge.names.asName import kotlin.properties.PropertyDelegateProvider import kotlin.properties.ReadOnlyProperty @@ -62,7 +63,7 @@ public class WorkspaceBuilder(private val parentContext: Context = Global) : Tas * Define a context for the workspace */ public fun context(block: ContextBuilder.() -> Unit = {}) { - this.context = parentContext.buildContext("workspace", block) + this.context = parentContext.buildContext("workspace".asName(), block) } /** diff --git a/gradle.properties b/gradle.properties index 2bae1b5e..5c9b6463 100644 --- a/gradle.properties +++ b/gradle.properties @@ -4,6 +4,6 @@ org.gradle.parallel=true kotlin.code.style=official kotlin.parallel.tasks.in.project=true -kotlin.mpp.enableGranularSourceSetsMetadata=true -kotlin.native.enableDependencyPropagation=false +#kotlin.mpp.enableGranularSourceSetsMetadata=true +#kotlin.native.enableDependencyPropagation=false kotlin.mpp.stability.nowarn=true