From c175dc7de4d6ff490b9150dcf6a4eddfcceccfc7 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Sat, 14 Sep 2019 10:36:47 +0300 Subject: [PATCH] Added strong typing to tasks and task dependencies --- .../kotlin/hep/dataforge/data/DataNode.kt | 65 ++++++++++++++---- .../kotlin/hep/dataforge/data/JoinAction.kt | 6 +- .../kotlin/hep/dataforge/data/PipeAction.kt | 10 +-- .../kotlin/hep/dataforge/data/SplitAction.kt | 6 +- .../dataforge/data/TypeFilteredDataNode.kt | 4 +- .../kotlin/hep/dataforge/data/dataJVM.kt | 14 ++-- .../hep/dataforge/workspace/Dependency.kt | 25 ++++--- .../hep/dataforge/workspace/GenericTask.kt | 2 +- .../hep/dataforge/workspace/TaskBuilder.kt | 60 +++++++++-------- .../hep/dataforge/workspace/TaskModel.kt | 2 +- .../dataforge/workspace/WorkspaceBuilder.kt | 66 +++++++++++-------- .../hep/dataforge/workspace/taskTemplates.kt | 4 +- .../workspace/SimpleWorkspaceTest.kt | 38 ++++++----- 13 files changed, 180 insertions(+), 122 deletions(-) diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataNode.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataNode.kt index 4cc5aaf0..d7158dab 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataNode.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataNode.kt @@ -1,8 +1,6 @@ package hep.dataforge.data -import hep.dataforge.meta.Meta -import hep.dataforge.meta.MetaRepr -import hep.dataforge.meta.buildMeta +import hep.dataforge.meta.* import hep.dataforge.names.* import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Job @@ -18,20 +16,20 @@ sealed class DataItem : MetaRepr { class Node(val value: DataNode) : DataItem() { override val type: KClass get() = value.type - override fun toMeta(): Meta = value.toMeta() + override fun toMeta(): Meta = value.toMeta() } class Leaf(val value: Data) : DataItem() { override val type: KClass get() = value.type - override fun toMeta(): Meta = value.toMeta() + override fun toMeta(): Meta = value.toMeta() } } /** * A tree-like data structure grouped into the node. All data inside the node must inherit its type */ -interface DataNode: MetaRepr { +interface DataNode : MetaRepr { /** * The minimal common ancestor to all data in the node @@ -41,7 +39,7 @@ interface DataNode: MetaRepr { val items: Map> override fun toMeta(): Meta = buildMeta { - "type" to (type.simpleName?:"undefined") + "type" to (type.simpleName ?: "undefined") "items" to { this@DataNode.items.forEach { it.key.toString() to it.value.toMeta() @@ -52,9 +50,12 @@ interface DataNode: MetaRepr { companion object { const val TYPE = "dataNode" - fun build(type: KClass, block: DataTreeBuilder.() -> Unit) = + operator fun invoke(type: KClass, block: DataTreeBuilder.() -> Unit) = DataTreeBuilder(type).apply(block).build() + inline operator fun invoke(noinline block: DataTreeBuilder.() -> Unit) = + DataTreeBuilder(T::class).apply(block).build() + fun builder(type: KClass) = DataTreeBuilder(type) } } @@ -142,7 +143,7 @@ private sealed class DataTreeBuilderItem { class DataTreeBuilder(private val type: KClass) { private val map = HashMap>() - operator fun set(token: NameToken, node: DataTreeBuilder) { + operator fun set(token: NameToken, node: DataTreeBuilder) { if (map.containsKey(token)) error("Tree entry with name $token is not empty") map[token] = DataTreeBuilderItem.Node(node) } @@ -154,9 +155,9 @@ class DataTreeBuilder(private val type: KClass) { private fun buildNode(token: NameToken): DataTreeBuilder { return if (!map.containsKey(token)) { - DataTreeBuilder(type).also { map[token] = DataTreeBuilderItem.Node(it) } + DataTreeBuilder(type).also { map[token] = DataTreeBuilderItem.Node(it) } } else { - (map[token] as? DataTreeBuilderItem.Node ?: error("The node with name $token is occupied by leaf")).tree + (map[token] as? DataTreeBuilderItem.Node ?: error("The node with name $token is occupied by leaf")).tree } } @@ -176,7 +177,7 @@ class DataTreeBuilder(private val type: KClass) { } } - operator fun set(name: Name, node: DataTreeBuilder) { + operator fun set(name: Name, node: DataTreeBuilder) { when (name.length) { 0 -> error("Can't add data with empty name") 1 -> set(name.first()!!, node) @@ -206,7 +207,7 @@ class DataTreeBuilder(private val type: KClass) { /** * Build and append node */ - infix fun String.to(block: DataTreeBuilder.() -> Unit) = set(toName(), DataTreeBuilder(type).apply(block)) + infix fun String.to(block: DataTreeBuilder.() -> Unit) = set(toName(), DataTreeBuilder(type).apply(block)) fun update(node: DataNode) { @@ -227,6 +228,42 @@ class DataTreeBuilder(private val type: KClass) { } } +fun DataTreeBuilder.datum(name: Name, data: Data) { + this[name] = data +} + +fun DataTreeBuilder.datum(name: String, data: Data) { + this[name.toName()] = data +} + +fun DataTreeBuilder.static(name: Name, data: T, meta: Meta = EmptyMeta) { + this[name] = Data.static(data, meta) +} + +fun DataTreeBuilder.static(name: Name, data: T, block: MetaBuilder.() -> Unit = {}) { + this[name] = Data.static(data, buildMeta(block)) +} + +fun DataTreeBuilder.static(name: String, data: T, block: MetaBuilder.() -> Unit = {}) { + this[name.toName()] = Data.static(data, buildMeta(block)) +} + +fun DataTreeBuilder.node(name: Name, node: DataNode) { + this[name] = node +} + +fun DataTreeBuilder.node(name: String, node: DataNode) { + this[name.toName()] = node +} + +inline fun DataTreeBuilder.node(name: Name, noinline block: DataTreeBuilder.() -> Unit) { + this[name] = DataNode(T::class, block) +} + +inline fun DataTreeBuilder.node(name: String, noinline block: DataTreeBuilder.() -> Unit) { + this[name.toName()] = DataNode(T::class, block) +} + /** * Generate a mutable builder from this node. Node content is not changed */ @@ -234,7 +271,7 @@ fun DataNode.builder(): DataTreeBuilder = DataTreeBuilder(type). dataSequence().forEach { (name, data) -> this[name] = data } } -fun DataNode.filter(predicate: (Name, Data) -> Boolean): DataNode = DataNode.build(type) { +fun DataNode.filter(predicate: (Name, Data) -> Boolean): DataNode = DataNode.invoke(type) { dataSequence().forEach { (name, data) -> if (predicate(name, data)) { this[name] = data diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/JoinAction.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/JoinAction.kt index 31a54a74..7ad4055e 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/JoinAction.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/JoinAction.kt @@ -74,14 +74,14 @@ class JoinGroupBuilder(val actionMeta: Meta) { * The same rules as for KPipe */ class JoinAction( - val inputType: KClass, - val outputType: KClass, + val inputType: KClass, + val outputType: KClass, private val action: JoinGroupBuilder.() -> Unit ) : Action { override fun invoke(node: DataNode, meta: Meta): DataNode { node.ensureType(inputType) - return DataNode.build(outputType) { + return DataNode.invoke(outputType) { JoinGroupBuilder(meta).apply(action).buildGroups(node).forEach { group -> val laminate = Laminate(group.meta, meta) diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/PipeAction.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/PipeAction.kt index 59162179..91a204d8 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/PipeAction.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/PipeAction.kt @@ -22,16 +22,16 @@ class PipeBuilder(var name: Name, var meta: MetaBuilder) { } -class PipeAction( - val inputType: KClass, - val outputType: KClass, +class PipeAction( + val inputType: KClass, + val outputType: KClass, private val block: PipeBuilder.() -> Unit ) : Action { override fun invoke(node: DataNode, meta: Meta): DataNode { node.ensureType(inputType) - return DataNode.build(outputType) { + return DataNode.invoke(outputType) { node.dataSequence().forEach { (name, data) -> //merging data meta with action meta (data meta is primary) val oldMeta = meta.builder().apply { update(data.meta) } @@ -53,7 +53,7 @@ class PipeAction( inline fun DataNode.pipe( meta: Meta, - noinline action: PipeBuilder.() -> Unit + noinline action: PipeBuilder.() -> Unit ): DataNode = PipeAction(T::class, R::class, action).invoke(this, meta) diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/SplitAction.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/SplitAction.kt index 1e7e6c7c..637baeeb 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/SplitAction.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/SplitAction.kt @@ -33,15 +33,15 @@ class SplitBuilder(val name: Name, val meta: Meta) { } class SplitAction( - val inputType: KClass, - val outputType: KClass, + val inputType: KClass, + val outputType: KClass, private val action: SplitBuilder.() -> Unit ) : Action { override fun invoke(node: DataNode, meta: Meta): DataNode { node.ensureType(inputType) - return DataNode.build(outputType) { + return DataNode.invoke(outputType) { node.dataSequence().forEach { (name, data) -> val laminate = Laminate(data.meta, meta) diff --git a/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/TypeFilteredDataNode.kt b/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/TypeFilteredDataNode.kt index 5e90c4ed..d24de964 100644 --- a/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/TypeFilteredDataNode.kt +++ b/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/TypeFilteredDataNode.kt @@ -12,12 +12,12 @@ class TypeFilteredDataNode(val origin: DataNode<*>, override val ty origin.items.mapNotNull { (key, item) -> when (item) { is DataItem.Leaf -> { - (item.value.withType(type))?.let { + (item.value.filterIsInstance(type))?.let { key to DataItem.Leaf(it) } } is DataItem.Node -> { - key to DataItem.Node(item.value.withType(type)) + key to DataItem.Node(item.value.filterIsInstance(type)) } } }.associate { it } diff --git a/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/dataJVM.kt b/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/dataJVM.kt index 395f92b1..5b5507b2 100644 --- a/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/dataJVM.kt +++ b/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/dataJVM.kt @@ -22,18 +22,18 @@ actual fun Data<*>.canCast(type: KClass): Boolean = /** * Cast the node to given type if the cast is possible or return null */ -fun Data<*>.withType(type: KClass): Data? = +fun Data<*>.filterIsInstance(type: KClass): Data? = if (canCast(type)) cast(type) else null /** * Filter a node by data and node type. Resulting node and its subnodes is guaranteed to have border type [type], * but could contain empty nodes */ -fun DataNode<*>.withType(type: KClass): DataNode { +fun DataNode<*>.filterIsInstance(type: KClass): DataNode { return if (canCast(type)) { cast(type) } else if (this is TypeFilteredDataNode) { - origin.withType(type) + origin.filterIsInstance(type) } else { TypeFilteredDataNode(this, type) } @@ -42,10 +42,10 @@ fun DataNode<*>.withType(type: KClass): DataNode { /** * Filter all elements of given data item that could be cast to given type. If no elements are available, return null. */ -fun DataItem<*>?.withType(type: KClass): DataItem? = when (this) { +fun DataItem<*>?.filterIsInstance(type: KClass): DataItem? = when (this) { null -> null - is DataItem.Node -> DataItem.Node(this.value.withType(type)) - is DataItem.Leaf -> this.value.withType(type)?.let { DataItem.Leaf(it) } + is DataItem.Node -> DataItem.Node(this.value.filterIsInstance(type)) + is DataItem.Leaf -> this.value.filterIsInstance(type)?.let { DataItem.Leaf(it) } } -inline fun DataItem<*>?.withType(): DataItem? = this@withType.withType(R::class) \ No newline at end of file +inline fun DataItem<*>?.filterIsInstance(): DataItem? = this@filterIsInstance.filterIsInstance(R::class) \ No newline at end of file diff --git a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/Dependency.kt b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/Dependency.kt index 1e1a9c3f..1f184095 100644 --- a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/Dependency.kt +++ b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/Dependency.kt @@ -21,7 +21,7 @@ class DataDependency(val filter: DataFilter, val placement: Name = EmptyName) : return if (placement.isEmpty()) { result } else { - DataNode.build(Any::class) { this[placement] = result } + DataNode.invoke(Any::class) { this[placement] = result } } } @@ -35,7 +35,7 @@ class AllDataDependency(val placement: Name = EmptyName) : Dependency() { override fun apply(workspace: Workspace): DataNode = if (placement.isEmpty()) { workspace.data } else { - DataNode.build(Any::class) { this[placement] = workspace.data } + DataNode.invoke(Any::class) { this[placement] = workspace.data } } override fun toMeta() = buildMeta { @@ -44,22 +44,25 @@ class AllDataDependency(val placement: Name = EmptyName) : Dependency() { } } -abstract class TaskDependency(val meta: Meta, val placement: Name = EmptyName) : Dependency() { - abstract fun resolveTask(workspace: Workspace): Task<*> +abstract class TaskDependency( + val meta: Meta, + val placement: Name = EmptyName +) : Dependency() { + abstract fun resolveTask(workspace: Workspace): Task /** * A name of the dependency for logging and serialization */ abstract val name: Name - override fun apply(workspace: Workspace): DataNode { + override fun apply(workspace: Workspace): DataNode { val task = resolveTask(workspace) if (task.isTerminal) TODO("Support terminal task") val result = workspace.run(task, meta) return if (placement.isEmpty()) { result } else { - DataNode.build(Any::class) { this[placement] = result } + DataNode(task.type) { this[placement] = result } } } @@ -70,8 +73,12 @@ abstract class TaskDependency(val meta: Meta, val placement: Name = EmptyName) : } } -class DirectTaskDependency(val task: Task<*>, meta: Meta, placement: Name) : TaskDependency(meta, placement) { - override fun resolveTask(workspace: Workspace): Task<*> = task +class DirectTaskDependency( + val task: Task, + meta: Meta, + placement: Name +) : TaskDependency(meta, placement) { + override fun resolveTask(workspace: Workspace): Task = task override val name: Name get() = DIRECT_TASK_NAME + task.name @@ -80,7 +87,7 @@ class DirectTaskDependency(val task: Task<*>, meta: Meta, placement: Name) : Tas } } -class WorkspaceTaskDependency(override val name: Name, meta: Meta, placement: Name) : TaskDependency(meta, placement) { +class WorkspaceTaskDependency(override val name: Name, meta: Meta, placement: Name) : TaskDependency(meta, placement) { override fun resolveTask(workspace: Workspace): Task<*> = workspace.tasks[name] ?: error("Task with name $name is not found in the workspace") } \ No newline at end of file diff --git a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/GenericTask.kt b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/GenericTask.kt index a46862a0..408f1745 100644 --- a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/GenericTask.kt +++ b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/GenericTask.kt @@ -20,7 +20,7 @@ class GenericTask( ) : Task { private fun gather(workspace: Workspace, model: TaskModel): DataNode { - return DataNode.build(Any::class) { + return DataNode.invoke(Any::class) { model.dependencies.forEach { dep -> update(dep.apply(workspace)) } diff --git a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskBuilder.kt b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskBuilder.kt index 41b90681..bdacb711 100644 --- a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskBuilder.kt +++ b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskBuilder.kt @@ -13,7 +13,7 @@ import hep.dataforge.names.toName import kotlin.reflect.KClass @TaskBuildScope -class TaskBuilder(val name: String) { +class TaskBuilder(val name: String, val type: KClass) { private var modelTransform: TaskModelBuilder.(Meta) -> Unit = { allData() } var descriptor: NodeDescriptor? = null private val dataTransforms: MutableList = ArrayList() @@ -21,12 +21,12 @@ class TaskBuilder(val name: String) { /** * TODO will look better as extension class */ - private class DataTransformation( + private inner class DataTransformation( val from: String = "", val to: String = "", - val transform: (Context, TaskModel, DataNode) -> DataNode + val transform: (Context, TaskModel, DataNode) -> DataNode ) { - operator fun invoke(workspace: Workspace, model: TaskModel, node: DataNode): DataNode? { + operator fun invoke(workspace: Workspace, model: TaskModel, node: DataNode): DataNode? { val localData = if (from.isEmpty()) { node } else { @@ -46,9 +46,9 @@ class TaskBuilder(val name: String) { fun rawTransform( from: String = "", to: String = "", - block: TaskEnv.(DataNode<*>) -> DataNode<*> + block: TaskEnv.(DataNode<*>) -> DataNode ) { - dataTransforms += DataTransformation(from, to){context, model, data-> + dataTransforms += DataTransformation(from, to) { context, model, data -> val env = TaskEnv(EmptyName, model.meta, context) env.block(data) } @@ -58,7 +58,7 @@ class TaskBuilder(val name: String) { inputType: KClass, from: String = "", to: String = "", - block: TaskEnv.(DataNode) -> DataNode + block: TaskEnv.(DataNode) -> DataNode ) { dataTransforms += DataTransformation(from, to) { context, model, data -> data.ensureType(inputType) @@ -70,7 +70,7 @@ class TaskBuilder(val name: String) { inline fun transform( from: String = "", to: String = "", - noinline block: TaskEnv.(DataNode) -> DataNode + noinline block: TaskEnv.(DataNode) -> DataNode ) { transform(T::class, from, to, block) } @@ -78,7 +78,7 @@ class TaskBuilder(val name: String) { /** * Perform given action on data elements in `from` node in input and put the result to `to` node */ - inline fun action( + inline fun action( from: String = "", to: String = "", crossinline block: TaskEnv.() -> Action @@ -93,7 +93,7 @@ class TaskBuilder(val name: String) { /** * A customized pipe action with ability to change meta and name */ - inline fun customPipe( + inline fun customPipe( from: String = "", to: String = "", crossinline block: PipeBuilder.(TaskEnv) -> Unit @@ -101,7 +101,7 @@ class TaskBuilder(val name: String) { action(from, to) { PipeAction( inputType = T::class, - outputType = R::class + outputType = type ) { block(this@action) } } } @@ -109,7 +109,7 @@ class TaskBuilder(val name: String) { /** * A simple pipe action without changing meta or name */ - inline fun pipe( + inline fun pipe( from: String = "", to: String = "", crossinline block: suspend TaskEnv.(T) -> R @@ -117,7 +117,7 @@ class TaskBuilder(val name: String) { action(from, to) { PipeAction( inputType = T::class, - outputType = R::class + outputType = type ) { //TODO automatically append task meta result = { data -> @@ -130,7 +130,7 @@ class TaskBuilder(val name: String) { /** * Join elements in gathered data by multiple groups */ - inline fun joinByGroup( + inline fun joinByGroup( from: String = "", to: String = "", crossinline block: JoinGroupBuilder.(TaskEnv) -> Unit @@ -138,7 +138,7 @@ class TaskBuilder(val name: String) { action(from, to) { JoinAction( inputType = T::class, - outputType = R::class + outputType = type ) { block(this@action) } } } @@ -146,7 +146,7 @@ class TaskBuilder(val name: String) { /** * Join all elemlents in gathered data matching input type */ - inline fun join( + inline fun join( from: String = "", to: String = "", crossinline block: suspend TaskEnv.(Map) -> R @@ -154,7 +154,7 @@ class TaskBuilder(val name: String) { action(from, to) { JoinAction( inputType = T::class, - outputType = R::class, + outputType = type, action = { result( actionMeta[TaskModel.MODEL_TARGET_KEY]?.string ?: "@anonymous" @@ -169,7 +169,7 @@ class TaskBuilder(val name: String) { /** * Split each element in gathered data into fixed number of fragments */ - inline fun split( + inline fun split( from: String = "", to: String = "", crossinline block: SplitBuilder.(TaskEnv) -> Unit @@ -177,7 +177,7 @@ class TaskBuilder(val name: String) { action(from, to) { SplitAction( inputType = T::class, - outputType = R::class + outputType = type ) { block(this@action) } } } @@ -189,22 +189,23 @@ class TaskBuilder(val name: String) { this.descriptor = NodeDescriptor.build(transform) } - internal fun build(): GenericTask = + internal fun build(): GenericTask = GenericTask( name.asName(), - Any::class, + type, descriptor ?: NodeDescriptor.empty(), modelTransform ) { val workspace = this - { data -> + return@GenericTask { data -> val model = this if (dataTransforms.isEmpty()) { //return data node as is logger.warn { "No transformation present, returning input data" } - data + data.ensureType(type) + data.cast(type) } else { - val builder = DataTreeBuilder(Any::class) + val builder = DataTreeBuilder(type) dataTransforms.forEach { transformation -> val res = transformation(workspace, model, data) if (res != null) { @@ -221,12 +222,13 @@ class TaskBuilder(val name: String) { } } -fun Workspace.Companion.task(name: String, builder: TaskBuilder.() -> Unit): GenericTask { - return TaskBuilder(name).apply(builder).build() +fun Workspace.Companion.task( + name: String, + type: KClass, + builder: TaskBuilder.() -> Unit +): GenericTask { + return TaskBuilder(name, type).apply(builder).build() } -fun WorkspaceBuilder.task(name: String, builder: TaskBuilder.() -> Unit) { - task(TaskBuilder(name).apply(builder).build()) -} //TODO add delegates to build gradle-like tasks \ No newline at end of file diff --git a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskModel.kt b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskModel.kt index e7443807..f9562438 100644 --- a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskModel.kt +++ b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskModel.kt @@ -36,7 +36,7 @@ data class TaskModel( "meta" to meta "dependsOn" to { val dataDependencies = dependencies.filterIsInstance() - val taskDependencies = dependencies.filterIsInstance() + val taskDependencies = dependencies.filterIsInstance>() setIndexed("data".toName(), dataDependencies.map { it.toMeta() }) setIndexed("task".toName(), taskDependencies.map { it.toMeta() }) { taskDependencies[it].name.toString() } //TODO ensure all dependencies are listed diff --git a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/WorkspaceBuilder.kt b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/WorkspaceBuilder.kt index 999ee50c..6bc4900a 100644 --- a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/WorkspaceBuilder.kt +++ b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/WorkspaceBuilder.kt @@ -2,12 +2,14 @@ package hep.dataforge.workspace import hep.dataforge.context.Context import hep.dataforge.context.ContextBuilder -import hep.dataforge.data.Data import hep.dataforge.data.DataNode import hep.dataforge.data.DataTreeBuilder import hep.dataforge.meta.* +import hep.dataforge.names.EmptyName import hep.dataforge.names.Name -import hep.dataforge.names.toName +import hep.dataforge.names.isEmpty +import kotlin.jvm.JvmName +import kotlin.reflect.KClass @TaskBuildScope interface WorkspaceBuilder { @@ -28,32 +30,26 @@ fun WorkspaceBuilder.context(name: String = "WORKSPACE", block: ContextBuilder.( context = ContextBuilder(name, parentContext).apply(block).build() } -fun WorkspaceBuilder.data(name: Name, data: Data) { - this.data[name] = data +inline fun WorkspaceBuilder.data( + name: Name = EmptyName, + noinline block: DataTreeBuilder.() -> Unit +): DataNode { + val node = DataTreeBuilder(T::class).apply(block) + if (name.isEmpty()) { + @Suppress("UNCHECKED_CAST") + data = node as DataTreeBuilder + } else { + data[name] = node + } + return node.build() } -fun WorkspaceBuilder.data(name: String, data: Data) = data(name.toName(), data) +@JvmName("rawData") +fun WorkspaceBuilder.data( + name: Name = EmptyName, + block: DataTreeBuilder.() -> Unit +): DataNode = data(name, block) -fun WorkspaceBuilder.static(name: Name, data: Any, meta: Meta = EmptyMeta) = - data(name, Data.static(data, meta)) - -fun WorkspaceBuilder.static(name: Name, data: Any, block: MetaBuilder.() -> Unit = {}) = - data(name, Data.static(data, buildMeta(block))) - -fun WorkspaceBuilder.static(name: String, data: Any, block: MetaBuilder.() -> Unit = {}) = - data(name, Data.static(data, buildMeta(block))) - -fun WorkspaceBuilder.data(name: Name, node: DataNode) { - this.data[name] = node -} - -fun WorkspaceBuilder.data(name: String, node: DataNode) = data(name.toName(), node) - -fun WorkspaceBuilder.data(name: Name, block: DataTreeBuilder.() -> Unit) { - this.data[name] = DataNode.build(Any::class, block) -} - -fun WorkspaceBuilder.data(name: String, block: DataTreeBuilder.() -> Unit) = data(name.toName(), block) fun WorkspaceBuilder.target(name: String, block: MetaBuilder.() -> Unit) { targets[name] = buildMeta(block).seal() @@ -70,17 +66,31 @@ fun WorkspaceBuilder.target(name: String, base: String, block: MetaBuilder.() -> .seal() } -fun WorkspaceBuilder.task(task: Task) { - this.tasks.add(task) +fun WorkspaceBuilder.task( + name: String, + type: KClass, + builder: TaskBuilder.() -> Unit +): Task { + return TaskBuilder(name, type).apply(builder).build().also { tasks.add(it) } } +inline fun WorkspaceBuilder.task( + name: String, + noinline builder: TaskBuilder.() -> Unit +): Task = task(name, T::class, builder) + +@JvmName("rawTask") +fun WorkspaceBuilder.task( + name: String, + builder: TaskBuilder.() -> Unit +): Task = task(name, Any::class, builder) /** * A builder for a simple workspace */ class SimpleWorkspaceBuilder(override val parentContext: Context) : WorkspaceBuilder { override var context: Context = parentContext - override var data = DataTreeBuilder(Any::class) + override var data: DataTreeBuilder = DataTreeBuilder(Any::class) override var tasks: MutableSet> = HashSet() override var targets: MutableMap = HashMap() diff --git a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/taskTemplates.kt b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/taskTemplates.kt index ab282015..a2602da3 100644 --- a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/taskTemplates.kt +++ b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/taskTemplates.kt @@ -1,5 +1,5 @@ package hep.dataforge.workspace //fun TaskBuilder.zip( -// val firstNo -//) = rawTransform { } \ No newline at end of file +//// val firstNo +////) = rawTransform { } \ No newline at end of file diff --git a/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/SimpleWorkspaceTest.kt b/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/SimpleWorkspaceTest.kt index 95233251..2adc27b0 100644 --- a/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/SimpleWorkspaceTest.kt +++ b/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/SimpleWorkspaceTest.kt @@ -5,7 +5,7 @@ import hep.dataforge.data.* import hep.dataforge.meta.boolean import hep.dataforge.meta.get import hep.dataforge.names.asName -import kotlin.test.Test +import org.junit.Test import kotlin.test.assertEquals import kotlin.test.assertTrue @@ -14,8 +14,8 @@ class SimpleWorkspaceTest { val testPlugin = object : WorkspacePlugin() { override val tag: PluginTag = PluginTag("test") - val contextTask = Workspace.task("test") { - pipe { + val contextTask = Workspace.task("test", Any::class) { + pipe { context.logger.info { "Test: $it" } } } @@ -28,8 +28,10 @@ class SimpleWorkspaceTest { plugin(testPlugin) } - repeat(100) { - static("myData[$it]", it) + data { + repeat(100) { + static("myData[$it]", it) + } } @@ -37,7 +39,7 @@ class SimpleWorkspaceTest { model { allData() } - pipe { data -> + pipe { data -> if (meta["testFlag"].boolean == true) { println("flag") } @@ -50,7 +52,7 @@ class SimpleWorkspaceTest { model { allData() } - pipe { data -> + pipe { data -> context.logger.info { "Starting linear on $data" } data * 2 + 1 } @@ -62,16 +64,16 @@ class SimpleWorkspaceTest { dependsOn("linear", placement = "linear".asName()) } transform { data -> - val squareNode = data["square"].withType().node!! - val linearNode = data["linear"].withType().node!! - return@transform DataNode.build(Int::class) { + val squareNode = data["square"].filterIsInstance().node!! + val linearNode = data["linear"].filterIsInstance().node!! + return@transform DataNode(Int::class) { squareNode.dataSequence().forEach { (name, _) -> - val newData = Data{ + val newData = Data { val squareValue = squareNode[name].data!!.get() val linearValue = linearNode[name].data!!.get() - squareValue+linearValue + squareValue + linearValue } - set(name,newData) + set(name, newData) } } } @@ -81,17 +83,17 @@ class SimpleWorkspaceTest { model { dependsOn("square") } - join { data -> + join { data -> context.logger.info { "Starting sum" } data.values.sum() } } - task("average") { + task("average") { model { allData() } - joinByGroup { env -> + joinByGroup { env -> group("even", filter = { name, _ -> name.toString().toInt() % 2 == 0 }) { result { data -> env.context.logger.info { "Starting even" } @@ -111,7 +113,7 @@ class SimpleWorkspaceTest { model { dependsOn("average") } - join { data -> + join { data -> data["even"]!! - data["odd"]!! } } @@ -140,7 +142,7 @@ class SimpleWorkspaceTest { } @Test - fun testFullSquare(){ + fun testFullSquare() { val node = workspace.run("fullsquare") println(node.toMeta()) }