kscience/dataforge-core
8
0
Fork 0
Code Issues Pull Requests 1 Packages Projects Releases 12 Wiki Activity

Added strong typing to tasks and task dependencies

Browse Source
This commit is contained in:
Alexander Nozik 2019-09-14 10:36:47 +03:00
parent a0abb99d88
commit c175dc7de4
13 changed files with 180 additions and 122 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

65
dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataNode.kt
View File

@ -1,8 +1,6 @@
package hep.dataforge.data
import hep.dataforge.meta.Meta
import hep.dataforge.meta.MetaRepr
import hep.dataforge.meta.buildMeta
import hep.dataforge.meta.*
import hep.dataforge.names.*
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Job
@ -18,20 +16,20 @@ sealed class DataItem<out T : Any> : MetaRepr {
class Node<out T : Any>(val value: DataNode<T>) : DataItem<T>() {
override val type: KClass<out T> get() = value.type
override fun toMeta(): Meta = value.toMeta()
override fun toMeta(): Meta = value.toMeta()
}
class Leaf<out T : Any>(val value: Data<T>) : DataItem<T>() {
override val type: KClass<out T> get() = value.type
override fun toMeta(): Meta = value.toMeta()
override fun toMeta(): Meta = value.toMeta()
}
}
/**
* A tree-like data structure grouped into the node. All data inside the node must inherit its type
*/
interface DataNode<out T : Any>: MetaRepr {
interface DataNode<out T : Any> : MetaRepr {
/**
* The minimal common ancestor to all data in the node
@ -41,7 +39,7 @@ interface DataNode<out T : Any>: MetaRepr {
val items: Map<NameToken, DataItem<T>>
override fun toMeta(): Meta = buildMeta {
"type" to (type.simpleName?:"undefined")
"type" to (type.simpleName ?: "undefined")
"items" to {
this@DataNode.items.forEach {
it.key.toString() to it.value.toMeta()
@ -52,9 +50,12 @@ interface DataNode<out T : Any>: MetaRepr {
companion object {
const val TYPE = "dataNode"
fun <T : Any> build(type: KClass<out T>, block: DataTreeBuilder<T>.() -> Unit) =
operator fun <T : Any> invoke(type: KClass<out T>, block: DataTreeBuilder<T>.() -> Unit) =
DataTreeBuilder(type).apply(block).build()
inline operator fun <reified T : Any> invoke(noinline block: DataTreeBuilder<T>.() -> Unit) =
DataTreeBuilder(T::class).apply(block).build()
fun <T : Any> builder(type: KClass<out T>) = DataTreeBuilder(type)
}
}
@ -142,7 +143,7 @@ private sealed class DataTreeBuilderItem<out T : Any> {
class DataTreeBuilder<T : Any>(private val type: KClass<out T>) {
private val map = HashMap<NameToken, DataTreeBuilderItem<T>>()
operator fun set(token: NameToken, node: DataTreeBuilder<T>) {
operator fun set(token: NameToken, node: DataTreeBuilder<out T>) {
if (map.containsKey(token)) error("Tree entry with name $token is not empty")
map[token] = DataTreeBuilderItem.Node(node)
}
@ -154,9 +155,9 @@ class DataTreeBuilder<T : Any>(private val type: KClass<out T>) {
private fun buildNode(token: NameToken): DataTreeBuilder<T> {
return if (!map.containsKey(token)) {
DataTreeBuilder<T>(type).also { map[token] = DataTreeBuilderItem.Node(it) }
DataTreeBuilder(type).also { map[token] = DataTreeBuilderItem.Node(it) }
} else {
(map[token] as? DataTreeBuilderItem.Node ?: error("The node with name $token is occupied by leaf")).tree
(map[token] as? DataTreeBuilderItem.Node<T> ?: error("The node with name $token is occupied by leaf")).tree
}
}
@ -176,7 +177,7 @@ class DataTreeBuilder<T : Any>(private val type: KClass<out T>) {
}
}
operator fun set(name: Name, node: DataTreeBuilder<T>) {
operator fun set(name: Name, node: DataTreeBuilder<out T>) {
when (name.length) {
0 -> error("Can't add data with empty name")
1 -> set(name.first()!!, node)
@ -206,7 +207,7 @@ class DataTreeBuilder<T : Any>(private val type: KClass<out T>) {
/**
* Build and append node
*/
infix fun String.to(block: DataTreeBuilder<T>.() -> Unit) = set(toName(), DataTreeBuilder<T>(type).apply(block))
infix fun String.to(block: DataTreeBuilder<out T>.() -> Unit) = set(toName(), DataTreeBuilder<T>(type).apply(block))
fun update(node: DataNode<T>) {
@ -227,6 +228,42 @@ class DataTreeBuilder<T : Any>(private val type: KClass<out T>) {
}
}
fun <T : Any> DataTreeBuilder<T>.datum(name: Name, data: Data<T>) {
this[name] = data
}
fun <T : Any> DataTreeBuilder<T>.datum(name: String, data: Data<T>) {
this[name.toName()] = data
}
fun <T : Any> DataTreeBuilder<T>.static(name: Name, data: T, meta: Meta = EmptyMeta) {
this[name] = Data.static(data, meta)
}
fun <T : Any> DataTreeBuilder<T>.static(name: Name, data: T, block: MetaBuilder.() -> Unit = {}) {
this[name] = Data.static(data, buildMeta(block))
}
fun <T : Any> DataTreeBuilder<T>.static(name: String, data: T, block: MetaBuilder.() -> Unit = {}) {
this[name.toName()] = Data.static(data, buildMeta(block))
}
fun <T : Any> DataTreeBuilder<T>.node(name: Name, node: DataNode<T>) {
this[name] = node
}
fun <T : Any> DataTreeBuilder<T>.node(name: String, node: DataNode<T>) {
this[name.toName()] = node
}
inline fun <reified T : Any> DataTreeBuilder<T>.node(name: Name, noinline block: DataTreeBuilder<T>.() -> Unit) {
this[name] = DataNode(T::class, block)
}
inline fun <reified T : Any> DataTreeBuilder<T>.node(name: String, noinline block: DataTreeBuilder<T>.() -> Unit) {
this[name.toName()] = DataNode(T::class, block)
}
/**
* Generate a mutable builder from this node. Node content is not changed
*/
@ -234,7 +271,7 @@ fun <T : Any> DataNode<T>.builder(): DataTreeBuilder<T> = DataTreeBuilder(type).
dataSequence().forEach { (name, data) -> this[name] = data }
}
fun <T : Any> DataNode<T>.filter(predicate: (Name, Data<T>) -> Boolean): DataNode<T> = DataNode.build(type) {
fun <T : Any> DataNode<T>.filter(predicate: (Name, Data<T>) -> Boolean): DataNode<T> = DataNode.invoke(type) {
dataSequence().forEach { (name, data) ->
if (predicate(name, data)) {
this[name] = data

6
dataforge-data/src/commonMain/kotlin/hep/dataforge/data/JoinAction.kt
View File

@ -74,14 +74,14 @@ class JoinGroupBuilder<T : Any, R : Any>(val actionMeta: Meta) {
* The same rules as for KPipe
*/
class JoinAction<T : Any, R : Any>(
val inputType: KClass<T>,
val outputType: KClass<R>,
val inputType: KClass<out T>,
val outputType: KClass<out R>,
private val action: JoinGroupBuilder<T, R>.() -> Unit
) : Action<T, R> {
override fun invoke(node: DataNode<T>, meta: Meta): DataNode<R> {
node.ensureType(inputType)
return DataNode.build(outputType) {
return DataNode.invoke(outputType) {
JoinGroupBuilder<T, R>(meta).apply(action).buildGroups(node).forEach { group ->
val laminate = Laminate(group.meta, meta)

10
dataforge-data/src/commonMain/kotlin/hep/dataforge/data/PipeAction.kt
View File

@ -22,16 +22,16 @@ class PipeBuilder<T, R>(var name: Name, var meta: MetaBuilder) {
}
class PipeAction<T : Any, R : Any>(
val inputType: KClass<T>,
val outputType: KClass<R>,
class PipeAction<T : Any, out R : Any>(
val inputType: KClass<out T>,
val outputType: KClass<out R>,
private val block: PipeBuilder<T, R>.() -> Unit
) : Action<T, R> {
override fun invoke(node: DataNode<T>, meta: Meta): DataNode<R> {
node.ensureType(inputType)
return DataNode.build(outputType) {
return DataNode.invoke(outputType) {
node.dataSequence().forEach { (name, data) ->
//merging data meta with action meta (data meta is primary)
val oldMeta = meta.builder().apply { update(data.meta) }
@ -53,7 +53,7 @@ class PipeAction<T : Any, R : Any>(
inline fun <reified T : Any, reified R : Any> DataNode<T>.pipe(
meta: Meta,
noinline action: PipeBuilder<T, R>.() -> Unit
noinline action: PipeBuilder<in T, out R>.() -> Unit
): DataNode<R> = PipeAction(T::class, R::class, action).invoke(this, meta)

6
dataforge-data/src/commonMain/kotlin/hep/dataforge/data/SplitAction.kt
View File

@ -33,15 +33,15 @@ class SplitBuilder<T : Any, R : Any>(val name: Name, val meta: Meta) {
}
class SplitAction<T : Any, R : Any>(
val inputType: KClass<T>,
val outputType: KClass<R>,
val inputType: KClass<out T>,
val outputType: KClass<out R>,
private val action: SplitBuilder<T, R>.() -> Unit
) : Action<T, R> {
override fun invoke(node: DataNode<T>, meta: Meta): DataNode<R> {
node.ensureType(inputType)
return DataNode.build(outputType) {
return DataNode.invoke(outputType) {
node.dataSequence().forEach { (name, data) ->
val laminate = Laminate(data.meta, meta)

4
dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/TypeFilteredDataNode.kt
View File

@ -12,12 +12,12 @@ class TypeFilteredDataNode<out T : Any>(val origin: DataNode<*>, override val ty
origin.items.mapNotNull { (key, item) ->
when (item) {
is DataItem.Leaf -> {
(item.value.withType(type))?.let {
(item.value.filterIsInstance(type))?.let {
key to DataItem.Leaf(it)
}
}
is DataItem.Node -> {
key to DataItem.Node(item.value.withType(type))
key to DataItem.Node(item.value.filterIsInstance(type))
}
}
}.associate { it }

14
dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/dataJVM.kt
View File

@ -22,18 +22,18 @@ actual fun <R : Any> Data<*>.canCast(type: KClass<out R>): Boolean =
/**
* Cast the node to given type if the cast is possible or return null
*/
fun <R : Any> Data<*>.withType(type: KClass<out R>): Data<R>? =
fun <R : Any> Data<*>.filterIsInstance(type: KClass<out R>): Data<R>? =
if (canCast(type)) cast(type) else null
/**
* Filter a node by data and node type. Resulting node and its subnodes is guaranteed to have border type [type],
* but could contain empty nodes
*/
fun <R : Any> DataNode<*>.withType(type: KClass<out R>): DataNode<R> {
fun <R : Any> DataNode<*>.filterIsInstance(type: KClass<out R>): DataNode<R> {
return if (canCast(type)) {
cast(type)
} else if (this is TypeFilteredDataNode) {
origin.withType(type)
origin.filterIsInstance(type)
} else {
TypeFilteredDataNode(this, type)
}
@ -42,10 +42,10 @@ fun <R : Any> DataNode<*>.withType(type: KClass<out R>): DataNode<R> {
/**
* Filter all elements of given data item that could be cast to given type. If no elements are available, return null.
*/
fun <R : Any> DataItem<*>?.withType(type: KClass<out R>): DataItem<R>? = when (this) {
fun <R : Any> DataItem<*>?.filterIsInstance(type: KClass<out R>): DataItem<R>? = when (this) {
null -> null
is DataItem.Node -> DataItem.Node(this.value.withType(type))
is DataItem.Leaf -> this.value.withType(type)?.let { DataItem.Leaf(it) }
is DataItem.Node -> DataItem.Node(this.value.filterIsInstance(type))
is DataItem.Leaf -> this.value.filterIsInstance(type)?.let { DataItem.Leaf(it) }
}
inline fun <reified R : Any> DataItem<*>?.withType(): DataItem<R>? = this@withType.withType(R::class)
inline fun <reified R : Any> DataItem<*>?.filterIsInstance(): DataItem<R>? = this@filterIsInstance.filterIsInstance(R::class)

25
dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/Dependency.kt
View File

@ -21,7 +21,7 @@ class DataDependency(val filter: DataFilter, val placement: Name = EmptyName) :
return if (placement.isEmpty()) {
result
} else {
DataNode.build(Any::class) { this[placement] = result }
DataNode.invoke(Any::class) { this[placement] = result }
}
}
@ -35,7 +35,7 @@ class AllDataDependency(val placement: Name = EmptyName) : Dependency() {
override fun apply(workspace: Workspace): DataNode<Any> = if (placement.isEmpty()) {
workspace.data
} else {
DataNode.build(Any::class) { this[placement] = workspace.data }
DataNode.invoke(Any::class) { this[placement] = workspace.data }
}
override fun toMeta() = buildMeta {
@ -44,22 +44,25 @@ class AllDataDependency(val placement: Name = EmptyName) : Dependency() {
}
}
abstract class TaskDependency(val meta: Meta, val placement: Name = EmptyName) : Dependency() {
abstract fun resolveTask(workspace: Workspace): Task<*>
abstract class TaskDependency<out T : Any>(
val meta: Meta,
val placement: Name = EmptyName
) : Dependency() {
abstract fun resolveTask(workspace: Workspace): Task<T>
/**
* A name of the dependency for logging and serialization
*/
abstract val name: Name
override fun apply(workspace: Workspace): DataNode<Any> {
override fun apply(workspace: Workspace): DataNode<T> {
val task = resolveTask(workspace)
if (task.isTerminal) TODO("Support terminal task")
val result = workspace.run(task, meta)
return if (placement.isEmpty()) {
result
} else {
DataNode.build(Any::class) { this[placement] = result }
DataNode(task.type) { this[placement] = result }
}
}
@ -70,8 +73,12 @@ abstract class TaskDependency(val meta: Meta, val placement: Name = EmptyName) :
}
}
class DirectTaskDependency(val task: Task<*>, meta: Meta, placement: Name) : TaskDependency(meta, placement) {
override fun resolveTask(workspace: Workspace): Task<*> = task
class DirectTaskDependency<T : Any>(
val task: Task<T>,
meta: Meta,
placement: Name
) : TaskDependency<T>(meta, placement) {
override fun resolveTask(workspace: Workspace): Task<T> = task
override val name: Name get() = DIRECT_TASK_NAME + task.name
@ -80,7 +87,7 @@ class DirectTaskDependency(val task: Task<*>, meta: Meta, placement: Name) : Tas
}
}
class WorkspaceTaskDependency(override val name: Name, meta: Meta, placement: Name) : TaskDependency(meta, placement) {
class WorkspaceTaskDependency(override val name: Name, meta: Meta, placement: Name) : TaskDependency<Any>(meta, placement) {
override fun resolveTask(workspace: Workspace): Task<*> =
workspace.tasks[name] ?: error("Task with name $name is not found in the workspace")
}

2
dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/GenericTask.kt
View File

@ -20,7 +20,7 @@ class GenericTask<R : Any>(
) : Task<R> {
private fun gather(workspace: Workspace, model: TaskModel): DataNode<Any> {
return DataNode.build(Any::class) {
return DataNode.invoke(Any::class) {
model.dependencies.forEach { dep ->
update(dep.apply(workspace))
}

60
dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskBuilder.kt
View File

@ -13,7 +13,7 @@ import hep.dataforge.names.toName
import kotlin.reflect.KClass
@TaskBuildScope
class TaskBuilder(val name: String) {
class TaskBuilder<R : Any>(val name: String, val type: KClass<out R>) {
private var modelTransform: TaskModelBuilder.(Meta) -> Unit = { allData() }
var descriptor: NodeDescriptor? = null
private val dataTransforms: MutableList<DataTransformation> = ArrayList()
@ -21,12 +21,12 @@ class TaskBuilder(val name: String) {
/**
* TODO will look better as extension class
*/
private class DataTransformation(
private inner class DataTransformation(
val from: String = "",
val to: String = "",
val transform: (Context, TaskModel, DataNode<Any>) -> DataNode<Any>
val transform: (Context, TaskModel, DataNode<Any>) -> DataNode<R>
) {
operator fun invoke(workspace: Workspace, model: TaskModel, node: DataNode<Any>): DataNode<Any>? {
operator fun invoke(workspace: Workspace, model: TaskModel, node: DataNode<Any>): DataNode<R>? {
val localData = if (from.isEmpty()) {
node
} else {
@ -46,9 +46,9 @@ class TaskBuilder(val name: String) {
fun rawTransform(
from: String = "",
to: String = "",
block: TaskEnv.(DataNode<*>) -> DataNode<*>
block: TaskEnv.(DataNode<*>) -> DataNode<R>
) {
dataTransforms += DataTransformation(from, to){context, model, data->
dataTransforms += DataTransformation(from, to) { context, model, data ->
val env = TaskEnv(EmptyName, model.meta, context)
env.block(data)
}
@ -58,7 +58,7 @@ class TaskBuilder(val name: String) {
inputType: KClass<out T>,
from: String = "",
to: String = "",
block: TaskEnv.(DataNode<T>) -> DataNode<Any>
block: TaskEnv.(DataNode<T>) -> DataNode<R>
) {
dataTransforms += DataTransformation(from, to) { context, model, data ->
data.ensureType(inputType)
@ -70,7 +70,7 @@ class TaskBuilder(val name: String) {
inline fun <reified T : Any> transform(
from: String = "",
to: String = "",
noinline block: TaskEnv.(DataNode<T>) -> DataNode<Any>
noinline block: TaskEnv.(DataNode<T>) -> DataNode<R>
) {
transform(T::class, from, to, block)
}
@ -78,7 +78,7 @@ class TaskBuilder(val name: String) {
/**
* Perform given action on data elements in `from` node in input and put the result to `to` node
*/
inline fun <reified T : Any, reified R : Any> action(
inline fun <reified T : Any> action(
from: String = "",
to: String = "",
crossinline block: TaskEnv.() -> Action<T, R>
@ -93,7 +93,7 @@ class TaskBuilder(val name: String) {
/**
* A customized pipe action with ability to change meta and name
*/
inline fun <reified T : Any, reified R : Any> customPipe(
inline fun <reified T : Any> customPipe(
from: String = "",
to: String = "",
crossinline block: PipeBuilder<T, R>.(TaskEnv) -> Unit
@ -101,7 +101,7 @@ class TaskBuilder(val name: String) {
action(from, to) {
PipeAction(
inputType = T::class,
outputType = R::class
outputType = type
) { block(this@action) }
}
}
@ -109,7 +109,7 @@ class TaskBuilder(val name: String) {
/**
* A simple pipe action without changing meta or name
*/
inline fun <reified T : Any, reified R : Any> pipe(
inline fun <reified T : Any> pipe(
from: String = "",
to: String = "",
crossinline block: suspend TaskEnv.(T) -> R
@ -117,7 +117,7 @@ class TaskBuilder(val name: String) {
action(from, to) {
PipeAction(
inputType = T::class,
outputType = R::class
outputType = type
) {
//TODO automatically append task meta
result = { data ->
@ -130,7 +130,7 @@ class TaskBuilder(val name: String) {
/**
* Join elements in gathered data by multiple groups
*/
inline fun <reified T : Any, reified R : Any> joinByGroup(
inline fun <reified T : Any> joinByGroup(
from: String = "",
to: String = "",
crossinline block: JoinGroupBuilder<T, R>.(TaskEnv) -> Unit
@ -138,7 +138,7 @@ class TaskBuilder(val name: String) {
action(from, to) {
JoinAction(
inputType = T::class,
outputType = R::class
outputType = type
) { block(this@action) }
}
}
@ -146,7 +146,7 @@ class TaskBuilder(val name: String) {
/**
* Join all elemlents in gathered data matching input type
*/
inline fun <reified T : Any, reified R : Any> join(
inline fun <reified T : Any> join(
from: String = "",
to: String = "",
crossinline block: suspend TaskEnv.(Map<Name, T>) -> R
@ -154,7 +154,7 @@ class TaskBuilder(val name: String) {
action(from, to) {
JoinAction(
inputType = T::class,
outputType = R::class,
outputType = type,
action = {
result(
actionMeta[TaskModel.MODEL_TARGET_KEY]?.string ?: "@anonymous"
@ -169,7 +169,7 @@ class TaskBuilder(val name: String) {
/**
* Split each element in gathered data into fixed number of fragments
*/
inline fun <reified T : Any, reified R : Any> split(
inline fun <reified T : Any> split(
from: String = "",
to: String = "",
crossinline block: SplitBuilder<T, R>.(TaskEnv) -> Unit
@ -177,7 +177,7 @@ class TaskBuilder(val name: String) {
action(from, to) {
SplitAction(
inputType = T::class,
outputType = R::class
outputType = type
) { block(this@action) }
}
}
@ -189,22 +189,23 @@ class TaskBuilder(val name: String) {
this.descriptor = NodeDescriptor.build(transform)
}
internal fun build(): GenericTask<Any> =
internal fun build(): GenericTask<R> =
GenericTask(
name.asName(),
Any::class,
type,
descriptor ?: NodeDescriptor.empty(),
modelTransform
) {
val workspace = this
{ data ->
return@GenericTask { data ->
val model = this
if (dataTransforms.isEmpty()) {
//return data node as is
logger.warn { "No transformation present, returning input data" }
data
data.ensureType(type)
data.cast(type)
} else {
val builder = DataTreeBuilder(Any::class)
val builder = DataTreeBuilder(type)
dataTransforms.forEach { transformation ->
val res = transformation(workspace, model, data)
if (res != null) {
@ -221,12 +222,13 @@ class TaskBuilder(val name: String) {
}
}
fun Workspace.Companion.task(name: String, builder: TaskBuilder.() -> Unit): GenericTask<Any> {
return TaskBuilder(name).apply(builder).build()
fun <T : Any> Workspace.Companion.task(
name: String,
type: KClass<out T>,
builder: TaskBuilder<T>.() -> Unit
): GenericTask<T> {
return TaskBuilder(name, type).apply(builder).build()
}
fun WorkspaceBuilder.task(name: String, builder: TaskBuilder.() -> Unit) {
task(TaskBuilder(name).apply(builder).build())
}
//TODO add delegates to build gradle-like tasks

2
dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskModel.kt
View File

@ -36,7 +36,7 @@ data class TaskModel(
"meta" to meta
"dependsOn" to {
val dataDependencies = dependencies.filterIsInstance<DataDependency>()
val taskDependencies = dependencies.filterIsInstance<TaskDependency>()
val taskDependencies = dependencies.filterIsInstance<TaskDependency<*>>()
setIndexed("data".toName(), dataDependencies.map { it.toMeta() })
setIndexed("task".toName(), taskDependencies.map { it.toMeta() }) { taskDependencies[it].name.toString() }
//TODO ensure all dependencies are listed

66
dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/WorkspaceBuilder.kt
View File

@ -2,12 +2,14 @@ package hep.dataforge.workspace
import hep.dataforge.context.Context
import hep.dataforge.context.ContextBuilder
import hep.dataforge.data.Data
import hep.dataforge.data.DataNode
import hep.dataforge.data.DataTreeBuilder
import hep.dataforge.meta.*
import hep.dataforge.names.EmptyName
import hep.dataforge.names.Name
import hep.dataforge.names.toName
import hep.dataforge.names.isEmpty
import kotlin.jvm.JvmName
import kotlin.reflect.KClass
@TaskBuildScope
interface WorkspaceBuilder {
@ -28,32 +30,26 @@ fun WorkspaceBuilder.context(name: String = "WORKSPACE", block: ContextBuilder.(
context = ContextBuilder(name, parentContext).apply(block).build()
}
fun WorkspaceBuilder.data(name: Name, data: Data<Any>) {
this.data[name] = data
inline fun <reified T : Any> WorkspaceBuilder.data(
name: Name = EmptyName,
noinline block: DataTreeBuilder<T>.() -> Unit
): DataNode<T> {
val node = DataTreeBuilder(T::class).apply(block)
if (name.isEmpty()) {
@Suppress("UNCHECKED_CAST")
data = node as DataTreeBuilder<Any>
} else {
data[name] = node
}
return node.build()
}
fun WorkspaceBuilder.data(name: String, data: Data<Any>) = data(name.toName(), data)
@JvmName("rawData")
fun WorkspaceBuilder.data(
name: Name = EmptyName,
block: DataTreeBuilder<Any>.() -> Unit
): DataNode<Any> = data<Any>(name, block)
fun WorkspaceBuilder.static(name: Name, data: Any, meta: Meta = EmptyMeta) =
data(name, Data.static(data, meta))
fun WorkspaceBuilder.static(name: Name, data: Any, block: MetaBuilder.() -> Unit = {}) =
data(name, Data.static(data, buildMeta(block)))
fun WorkspaceBuilder.static(name: String, data: Any, block: MetaBuilder.() -> Unit = {}) =
data(name, Data.static(data, buildMeta(block)))
fun WorkspaceBuilder.data(name: Name, node: DataNode<Any>) {
this.data[name] = node
}
fun WorkspaceBuilder.data(name: String, node: DataNode<Any>) = data(name.toName(), node)
fun WorkspaceBuilder.data(name: Name, block: DataTreeBuilder<Any>.() -> Unit) {
this.data[name] = DataNode.build(Any::class, block)
}
fun WorkspaceBuilder.data(name: String, block: DataTreeBuilder<Any>.() -> Unit) = data(name.toName(), block)
fun WorkspaceBuilder.target(name: String, block: MetaBuilder.() -> Unit) {
targets[name] = buildMeta(block).seal()
@ -70,17 +66,31 @@ fun WorkspaceBuilder.target(name: String, base: String, block: MetaBuilder.() ->
.seal()
}
fun WorkspaceBuilder.task(task: Task<Any>) {
this.tasks.add(task)
fun <T : Any> WorkspaceBuilder.task(
name: String,
type: KClass<out T>,
builder: TaskBuilder<T>.() -> Unit
): Task<T> {
return TaskBuilder(name, type).apply(builder).build().also { tasks.add(it) }
}
inline fun <reified T : Any> WorkspaceBuilder.task(
name: String,
noinline builder: TaskBuilder<T>.() -> Unit
): Task<T> = task(name, T::class, builder)
@JvmName("rawTask")
fun WorkspaceBuilder.task(
name: String,
builder: TaskBuilder<Any>.() -> Unit
): Task<Any> = task(name, Any::class, builder)
/**
* A builder for a simple workspace
*/
class SimpleWorkspaceBuilder(override val parentContext: Context) : WorkspaceBuilder {
override var context: Context = parentContext
override var data = DataTreeBuilder(Any::class)
override var data: DataTreeBuilder<Any> = DataTreeBuilder(Any::class)
override var tasks: MutableSet<Task<Any>> = HashSet()
override var targets: MutableMap<String, Meta> = HashMap()

4
dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/taskTemplates.kt
View File

@ -1,5 +1,5 @@
package hep.dataforge.workspace
//fun <T1: Any, T2: Any, R: Any> TaskBuilder.zip(
// val firstNo
//) = rawTransform { }
//// val firstNo
////) = rawTransform { }

38
dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/SimpleWorkspaceTest.kt
View File

@ -5,7 +5,7 @@ import hep.dataforge.data.*
import hep.dataforge.meta.boolean
import hep.dataforge.meta.get
import hep.dataforge.names.asName
import kotlin.test.Test
import org.junit.Test
import kotlin.test.assertEquals
import kotlin.test.assertTrue
@ -14,8 +14,8 @@ class SimpleWorkspaceTest {
val testPlugin = object : WorkspacePlugin() {
override val tag: PluginTag = PluginTag("test")
val contextTask = Workspace.task("test") {
pipe<Any, Unit> {
val contextTask = Workspace.task("test", Any::class) {
pipe<Any> {
context.logger.info { "Test: $it" }
}
}
@ -28,8 +28,10 @@ class SimpleWorkspaceTest {
plugin(testPlugin)
}
repeat(100) {
static("myData[$it]", it)
data {
repeat(100) {
static("myData[$it]", it)
}
}
@ -37,7 +39,7 @@ class SimpleWorkspaceTest {
model {
allData()
}
pipe<Int, Int> { data ->
pipe<Int> { data ->
if (meta["testFlag"].boolean == true) {
println("flag")
}
@ -50,7 +52,7 @@ class SimpleWorkspaceTest {
model {
allData()
}
pipe<Int, Int> { data ->
pipe<Int> { data ->
context.logger.info { "Starting linear on $data" }
data * 2 + 1
}
@ -62,16 +64,16 @@ class SimpleWorkspaceTest {
dependsOn("linear", placement = "linear".asName())
}
transform<Any> { data ->
val squareNode = data["square"].withType<Int>().node!!
val linearNode = data["linear"].withType<Int>().node!!
return@transform DataNode.build(Int::class) {
val squareNode = data["square"].filterIsInstance<Int>().node!!
val linearNode = data["linear"].filterIsInstance<Int>().node!!
return@transform DataNode(Int::class) {
squareNode.dataSequence().forEach { (name, _) ->
val newData = Data{
val newData = Data {
val squareValue = squareNode[name].data!!.get()
val linearValue = linearNode[name].data!!.get()
squareValue+linearValue
squareValue + linearValue
}
set(name,newData)
set(name, newData)
}
}
}
@ -81,17 +83,17 @@ class SimpleWorkspaceTest {
model {
dependsOn("square")
}
join<Int, Int> { data ->
join<Int> { data ->
context.logger.info { "Starting sum" }
data.values.sum()
}
}
task("average") {
task<Double>("average") {
model {
allData()
}
joinByGroup<Int, Double> { env ->
joinByGroup<Int> { env ->
group("even", filter = { name, _ -> name.toString().toInt() % 2 == 0 }) {
result { data ->
env.context.logger.info { "Starting even" }
@ -111,7 +113,7 @@ class SimpleWorkspaceTest {
model {
dependsOn("average")
}
join<Double, Double> { data ->
join<Double> { data ->
data["even"]!! - data["odd"]!!
}
}
@ -140,7 +142,7 @@ class SimpleWorkspaceTest {
}
@Test
fun testFullSquare(){
fun testFullSquare() {
val node = workspace.run("fullsquare")
println(node.toMeta())
}