From 012ee93ab2090b2160e5fa843c507c677f8cc614 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Wed, 30 Jan 2019 18:57:14 +0300 Subject: [PATCH] Basic design for workspaces --- .../kotlin/hep/dataforge/context/Context.kt | 14 +- .../kotlin/hep/dataforge/provider/Provider.kt | 8 +- .../kotlin/hep/dataforge/provider/Type.kt | 10 + .../kotlin/hep/dataforge/provider/Types.kt | 26 --- .../kotlin/hep/dataforge/provider/Types.kt | 37 ++++ dataforge-data/build.gradle | 38 ---- dataforge-data/build.gradle.kts | 30 +++ .../kotlin/hep/dataforge/data/Action.kt | 2 +- .../kotlin/hep/dataforge/data/Data.kt | 172 +--------------- .../kotlin/hep/dataforge/data/DataFilter.kt | 46 +++++ .../kotlin/hep/dataforge/data/DataNode.kt | 192 ++++++++++++++++++ .../kotlin/hep/dataforge/data/_Data.kt | 8 + .../kotlin/hep/dataforge/io/TextOutput.kt | 12 +- .../kotlin/hep/dataforge/meta/Meta.kt | 1 + .../hep/dataforge/meta/MutableMetaNode.kt | 3 +- .../hep/dataforge/meta/Specification.kt | 25 ++- .../kotlin/hep/dataforge/names/Name.kt | 4 +- .../kotlin/hep/dataforge/values/Value.kt | 2 +- dataforge-workspace/build.gradle | 25 --- dataforge-workspace/build.gradle.kts | 16 ++ .../hep/dataforge/workspace/Dependency.kt | 51 +++++ .../kotlin/hep/dataforge/workspace/Task.kt | 52 +++++ .../hep/dataforge/workspace/TaskModel.kt | 97 +++++++++ .../hep/dataforge/workspace/Workspace.kt | 55 +++++ settings.gradle.kts | 3 +- 25 files changed, 637 insertions(+), 292 deletions(-) create mode 100644 dataforge-context/src/commonMain/kotlin/hep/dataforge/provider/Type.kt delete mode 100644 dataforge-context/src/commonMain/kotlin/hep/dataforge/provider/Types.kt create mode 100644 dataforge-context/src/jvmMain/kotlin/hep/dataforge/provider/Types.kt delete mode 100644 dataforge-data/build.gradle create mode 100644 dataforge-data/build.gradle.kts create mode 100644 dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataFilter.kt create mode 100644 dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataNode.kt create mode 100644 dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/_Data.kt delete mode 100644 dataforge-workspace/build.gradle create mode 100644 dataforge-workspace/build.gradle.kts create mode 100644 dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/Dependency.kt create mode 100644 dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/Task.kt create mode 100644 dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskModel.kt create mode 100644 dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/Workspace.kt diff --git a/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/Context.kt b/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/Context.kt index a9366c35..63483f91 100644 --- a/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/Context.kt +++ b/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/Context.kt @@ -4,8 +4,6 @@ import hep.dataforge.meta.* import hep.dataforge.names.Name import hep.dataforge.names.toName import hep.dataforge.provider.Provider -import hep.dataforge.provider.Types -import hep.dataforge.provider.provideAll import hep.dataforge.values.Value import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers @@ -54,7 +52,7 @@ interface Context : Named, MetaRepr, Provider, CoroutineScope { override fun provideTop(target: String, name: Name): Any? { return when (target) { Plugin.PLUGIN_TARGET -> plugins[PluginTag.fromString(name.toString())] - Value.VALUE_TARGET -> properties[name]?.value + Value.TYPE -> properties[name]?.value else -> null } } @@ -62,7 +60,7 @@ interface Context : Named, MetaRepr, Provider, CoroutineScope { override fun listTop(target: String): Sequence { return when (target) { Plugin.PLUGIN_TARGET -> plugins.asSequence().map { it.name.toName() } - Value.VALUE_TARGET -> properties.asValueSequence().map { it.first } + Value.TYPE -> properties.asValueSequence().map { it.first } else -> emptySequence() } } @@ -92,15 +90,9 @@ interface Context : Named, MetaRepr, Provider, CoroutineScope { } } -/** - * A sequences of all objects provided by plugins with given target and type - */ -inline fun Context.provideAll(target: String = Types[T::class]): Sequence { - return plugins.asSequence().flatMap { it.provideAll(target) }.filterIsInstance() -} /** - * A global root context + * A global root context. Closing [Global] terminates the framework. */ expect object Global : Context { fun getContext(name: String): Context diff --git a/dataforge-context/src/commonMain/kotlin/hep/dataforge/provider/Provider.kt b/dataforge-context/src/commonMain/kotlin/hep/dataforge/provider/Provider.kt index 967ee43b..79b94dce 100644 --- a/dataforge-context/src/commonMain/kotlin/hep/dataforge/provider/Provider.kt +++ b/dataforge-context/src/commonMain/kotlin/hep/dataforge/provider/Provider.kt @@ -88,10 +88,4 @@ fun Provider.provideAll(target: String): Sequence { return listTop(target).map { provideTop(target, it) ?: error("The element $it is declared but not provided") } } -/** - * Provide an object with given name inferring target from its type using [Type] annotation - */ -inline fun Provider.provideByType(name: String): T? { - val target = Types[T::class] - return provide(target, name) -} + diff --git a/dataforge-context/src/commonMain/kotlin/hep/dataforge/provider/Type.kt b/dataforge-context/src/commonMain/kotlin/hep/dataforge/provider/Type.kt new file mode 100644 index 00000000..a31f1fdb --- /dev/null +++ b/dataforge-context/src/commonMain/kotlin/hep/dataforge/provider/Type.kt @@ -0,0 +1,10 @@ +package hep.dataforge.provider + +/** + * A text label for internal DataForge type classification. Alternative for mime container type. + * + * The DataForge type notation presumes that type `A.B.C` is the subtype of `A.B` + */ +@MustBeDocumented +@Target(AnnotationTarget.CLASS) +annotation class Type(val id: String) diff --git a/dataforge-context/src/commonMain/kotlin/hep/dataforge/provider/Types.kt b/dataforge-context/src/commonMain/kotlin/hep/dataforge/provider/Types.kt deleted file mode 100644 index 97f22aa6..00000000 --- a/dataforge-context/src/commonMain/kotlin/hep/dataforge/provider/Types.kt +++ /dev/null @@ -1,26 +0,0 @@ -package hep.dataforge.provider - -import kotlin.reflect.KClass - -/** - * A text label for internal DataForge type classification. Alternative for mime container type. - * - * The DataForge type notation presumes that type `A.B.C` is the subtype of `A.B` - */ -@MustBeDocumented -@Target(AnnotationTarget.CLASS) -annotation class Type(val id: String) - -/** - * Utils to get type of classes and objects - */ -object Types { - operator fun get(cl: KClass<*>): String { - return cl.annotations.filterIsInstance().firstOrNull()?.id ?: cl.simpleName ?: "" - } - - operator fun get(obj: Any): String { - return get(obj::class) - } -} - diff --git a/dataforge-context/src/jvmMain/kotlin/hep/dataforge/provider/Types.kt b/dataforge-context/src/jvmMain/kotlin/hep/dataforge/provider/Types.kt new file mode 100644 index 00000000..df283776 --- /dev/null +++ b/dataforge-context/src/jvmMain/kotlin/hep/dataforge/provider/Types.kt @@ -0,0 +1,37 @@ +package hep.dataforge.provider + +import hep.dataforge.context.Context +import kotlin.reflect.KClass +import kotlin.reflect.full.findAnnotation + + +object Types { + operator fun get(cl: KClass<*>): String { + return cl.findAnnotation()?.id ?: cl.simpleName ?: "" + } + + operator fun get(obj: Any): String { + return get(obj::class) + } +} + +/** + * Provide an object with given name inferring target from its type using [Type] annotation + */ +inline fun Provider.provideByType(name: String): T? { + val target = Types[T::class] + return provide(target, name) +} + +inline fun Provider.provideAllByType(): Sequence { + val target = Types[T::class] + return provideAll(target).filterIsInstance() +} + +/** + * A sequences of all objects provided by plugins with given target and type + */ +inline fun Context.components(): Sequence { + return plugins.asSequence().flatMap { it.provideAll(Types[T::class]) }.filterIsInstance() +} + diff --git a/dataforge-data/build.gradle b/dataforge-data/build.gradle deleted file mode 100644 index 10205fb1..00000000 --- a/dataforge-data/build.gradle +++ /dev/null @@ -1,38 +0,0 @@ -plugins { - id "org.jetbrains.kotlin.multiplatform" -} - -repositories { - jcenter() -} - -kotlin { - targets { - fromPreset(presets.jvm, 'jvm') - fromPreset(presets.js, 'js') - // For ARM, preset should be changed to presets.iosArm32 or presets.iosArm64 - // For Linux, preset should be changed to e.g. presets.linuxX64 - // For MacOS, preset should be changed to e.g. presets.macosX64 - //fromPreset(presets.iosX64, 'ios') - } - sourceSets { - commonMain { - dependencies { - api project(":dataforge-meta") - api "org.jetbrains.kotlinx:kotlinx-coroutines-core-common:$coroutinesVersion" - } - } - - jvmMain { - dependencies { - api "org.jetbrains.kotlinx:kotlinx-coroutines-core:$coroutinesVersion" - } - } - - jsMain { - dependencies { - api "org.jetbrains.kotlinx:kotlinx-coroutines-core-js:$coroutinesVersion" - } - } - } -} \ No newline at end of file diff --git a/dataforge-data/build.gradle.kts b/dataforge-data/build.gradle.kts new file mode 100644 index 00000000..eb6b3669 --- /dev/null +++ b/dataforge-data/build.gradle.kts @@ -0,0 +1,30 @@ +plugins { + kotlin("multiplatform") +} + +val coroutinesVersion: String by rootProject.extra + +kotlin { + jvm() + js() + sourceSets { + val commonMain by getting{ + dependencies { + api(project(":dataforge-meta")) + api("org.jetbrains.kotlinx:kotlinx-coroutines-core-common:$coroutinesVersion") + } + } + + val jvmMain by getting{ + dependencies { + api("org.jetbrains.kotlinx:kotlinx-coroutines-core:$coroutinesVersion") + } + } + + val jsMain by getting{ + dependencies { + api("org.jetbrains.kotlinx:kotlinx-coroutines-core-js:$coroutinesVersion") + } + } + } +} \ No newline at end of file diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/Action.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/Action.kt index 2a8e2fb6..599101c0 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/Action.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/Action.kt @@ -38,7 +38,7 @@ infix fun Action.then(action: Action): A */ class PipeAction(val transform: (Name, Data, Meta) -> Data?) : Action { override fun invoke(node: DataNode, meta: Meta): DataNode = DataNode.build { - node.asSequence().forEach { (name, data) -> + node.dataSequence().forEach { (name, data) -> val res = transform(name, data, meta) if (res != null) { set(name, res) diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/Data.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/Data.kt index 6d74a788..6096a799 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/Data.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/Data.kt @@ -2,10 +2,6 @@ package hep.dataforge.data import hep.dataforge.meta.Meta import hep.dataforge.meta.MetaRepr -import hep.dataforge.names.Name -import hep.dataforge.names.NameToken -import hep.dataforge.names.plus -import hep.dataforge.names.toName import kotlin.coroutines.CoroutineContext import kotlin.reflect.KClass @@ -30,6 +26,8 @@ interface Data : MetaRepr { override fun toMeta(): Meta = meta companion object { + const val TYPE = "data" + fun of(type: KClass, goal: Goal, meta: Meta): Data = DataImpl(type, goal, meta) inline fun of(goal: Goal, meta: Meta): Data = of(T::class, goal, meta) fun of(name: String, type: KClass, goal: Goal, meta: Meta): Data = @@ -43,6 +41,8 @@ interface Data : MetaRepr { } } +suspend fun Data.await(): T = goal.await() + /** * Generic Data implementation */ @@ -54,167 +54,3 @@ private class DataImpl( class NamedData(val name: String, data: Data) : Data by data -/** - * A tree-like data structure grouped into the node. All data inside the node must inherit its type - */ -interface DataNode { - /** - * Get the specific data if it exists - */ - operator fun get(name: Name): Data? - - /** - * Get a subnode with given name if it exists. - */ - fun getNode(name: Name): DataNode? - - /** - * Walk the tree upside down and provide all data nodes with full names - */ - fun asSequence(): Sequence>> - - operator fun iterator(): Iterator>> = asSequence().iterator() - - companion object { - fun build(block: DataTreeBuilder.() -> Unit) = DataTreeBuilder().apply(block).build() - } - -} - -internal sealed class DataTreeItem { - class Node(val tree: DataTree) : DataTreeItem() - class Value(val value: Data) : DataTreeItem() -} - -class DataTree internal constructor(private val items: Map>) : DataNode { - //TODO add node-level meta? - - override fun get(name: Name): Data? = when (name.length) { - 0 -> error("Empty name") - 1 -> (items[name.first()] as? DataTreeItem.Value)?.value - else -> getNode(name.first()!!.toName())?.get(name.cutFirst()) - } - - override fun getNode(name: Name): DataTree? = when (name.length) { - 0 -> this - 1 -> (items[name.first()] as? DataTreeItem.Node)?.tree - else -> getNode(name.first()!!.toName())?.getNode(name.cutFirst()) - } - - override fun asSequence(): Sequence>> { - return kotlin.sequences.sequence { - items.forEach { (head, tree) -> - when (tree) { - is DataTreeItem.Value -> yield(head.toName() to tree.value) - is DataTreeItem.Node -> { - val subSequence = tree.tree.asSequence().map { (name, data) -> (head.toName() + name) to data } - yieldAll(subSequence) - } - } - } - } - } -} - -private sealed class DataTreeBuilderItem { - class Node(val tree: DataTreeBuilder) : DataTreeBuilderItem() - class Value(val value: Data) : DataTreeBuilderItem() -} - -/** - * A builder for a DataTree. - */ -class DataTreeBuilder { - private val map = HashMap>() - - operator fun set(token: NameToken, node: DataTreeBuilder) { - if (map.containsKey(token)) error("Tree entry with name $token is not empty") - map[token] = DataTreeBuilderItem.Node(node) - } - - operator fun set(token: NameToken, data: Data) { - if (map.containsKey(token)) error("Tree entry with name $token is not empty") - map[token] = DataTreeBuilderItem.Value(data) - } - - private fun buildNode(token: NameToken): DataTreeBuilder { - return if (!map.containsKey(token)) { - DataTreeBuilder().also { map[token] = DataTreeBuilderItem.Node(it) } - } else { - (map[token] as? DataTreeBuilderItem.Node ?: error("The node with name $token is occupied by leaf")).tree - } - } - - private fun buildNode(name: Name): DataTreeBuilder { - return when (name.length) { - 0 -> this - 1 -> buildNode(name.first()!!) - else -> buildNode(name.first()!!).buildNode(name.cutFirst()) - } - } - - operator fun set(name: Name, data: Data) { - when (name.length) { - 0 -> error("Can't add data with empty name") - 1 -> set(name.first()!!, data) - 2 -> buildNode(name.cutLast())[name.last()!!] = data - } - } - - operator fun set(name: Name, node: DataTreeBuilder) { - when (name.length) { - 0 -> error("Can't add data with empty name") - 1 -> set(name.first()!!, node) - 2 -> buildNode(name.cutLast())[name.last()!!] = node - } - } - - operator fun set(name: Name, node: DataNode) = set(name, node.builder()) - - /** - * Append data to node - */ - infix fun String.to(data: Data) = set(toName(), data) - - /** - * Append node - */ - infix fun String.to(node: DataNode) = set(toName(), node) - - /** - * Build and append node - */ - infix fun String.to(block: DataTreeBuilder.() -> Unit) = set(toName(), DataTreeBuilder().apply(block)) - - fun build(): DataTree { - val resMap = map.mapValues { (_, value) -> - when (value) { - is DataTreeBuilderItem.Value -> DataTreeItem.Value(value.value) - is DataTreeBuilderItem.Node -> DataTreeItem.Node(value.tree.build()) - } - } - return DataTree(resMap) - } -} - -/** - * Generate a mutable builder from this node. Node content is not changed - */ -fun DataNode.builder(): DataTreeBuilder = DataTreeBuilder().apply { - asSequence().forEach { (name, data) -> this[name] = data } -} - -/** - * Start computation for all goals in data node - */ -fun DataNode<*>.startAll() = asSequence().forEach { (_, data) -> data.goal.start() } - -fun DataNode.filter(predicate: (Name, Data) -> Boolean): DataNode = DataNode.build { - asSequence().forEach { (name, data) -> - if (predicate(name, data)) { - this[name] = data - } - } -} - -//fun DataNode.filterIsInstance(type: KClass): DataNode = filter{_,data -> type.} \ No newline at end of file diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataFilter.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataFilter.kt new file mode 100644 index 00000000..6f230c0f --- /dev/null +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataFilter.kt @@ -0,0 +1,46 @@ +package hep.dataforge.data + +import hep.dataforge.meta.* +import hep.dataforge.names.toName + + +class DataFilter(override val config: Config) : Specification { + var from by string() + var to by string() + var pattern by string("*.") +// val prefix by string() +// val suffix by string() + + companion object : SpecificationCompanion { + override fun wrap(config: Config): DataFilter = DataFilter(config) + } +} + +/** + * Apply meta-based filter to given data node + */ +fun DataNode.filter(filter: DataFilter): DataNode { + val sourceNode = filter.from?.let { getNode(it.toName()) } ?: this@filter + val regex = filter.pattern.toRegex() + val targetNode = DataTreeBuilder().apply { + sourceNode.dataSequence().forEach { (name, data) -> + if (name.toString().matches(regex)) { + this[name] = data + } + } + } + return filter.to?.let { + DataTreeBuilder().apply { this[it.toName()] = targetNode }.build() + } ?: targetNode.build() +} + +/** + * Filter data using [DataFilter] specification + */ +fun DataNode.filter(filter: Meta): DataNode = filter(DataFilter.wrap(filter)) + +/** + * Filter data using [DataFilter] builder + */ +fun DataNode.filter(filterBuilder: DataFilter.() -> Unit): DataNode = + filter(DataFilter.build(filterBuilder)) \ No newline at end of file diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataNode.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataNode.kt new file mode 100644 index 00000000..bf8f6f81 --- /dev/null +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataNode.kt @@ -0,0 +1,192 @@ +package hep.dataforge.data + +import hep.dataforge.names.Name +import hep.dataforge.names.NameToken +import hep.dataforge.names.plus +import hep.dataforge.names.toName + +/** + * A tree-like data structure grouped into the node. All data inside the node must inherit its type + */ +interface DataNode { + /** + * Get the specific data if it exists + */ + operator fun get(name: Name): Data? + + /** + * Get a subnode with given name if it exists. + */ + fun getNode(name: Name): DataNode? + + /** + * Walk the tree upside down and provide all data nodes with full names + */ + fun dataSequence(): Sequence>> + + /** + * A sequence of all nodes in the tree walking upside down, excluding self + */ + fun nodeSequence(): Sequence>> + + operator fun iterator(): Iterator>> = dataSequence().iterator() + + companion object { + const val TYPE = "dataNode" + + fun build(block: DataTreeBuilder.() -> Unit) = DataTreeBuilder().apply(block).build() + } + +} + +internal sealed class DataTreeItem { + class Node(val tree: DataTree) : DataTreeItem() + class Value(val value: Data) : DataTreeItem() +} + +class DataTree internal constructor(private val items: Map>) : DataNode { + //TODO add node-level meta? + + override fun get(name: Name): Data? = when (name.length) { + 0 -> error("Empty name") + 1 -> (items[name.first()] as? DataTreeItem.Value)?.value + else -> getNode(name.first()!!.toName())?.get(name.cutFirst()) + } + + override fun getNode(name: Name): DataTree? = when (name.length) { + 0 -> this + 1 -> (items[name.first()] as? DataTreeItem.Node)?.tree + else -> getNode(name.first()!!.toName())?.getNode(name.cutFirst()) + } + + override fun dataSequence(): Sequence>> { + return sequence { + items.forEach { (head, tree) -> + when (tree) { + is DataTreeItem.Value -> yield(head.toName() to tree.value) + is DataTreeItem.Node -> { + val subSequence = + tree.tree.dataSequence().map { (name, data) -> (head.toName() + name) to data } + yieldAll(subSequence) + } + } + } + } + } + + override fun nodeSequence(): Sequence>> { + return sequence { + items.forEach { (head, tree) -> + if (tree is DataTreeItem.Node) { + yield(head.toName() to tree.tree) + val subSequence = + tree.tree.nodeSequence().map { (name, node) -> (head.toName() + name) to node } + yieldAll(subSequence) + } + } + } + } +} + +private sealed class DataTreeBuilderItem { + class Node(val tree: DataTreeBuilder) : DataTreeBuilderItem() + class Value(val value: Data) : DataTreeBuilderItem() +} + +/** + * A builder for a DataTree. + */ +class DataTreeBuilder { + private val map = HashMap>() + + operator fun set(token: NameToken, node: DataTreeBuilder) { + if (map.containsKey(token)) error("Tree entry with name $token is not empty") + map[token] = DataTreeBuilderItem.Node(node) + } + + operator fun set(token: NameToken, data: Data) { + if (map.containsKey(token)) error("Tree entry with name $token is not empty") + map[token] = DataTreeBuilderItem.Value(data) + } + + private fun buildNode(token: NameToken): DataTreeBuilder { + return if (!map.containsKey(token)) { + DataTreeBuilder().also { map[token] = DataTreeBuilderItem.Node(it) } + } else { + (map[token] as? DataTreeBuilderItem.Node ?: error("The node with name $token is occupied by leaf")).tree + } + } + + private fun buildNode(name: Name): DataTreeBuilder { + return when (name.length) { + 0 -> this + 1 -> buildNode(name.first()!!) + else -> buildNode(name.first()!!).buildNode(name.cutFirst()) + } + } + + operator fun set(name: Name, data: Data) { + when (name.length) { + 0 -> error("Can't add data with empty name") + 1 -> set(name.first()!!, data) + 2 -> buildNode(name.cutLast())[name.last()!!] = data + } + } + + operator fun set(name: Name, node: DataTreeBuilder) { + when (name.length) { + 0 -> error("Can't add data with empty name") + 1 -> set(name.first()!!, node) + 2 -> buildNode(name.cutLast())[name.last()!!] = node + } + } + + operator fun set(name: Name, node: DataNode) = set(name, node.builder()) + + /** + * Append data to node + */ + infix fun String.to(data: Data) = set(toName(), data) + + /** + * Append node + */ + infix fun String.to(node: DataNode) = set(toName(), node) + + /** + * Build and append node + */ + infix fun String.to(block: DataTreeBuilder.() -> Unit) = set(toName(), DataTreeBuilder().apply(block)) + + fun build(): DataTree { + val resMap = map.mapValues { (_, value) -> + when (value) { + is DataTreeBuilderItem.Value -> DataTreeItem.Value(value.value) + is DataTreeBuilderItem.Node -> DataTreeItem.Node(value.tree.build()) + } + } + return DataTree(resMap) + } +} + +/** + * Generate a mutable builder from this node. Node content is not changed + */ +fun DataNode.builder(): DataTreeBuilder = DataTreeBuilder().apply { + dataSequence().forEach { (name, data) -> this[name] = data } +} + +/** + * Start computation for all goals in data node + */ +fun DataNode<*>.startAll() = dataSequence().forEach { (_, data) -> data.goal.start() } + +fun DataNode.filter(predicate: (Name, Data) -> Boolean): DataNode = DataNode.build { + dataSequence().forEach { (name, data) -> + if (predicate(name, data)) { + this[name] = data + } + } +} + +//fun DataNode.filterIsInstance(type: KClass): DataNode = filter{_,data -> type.} \ No newline at end of file diff --git a/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/_Data.kt b/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/_Data.kt new file mode 100644 index 00000000..00c9e656 --- /dev/null +++ b/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/_Data.kt @@ -0,0 +1,8 @@ +package hep.dataforge.data + +import kotlinx.coroutines.runBlocking + +/** + * Block the thread and get data content + */ +fun Data.get(): T = runBlocking { await() } \ No newline at end of file diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TextOutput.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TextOutput.kt index f094eff4..446eed31 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TextOutput.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TextOutput.kt @@ -1,8 +1,10 @@ package hep.dataforge.io import hep.dataforge.context.Context -import hep.dataforge.context.provideAll +import hep.dataforge.io.TextRenderer.Companion.TEXT_RENDERER_TYPE import hep.dataforge.meta.Meta +import hep.dataforge.provider.Type +import hep.dataforge.provider.provideAll import kotlinx.coroutines.launch import kotlin.reflect.KClass @@ -18,7 +20,8 @@ class TextOutput(override val context: Context, private val output: kotlinx.io.c } else { val value = cache[obj::class] if (value == null) { - val answer = context.provideAll().filter { it.type.isInstance(obj) }.firstOrNull() + val answer = context.provideAll(TEXT_RENDERER_TYPE).filterIsInstance() + .filter { it.type.isInstance(obj) }.firstOrNull() if (answer != null) { cache[obj::class] = answer answer @@ -35,6 +38,7 @@ class TextOutput(override val context: Context, private val output: kotlinx.io.c } } +@Type(TEXT_RENDERER_TYPE) interface TextRenderer { /** * The priority of this renderer compared to other renderers @@ -46,6 +50,10 @@ interface TextRenderer { val type: KClass<*> suspend fun kotlinx.io.core.Output.render(obj: Any) + + companion object { + const val TEXT_RENDERER_TYPE = "dataforge.textRenderer" + } } object DefaultTextRenderer : TextRenderer { diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Meta.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Meta.kt index 56590dbd..bcc777c1 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Meta.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Meta.kt @@ -42,6 +42,7 @@ interface Meta : MetaRepr { override fun toMeta(): Meta = this companion object { + const val TYPE = "meta" /** * A key for single value node */ diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MutableMetaNode.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MutableMetaNode.kt index fe48a299..31998b46 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MutableMetaNode.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MutableMetaNode.kt @@ -161,7 +161,8 @@ fun > M.setIndexed( metas: Iterable, queryFactory: (Int) -> String = { it.toString() } ) { - setIndexed(name, metas.map { wrap(name, it) }, queryFactory) + setIndexed(name, metas.map { MetaItem.NodeItem(wrap(name, it)) }, queryFactory) } operator fun > M.set(name: Name, metas: Iterable) = setIndexed(name, metas) +operator fun > M.set(name: String, metas: Iterable) = setIndexed(name.toName(), metas) diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Specification.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Specification.kt index bdd11429..fb3e909f 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Specification.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Specification.kt @@ -8,47 +8,52 @@ interface Specification : Configurable { } /** - * Specification allows to apply custom configuration in a type safe way to simple untyped configuration + * Allows to apply custom configuration in a type safe way to simple untyped configuration. + * By convention [Specification] companion should inherit this class + * */ -interface SpecificationBuilder { +interface SpecificationCompanion { /** * Update given configuration using given type as a builder */ - fun update(config: Config, action: T.() -> Unit) { - wrap(config).apply(action) + fun update(config: Config, action: T.() -> Unit): T { + return wrap(config).apply(action) } + fun build(action: T.() -> Unit) = update(Config(), action) + /** * Wrap generic configuration producing instance of desired type */ fun wrap(config: Config): T fun wrap(meta: Meta): T = wrap(meta.toConfig()) + } -fun specification(wrapper: (Config) -> T): SpecificationBuilder = - object : SpecificationBuilder { +fun specification(wrapper: (Config) -> T): SpecificationCompanion = + object : SpecificationCompanion { override fun wrap(config: Config): T = wrapper(config) } /** * Apply specified configuration to configurable */ -fun > T.configure(spec: S, action: C.() -> Unit) = +fun > T.configure(spec: S, action: C.() -> Unit) = apply { spec.update(config, action) } /** * Update configuration using given specification */ -fun > Specification.update(spec: S, action: C.() -> Unit) = +fun > Specification.update(spec: S, action: C.() -> Unit) = apply { spec.update(config, action) } /** * Create a style based on given specification */ -fun > S.createStyle(action: C.() -> Unit): Meta = +fun > S.createStyle(action: C.() -> Unit): Meta = Config().also { update(it, action) } -fun Specification.spec(spec: SpecificationBuilder, key: String? = null) = +fun Specification.spec(spec: SpecificationCompanion, key: String? = null) = ChildConfigDelegate(key) { spec.wrap(config) } \ No newline at end of file diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/names/Name.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/names/Name.kt index b162e5f3..78dba871 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/names/Name.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/names/Name.kt @@ -103,4 +103,6 @@ operator fun Name.plus(other: String): Name = this + other.toName() fun NameToken.toName() = Name(listOf(this)) -val EmptyName = Name(emptyList()) \ No newline at end of file +val EmptyName = Name(emptyList()) + +fun Name.isEmpty(): Boolean = this.length == 0 \ No newline at end of file diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/values/Value.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/values/Value.kt index 53cc1996..2f92bd60 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/values/Value.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/values/Value.kt @@ -44,7 +44,7 @@ interface Value { get() = listOf(this) companion object { - const val VALUE_TARGET = "value" + const val TYPE = "value" /** * Convert object to value diff --git a/dataforge-workspace/build.gradle b/dataforge-workspace/build.gradle deleted file mode 100644 index a0011494..00000000 --- a/dataforge-workspace/build.gradle +++ /dev/null @@ -1,25 +0,0 @@ -plugins { - id "org.jetbrains.kotlin.multiplatform" -} - -repositories { - jcenter() -} - -kotlin { - targets { - fromPreset(presets.jvm, 'jvm') - //fromPreset(presets.js, 'js') - // For ARM, preset should be changed to presets.iosArm32 or presets.iosArm64 - // For Linux, preset should be changed to e.g. presets.linuxX64 - // For MacOS, preset should be changed to e.g. presets.macosX64 - //fromPreset(presets.iosX64, 'ios') - } - sourceSets { - commonMain { - dependencies { - api project(":dataforge-context") - } - } - } -} \ No newline at end of file diff --git a/dataforge-workspace/build.gradle.kts b/dataforge-workspace/build.gradle.kts new file mode 100644 index 00000000..cb2ce82c --- /dev/null +++ b/dataforge-workspace/build.gradle.kts @@ -0,0 +1,16 @@ +plugins { + kotlin("multiplatform") +} + +kotlin { + jvm() + js() + sourceSets { + val commonMain by getting{ + dependencies { + api(project(":dataforge-context")) + api(project(":dataforge-data")) + } + } + } +} \ No newline at end of file diff --git a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/Dependency.kt b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/Dependency.kt new file mode 100644 index 00000000..0fedc3e9 --- /dev/null +++ b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/Dependency.kt @@ -0,0 +1,51 @@ +package hep.dataforge.workspace + +import hep.dataforge.data.DataFilter +import hep.dataforge.data.DataNode +import hep.dataforge.data.DataTreeBuilder +import hep.dataforge.data.filter +import hep.dataforge.meta.Meta +import hep.dataforge.meta.MetaRepr +import hep.dataforge.meta.buildMeta +import hep.dataforge.names.EmptyName +import hep.dataforge.names.Name +import hep.dataforge.names.isEmpty + +/** + * A dependency of the task which allows to lazily create a data tree for single dependency + */ +sealed class Dependency : MetaRepr { + abstract fun apply(workspace: Workspace): DataNode +} + +class DataDependency(val filter: DataFilter) : Dependency() { + override fun apply(workspace: Workspace): DataNode = + workspace.data.filter(filter) + + override fun toMeta(): Meta = filter.config + + companion object { + val all: DataDependency = DataDependency(DataFilter.build { }) + } +} + +class TaskModelDependency(val name: String, val meta: Meta, val placement: Name = EmptyName) : Dependency() { + override fun apply(workspace: Workspace): DataNode { + val model = + workspace.tasks[name]?.build(workspace, meta) ?: error("Task with name $name not found in $workspace") + + val task = workspace.tasks[model.name] ?: error("Task with name ${model.name} is not found in the workspace") + if (task.isTerminal) TODO("Support terminal task") + val result = task.run(model) + return if (placement.isEmpty()) { + result + } else { + DataTreeBuilder().apply { this[placement] = result }.build() + } + } + + override fun toMeta(): Meta = buildMeta { + "name" to name + "meta" to meta + } +} \ No newline at end of file diff --git a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/Task.kt b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/Task.kt new file mode 100644 index 00000000..bb6d33f3 --- /dev/null +++ b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/Task.kt @@ -0,0 +1,52 @@ +package hep.dataforge.workspace + +import hep.dataforge.context.Named +import hep.dataforge.data.DataNode +import hep.dataforge.meta.Meta +import hep.dataforge.provider.Type +import hep.dataforge.workspace.Task.Companion.TYPE +import kotlin.reflect.KClass + +@Type(TYPE) +interface Task : Named { + /** + * Terminal task is the one that could not build model lazily + */ + val isTerminal: Boolean get() = false + + /** + * The explicit type of the node returned by the task + */ + val type: KClass + + /** + * Build a model for this task + * + * @param workspace + * @param taskConfig + * @return + */ + fun build(workspace: Workspace, taskConfig: Meta): TaskModel + + /** + * Check if the model is valid and is acceptable by the task. Throw exception if not. + * + * @param model + */ + fun validate(model: TaskModel) { + if(this.name != model.name) error("The task $name could not be run with model from task ${model.name}") + } + + /** + * Run given task model. Type check expected to be performed before actual + * calculation. + * + * @param model + * @return + */ + fun run(model: TaskModel): DataNode + + companion object { + const val TYPE = "task" + } +} \ No newline at end of file diff --git a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskModel.kt b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskModel.kt new file mode 100644 index 00000000..8d0de0c0 --- /dev/null +++ b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskModel.kt @@ -0,0 +1,97 @@ +/* + * To change this license header, choose License Headers in Project Properties. + * To change this template file, choose Tools | Templates + * and open the template in the editor. + */ +package hep.dataforge.workspace + +import hep.dataforge.data.DataFilter +import hep.dataforge.data.DataTree +import hep.dataforge.data.DataTreeBuilder +import hep.dataforge.meta.* +import hep.dataforge.names.EmptyName +import hep.dataforge.names.Name +import hep.dataforge.names.toName + + +/** + * A model for task execution + * @param name the name of the task + * @param meta the meta for the task (not for the whole configuration) + * @param dependencies a list of direct dependencies for this task + */ +data class TaskModel( + val name: String, + val meta: Meta, + val dependencies: Collection +) : MetaRepr { + //TODO provide a way to get task descriptor + //TODO add pre-run check of task result type? + + override fun toMeta(): Meta = buildMeta { + "name" to name + "meta" to meta + "dependsOn" to { + val dataDependencies = dependencies.filterIsInstance() + val taskDependencies = dependencies.filterIsInstance() + setIndexed("data".toName(), dataDependencies.map { it.toMeta() }) + setIndexed("task".toName(), taskDependencies.map { it.toMeta() }) { taskDependencies[it].name } + //TODO ensure all dependencies are listed + } + } +} + +/** + * Build input for the task + */ +fun TaskModel.buildInput(workspace: Workspace): DataTree { + return DataTreeBuilder().apply { + dependencies.asSequence().flatMap { it.apply(workspace).dataSequence() }.forEach { (name, data) -> + //TODO add concise error on replacement + this[name] = data + } + }.build() +} + +/** + * A builder for [TaskModel] + */ +class TaskModelBuilder(val name: String, meta: Meta = EmptyMeta) { + /** + * Meta for current task. By default uses the whole input meta + */ + var meta: MetaBuilder = meta.builder() + val dependencies = HashSet() + + /** + * Add dependency for + */ + fun dependsOn(name: String, meta: Meta, placement: Name = EmptyName) { + dependencies.add(TaskModelDependency(name, meta, placement)) + } + + /** + * Add custom data dependency + */ + fun data(action: DataFilter.() -> Unit) { + dependencies.add(DataDependency(DataFilter.build(action))) + } + + /** + * User-friendly way to add data dependency + */ + fun data(pattern: String? = null, from: String? = null, to: String? = null) = data { + pattern?.let { this.pattern = it } + from?.let { this.from = it } + to?.let { this.to = it } + } + + /** + * Add all data as root node + */ + fun allData() { + dependencies.add(DataDependency.all) + } + + fun build(): TaskModel = TaskModel(name, meta.seal(), dependencies) +} diff --git a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/Workspace.kt b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/Workspace.kt new file mode 100644 index 00000000..a7fbfea7 --- /dev/null +++ b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/Workspace.kt @@ -0,0 +1,55 @@ +package hep.dataforge.workspace + +import hep.dataforge.context.ContextAware +import hep.dataforge.context.Named +import hep.dataforge.data.Data +import hep.dataforge.data.DataNode +import hep.dataforge.meta.Meta +import hep.dataforge.names.Name +import hep.dataforge.names.toName +import hep.dataforge.provider.Provider +import hep.dataforge.provider.Type + + +@Type(Workspace.TYPE) +interface Workspace : ContextAware, Named, Provider { + /** + * The whole data node for current workspace + */ + val data: DataNode + + /** + * All targets associated with the workspace + */ + val targets: Map + + /** + * All tasks associated with the workspace + */ + val tasks: Map> + + override fun provideTop(target: String, name: Name): Any? { + return when (target) { + "target", Meta.TYPE -> targets[name.toString()] + Task.TYPE -> tasks[name.toString()] + Data.TYPE -> data[name] + DataNode.TYPE -> data.getNode(name) + else -> null + } + } + + override fun listTop(target: String): Sequence { + return when (target) { + "target", Meta.TYPE -> targets.keys.asSequence().map { it.toName() } + Task.TYPE -> tasks.keys.asSequence().map { it.toName() } + Data.TYPE -> data.dataSequence().map { it.first } + DataNode.TYPE -> data.nodeSequence().map { it.first } + else -> emptySequence() + } + } + + companion object { + const val TYPE = "workspace" + } +} + diff --git a/settings.gradle.kts b/settings.gradle.kts index 216dd001..670d7438 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -13,5 +13,6 @@ include( ":dataforge-meta-io", ":dataforge-context", ":dataforge-data", - ":dataforge-io" + ":dataforge-io", + ":dataforge-workspace" )