WIP full data refactor
This commit is contained in:
parent
23fae9794f
commit
b916a038f7
@ -44,7 +44,7 @@ public interface Data<out T : Any> : Goal<T>, MetaRepr {
|
||||
/**
|
||||
* An empty data containing only meta
|
||||
*/
|
||||
public fun empty(meta: Meta): Data<Nothing> = object : Data<Nothing> {
|
||||
public fun <T> empty(meta: Meta): Data<Nothing> = object : Data<Nothing> {
|
||||
override val type: KClass<out Nothing> = Nothing::class
|
||||
override val meta: Meta = meta
|
||||
override val dependencies: Collection<Goal<*>> = emptyList()
|
||||
|
@ -67,6 +67,8 @@ public interface DataTree<out T : Any> : DataSet<T> {
|
||||
}
|
||||
}
|
||||
|
||||
public suspend fun <T: Any> DataSet<T>.getData(name: String): Data<T>? = getData(name.toName())
|
||||
|
||||
/**
|
||||
* Get a [DataTreeItem] with given [name] or null if the item does not exist
|
||||
*/
|
||||
@ -97,10 +99,10 @@ public fun <T : Any> DataTree<T>.itemFlow(): Flow<Pair<Name, DataTreeItem<T>>> =
|
||||
* Get a branch of this [DataTree] with a given [branchName].
|
||||
* The difference from similar method for [DataSet] is that internal logic is more simple and the return value is a [DataTree]
|
||||
*/
|
||||
public operator fun <T : Any> DataTree<T>.get(branchName: Name): DataTree<T> = object : DataTree<T> {
|
||||
override val dataType: KClass<out T> get() = this@get.dataType
|
||||
public fun <T : Any> DataTree<T>.branch(branchName: Name): DataTree<T> = object : DataTree<T> {
|
||||
override val dataType: KClass<out T> get() = this@branch.dataType
|
||||
|
||||
override val updates: Flow<Name> = this@get.updates.mapNotNull { it.removeHeadOrNull(branchName) }
|
||||
override val updates: Flow<Name> = this@branch.updates.mapNotNull { it.removeHeadOrNull(branchName) }
|
||||
|
||||
override suspend fun items(): Map<NameToken, DataTreeItem<T>> = getItem(branchName).tree?.items() ?: emptyMap()
|
||||
}
|
||||
|
@ -154,6 +154,3 @@ public suspend fun <T : Any> DataSet<T>.toMutableTree(
|
||||
set(it, getData(it))
|
||||
}.launchIn(scope)
|
||||
}
|
||||
|
||||
public fun <T : Any> MutableDataTree<T>.get(branchName: Name): MutableDataTree<T> =
|
||||
(this as DataTree<T>).get(branchName) as MutableDataTree<T>
|
||||
|
@ -51,22 +51,22 @@ else object : DataSet<T> {
|
||||
/**
|
||||
* Get a subset of data starting with a given [branchName]
|
||||
*/
|
||||
public operator fun <T : Any> DataSet<T>.get(branchName: Name): DataSet<T> = if (branchName.isEmpty()) this
|
||||
public fun <T : Any> DataSet<T>.branch(branchName: Name): DataSet<T> = if (branchName.isEmpty()) this
|
||||
else object : DataSet<T> {
|
||||
override val dataType: KClass<out T> get() = this@get.dataType
|
||||
override val dataType: KClass<out T> get() = this@branch.dataType
|
||||
|
||||
override fun flow(): Flow<NamedData<T>> = this@get.flow().mapNotNull {
|
||||
override fun flow(): Flow<NamedData<T>> = this@branch.flow().mapNotNull {
|
||||
it.name.removeHeadOrNull(branchName)?.let { name ->
|
||||
it.data.named(name)
|
||||
}
|
||||
}
|
||||
|
||||
override suspend fun getData(name: Name): Data<T>? = this@get.getData(branchName + name)
|
||||
override suspend fun getData(name: Name): Data<T>? = this@branch.getData(branchName + name)
|
||||
|
||||
override val updates: Flow<Name> get() = this@get.updates.mapNotNull { it.removeHeadOrNull(branchName) }
|
||||
override val updates: Flow<Name> get() = this@branch.updates.mapNotNull { it.removeHeadOrNull(branchName) }
|
||||
}
|
||||
|
||||
public operator fun <T : Any> DataSet<T>.get(branchName: String): DataSet<T> = this@get.get(branchName.toName())
|
||||
public fun <T : Any> DataSet<T>.branch(branchName: String): DataSet<T> = this@branch.branch(branchName.toName())
|
||||
|
||||
@DFExperimental
|
||||
public suspend fun <T : Any> DataSet<T>.rootData(): Data<T>? = getData(Name.EMPTY)
|
||||
|
@ -27,8 +27,8 @@ internal class DataTreeBuilderTest{
|
||||
}
|
||||
|
||||
|
||||
assertTrue { node["update.a"] != null }
|
||||
assertTrue { node["primary.a"] != null }
|
||||
assertTrue { node.branch("update.a") != null }
|
||||
assertTrue { node.branch("primary.a") != null }
|
||||
|
||||
}
|
||||
}
|
@ -57,7 +57,7 @@ public class TaskBuilder<R : Any>(private val name: Name, public val type: KClas
|
||||
) {
|
||||
dataTransforms += { context, model, data ->
|
||||
val env = TaskEnv(Name.EMPTY, model.meta, context, data)
|
||||
val startData = data.get(from)
|
||||
val startData = data.branch(from)
|
||||
env.block(startData).withNamePrefix(to)
|
||||
}
|
||||
}
|
||||
|
@ -3,10 +3,8 @@ package hep.dataforge.workspace
|
||||
import hep.dataforge.context.Context
|
||||
import hep.dataforge.context.ContextBuilder
|
||||
import hep.dataforge.context.Global
|
||||
import hep.dataforge.data.DataTree
|
||||
import hep.dataforge.data.MutableDataTree
|
||||
import hep.dataforge.meta.*
|
||||
import hep.dataforge.names.Name
|
||||
import hep.dataforge.names.toName
|
||||
import kotlin.reflect.KClass
|
||||
|
||||
@ -28,19 +26,11 @@ public fun WorkspaceBuilder.context(name: String = "WORKSPACE", block: ContextBu
|
||||
context = ContextBuilder(parentContext, name).apply(block).build()
|
||||
}
|
||||
|
||||
public inline fun <reified T : Any> WorkspaceBuilder.data(
|
||||
name: Name = Name.EMPTY,
|
||||
noinline block: MutableDataTree<T>.() -> Unit,
|
||||
): DataTree<T> {
|
||||
TODO()
|
||||
//data.branch(name).apply(block)
|
||||
}
|
||||
|
||||
@JvmName("rawData")
|
||||
public fun WorkspaceBuilder.data(
|
||||
name: Name = Name.EMPTY,
|
||||
public inline fun WorkspaceBuilder.data(
|
||||
block: MutableDataTree<Any>.() -> Unit,
|
||||
): DataTree<Any> = data<Any>(name, block)
|
||||
): Unit{
|
||||
data.apply(block)
|
||||
}
|
||||
|
||||
|
||||
public fun WorkspaceBuilder.target(name: String, block: MetaBuilder.() -> Unit) {
|
||||
@ -80,7 +70,7 @@ public fun WorkspaceBuilder.task(
|
||||
*/
|
||||
public class SimpleWorkspaceBuilder(override val parentContext: Context) : WorkspaceBuilder {
|
||||
override var context: Context = parentContext
|
||||
override var data: MutableDataTree<Any> = MutableDataTree(Any::class,context)
|
||||
override var data: MutableDataTree<Any> = MutableDataTree(Any::class, context)
|
||||
override var tasks: MutableSet<Task<Any>> = HashSet()
|
||||
override var targets: MutableMap<String, Meta> = HashMap()
|
||||
|
||||
|
@ -1,9 +0,0 @@
|
||||
package hep.dataforge.workspace
|
||||
|
||||
import hep.dataforge.data.DataSet
|
||||
import hep.dataforge.meta.MetaBuilder
|
||||
import kotlinx.coroutines.runBlocking
|
||||
|
||||
public fun Workspace.runBlocking(task: String, block: MetaBuilder.() -> Unit = {}): DataSet<Any> = runBlocking{
|
||||
run(task, block)
|
||||
}
|
@ -13,7 +13,7 @@ import kotlin.reflect.KClass
|
||||
import kotlin.test.Test
|
||||
import kotlin.test.assertEquals
|
||||
|
||||
public fun <T : Any> DataSet<T>.first(): NamedData<T>? = runBlocking { flow().firstOrNull() }
|
||||
fun <T : Any> DataSet<T>.first(): NamedData<T>? = runBlocking { flow().firstOrNull() }
|
||||
|
||||
class DataPropagationTestPlugin : WorkspacePlugin() {
|
||||
override val tag: PluginTag = Companion.tag
|
||||
|
@ -67,8 +67,10 @@ class FileDataTest {
|
||||
}
|
||||
println(dir.toUri().toString())
|
||||
val reconstructed = readDataDirectory(dir,StringFormatResolver)
|
||||
assertEquals(dataNode["dir.a"].data?.meta, reconstructed["dir.a"].data?.meta)
|
||||
assertEquals(dataNode["b"]?.data?.value(), reconstructed["b"]?.data?.value())
|
||||
runBlocking {
|
||||
assertEquals(dataNode.getData("dir.a")?.meta, reconstructed.getData("dir.a")?.meta)
|
||||
assertEquals(dataNode.getData("b")?.value(), reconstructed.getData("b")?.value())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -83,8 +85,10 @@ class FileDataTest {
|
||||
}
|
||||
println(zip.toUri().toString())
|
||||
val reconstructed = readDataDirectory(zip, StringFormatResolver)
|
||||
assertEquals(dataNode["dir.a"].data?.meta, reconstructed["dir.a"].data?.meta)
|
||||
assertEquals(dataNode["b"]?.data?.value(), reconstructed["b"]?.data?.value())
|
||||
runBlocking {
|
||||
assertEquals(dataNode.getData("dir.a")?.meta, reconstructed.getData("dir.a")?.meta)
|
||||
assertEquals(dataNode.getData("b")?.value(), reconstructed.getData("b")?.value())
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
@ -6,6 +6,7 @@ import hep.dataforge.meta.*
|
||||
import hep.dataforge.names.plus
|
||||
import kotlinx.coroutines.flow.collect
|
||||
import kotlinx.coroutines.runBlocking
|
||||
import org.junit.jupiter.api.Timeout
|
||||
import kotlin.reflect.KClass
|
||||
import kotlin.test.Test
|
||||
import kotlin.test.assertEquals
|
||||
@ -21,15 +22,8 @@ public inline fun <reified P : Plugin> P.toFactory(): PluginFactory<P> = object
|
||||
override val type: KClass<out P> = P::class
|
||||
}
|
||||
|
||||
public fun DataTree<*>.toMeta(): Meta = Meta {
|
||||
"type" put (dataType.simpleName ?: "undefined")
|
||||
"items" put {
|
||||
runBlocking {
|
||||
flow().collect {
|
||||
it.name.toString() put it.data.meta
|
||||
}
|
||||
}
|
||||
}
|
||||
public fun Workspace.runBlocking(task: String, block: MetaBuilder.() -> Unit = {}): DataSet<Any> = runBlocking{
|
||||
run(task, block)
|
||||
}
|
||||
|
||||
|
||||
@ -89,8 +83,8 @@ class SimpleWorkspaceTest {
|
||||
val linearDep = dependsOn(linear, placement = DataPlacement.into("linear"))
|
||||
}
|
||||
transform<Int> { data ->
|
||||
val squareNode = data["square"].filterIsInstance<Int>() //squareDep()
|
||||
val linearNode = data["linear"].filterIsInstance<Int>() //linearDep()
|
||||
val squareNode = data.branch("square").filterIsInstance<Int>() //squareDep()
|
||||
val linearNode = data.branch("linear").filterIsInstance<Int>() //linearDep()
|
||||
DataTree.dynamic<Int>(context) {
|
||||
squareNode.flow().collect {
|
||||
val newData: Data<Int> = Data {
|
||||
@ -163,6 +157,7 @@ class SimpleWorkspaceTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
@Timeout(400)
|
||||
fun testMetaPropagation() {
|
||||
val node = workspace.runBlocking("sum") { "testFlag" put true }
|
||||
val res = node.first()?.value()
|
||||
|
Loading…
Reference in New Issue
Block a user