kscience/dataforge-core
7
0
Fork 0
Code Issues Pull Requests 1 Packages Projects Releases 12 Wiki Activity

WIP full data refactor

Browse Source
This commit is contained in:
Alexander Nozik 2021-01-10 18:18:34 +03:00
parent 23fae9794f
commit b916a038f7
11 changed files with 35 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
dataforge-data/src/commonMain/kotlin/hep/dataforge/data/Data.kt
View File

@ -44,7 +44,7 @@ public interface Data<out T : Any> : Goal<T>, MetaRepr {
/**
* An empty data containing only meta
*/
public fun empty(meta: Meta): Data<Nothing> = object : Data<Nothing> {
public fun <T> empty(meta: Meta): Data<Nothing> = object : Data<Nothing> {
override val type: KClass<out Nothing> = Nothing::class
override val meta: Meta = meta
override val dependencies: Collection<Goal<*>> = emptyList()

8
dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataTree.kt
View File

@ -67,6 +67,8 @@ public interface DataTree<out T : Any> : DataSet<T> {
}
}
public suspend fun <T: Any> DataSet<T>.getData(name: String): Data<T>? = getData(name.toName())
/**
* Get a [DataTreeItem] with given [name] or null if the item does not exist
*/
@ -97,10 +99,10 @@ public fun <T : Any> DataTree<T>.itemFlow(): Flow<Pair<Name, DataTreeItem<T>>> =
* Get a branch of this [DataTree] with a given [branchName].
* The difference from similar method for [DataSet] is that internal logic is more simple and the return value is a [DataTree]
*/
public operator fun <T : Any> DataTree<T>.get(branchName: Name): DataTree<T> = object : DataTree<T> {
override val dataType: KClass<out T> get() = this@get.dataType
public fun <T : Any> DataTree<T>.branch(branchName: Name): DataTree<T> = object : DataTree<T> {
override val dataType: KClass<out T> get() = this@branch.dataType
override val updates: Flow<Name> = this@get.updates.mapNotNull { it.removeHeadOrNull(branchName) }
override val updates: Flow<Name> = this@branch.updates.mapNotNull { it.removeHeadOrNull(branchName) }
override suspend fun items(): Map<NameToken, DataTreeItem<T>> = getItem(branchName).tree?.items() ?: emptyMap()
}

3
dataforge-data/src/commonMain/kotlin/hep/dataforge/data/MutableDataTree.kt
View File

@ -154,6 +154,3 @@ public suspend fun <T : Any> DataSet<T>.toMutableTree(
set(it, getData(it))
}.launchIn(scope)
}
public fun <T : Any> MutableDataTree<T>.get(branchName: Name): MutableDataTree<T> =
(this as DataTree<T>).get(branchName) as MutableDataTree<T>

12
dataforge-data/src/commonMain/kotlin/hep/dataforge/data/dataSetOperations.kt
View File

@ -51,22 +51,22 @@ else object : DataSet<T> {
/**
* Get a subset of data starting with a given [branchName]
*/
public operator fun <T : Any> DataSet<T>.get(branchName: Name): DataSet<T> = if (branchName.isEmpty()) this
public fun <T : Any> DataSet<T>.branch(branchName: Name): DataSet<T> = if (branchName.isEmpty()) this
else object : DataSet<T> {
override val dataType: KClass<out T> get() = this@get.dataType
override val dataType: KClass<out T> get() = this@branch.dataType
override fun flow(): Flow<NamedData<T>> = this@get.flow().mapNotNull {
override fun flow(): Flow<NamedData<T>> = this@branch.flow().mapNotNull {
it.name.removeHeadOrNull(branchName)?.let { name ->
it.data.named(name)
}
}
override suspend fun getData(name: Name): Data<T>? = this@get.getData(branchName + name)
override suspend fun getData(name: Name): Data<T>? = this@branch.getData(branchName + name)
override val updates: Flow<Name> get() = this@get.updates.mapNotNull { it.removeHeadOrNull(branchName) }
override val updates: Flow<Name> get() = this@branch.updates.mapNotNull { it.removeHeadOrNull(branchName) }
}
public operator fun <T : Any> DataSet<T>.get(branchName: String): DataSet<T> = this@get.get(branchName.toName())
public fun <T : Any> DataSet<T>.branch(branchName: String): DataSet<T> = this@branch.branch(branchName.toName())
@DFExperimental
public suspend fun <T : Any> DataSet<T>.rootData(): Data<T>? = getData(Name.EMPTY)

4
dataforge-data/src/jvmTest/kotlin/hep/dataforge/data/DataTreeBuilderTest.kt
View File

@ -27,8 +27,8 @@ internal class DataTreeBuilderTest{
}
assertTrue { node["update.a"] != null }
assertTrue { node["primary.a"] != null }
assertTrue { node.branch("update.a") != null }
assertTrue { node.branch("primary.a") != null }
}
}

2
dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/TaskBuilder.kt
View File

@ -57,7 +57,7 @@ public class TaskBuilder<R : Any>(private val name: Name, public val type: KClas
) {
dataTransforms += { context, model, data ->
val env = TaskEnv(Name.EMPTY, model.meta, context, data)
val startData = data.get(from)
val startData = data.branch(from)
env.block(startData).withNamePrefix(to)
}
}

20
dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/WorkspaceBuilder.kt
View File

@ -3,10 +3,8 @@ package hep.dataforge.workspace
import hep.dataforge.context.Context
import hep.dataforge.context.ContextBuilder
import hep.dataforge.context.Global
import hep.dataforge.data.DataTree
import hep.dataforge.data.MutableDataTree
import hep.dataforge.meta.*
import hep.dataforge.names.Name
import hep.dataforge.names.toName
import kotlin.reflect.KClass
@ -28,19 +26,11 @@ public fun WorkspaceBuilder.context(name: String = "WORKSPACE", block: ContextBu
context = ContextBuilder(parentContext, name).apply(block).build()
}
public inline fun <reified T : Any> WorkspaceBuilder.data(
name: Name = Name.EMPTY,
noinline block: MutableDataTree<T>.() -> Unit,
): DataTree<T> {
TODO()
//data.branch(name).apply(block)
}
@JvmName("rawData")
public fun WorkspaceBuilder.data(
name: Name = Name.EMPTY,
public inline fun WorkspaceBuilder.data(
block: MutableDataTree<Any>.() -> Unit,
): DataTree<Any> = data<Any>(name, block)
): Unit{
data.apply(block)
}
public fun WorkspaceBuilder.target(name: String, block: MetaBuilder.() -> Unit) {
@ -80,7 +70,7 @@ public fun WorkspaceBuilder.task(
*/
public class SimpleWorkspaceBuilder(override val parentContext: Context) : WorkspaceBuilder {
override var context: Context = parentContext
override var data: MutableDataTree<Any> = MutableDataTree(Any::class,context)
override var data: MutableDataTree<Any> = MutableDataTree(Any::class, context)
override var tasks: MutableSet<Task<Any>> = HashSet()
override var targets: MutableMap<String, Meta> = HashMap()

9
dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/tasksJVM.kt
View File

@ -1,9 +0,0 @@
package hep.dataforge.workspace
import hep.dataforge.data.DataSet
import hep.dataforge.meta.MetaBuilder
import kotlinx.coroutines.runBlocking
public fun Workspace.runBlocking(task: String, block: MetaBuilder.() -> Unit = {}): DataSet<Any> = runBlocking{
run(task, block)
}

2
dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/DataPropagationTest.kt
View File

@ -13,7 +13,7 @@ import kotlin.reflect.KClass
import kotlin.test.Test
import kotlin.test.assertEquals
public fun <T : Any> DataSet<T>.first(): NamedData<T>? = runBlocking { flow().firstOrNull() }
fun <T : Any> DataSet<T>.first(): NamedData<T>? = runBlocking { flow().firstOrNull() }
class DataPropagationTestPlugin : WorkspacePlugin() {
override val tag: PluginTag = Companion.tag

12
dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/FileDataTest.kt
View File

@ -67,8 +67,10 @@ class FileDataTest {
}
println(dir.toUri().toString())
val reconstructed = readDataDirectory(dir,StringFormatResolver)
assertEquals(dataNode["dir.a"].data?.meta, reconstructed["dir.a"].data?.meta)
assertEquals(dataNode["b"]?.data?.value(), reconstructed["b"]?.data?.value())
runBlocking {
assertEquals(dataNode.getData("dir.a")?.meta, reconstructed.getData("dir.a")?.meta)
assertEquals(dataNode.getData("b")?.value(), reconstructed.getData("b")?.value())
}
}
}
@ -83,8 +85,10 @@ class FileDataTest {
}
println(zip.toUri().toString())
val reconstructed = readDataDirectory(zip, StringFormatResolver)
assertEquals(dataNode["dir.a"].data?.meta, reconstructed["dir.a"].data?.meta)
assertEquals(dataNode["b"]?.data?.value(), reconstructed["b"]?.data?.value())
runBlocking {
assertEquals(dataNode.getData("dir.a")?.meta, reconstructed.getData("dir.a")?.meta)
assertEquals(dataNode.getData("b")?.value(), reconstructed.getData("b")?.value())
}
}
}
}

17
dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/SimpleWorkspaceTest.kt
View File

@ -6,6 +6,7 @@ import hep.dataforge.meta.*
import hep.dataforge.names.plus
import kotlinx.coroutines.flow.collect
import kotlinx.coroutines.runBlocking
import org.junit.jupiter.api.Timeout
import kotlin.reflect.KClass
import kotlin.test.Test
import kotlin.test.assertEquals
@ -21,15 +22,8 @@ public inline fun <reified P : Plugin> P.toFactory(): PluginFactory<P> = object
override val type: KClass<out P> = P::class
}
public fun DataTree<*>.toMeta(): Meta = Meta {
"type" put (dataType.simpleName ?: "undefined")
"items" put {
runBlocking {
flow().collect {
it.name.toString() put it.data.meta
}
}
}
public fun Workspace.runBlocking(task: String, block: MetaBuilder.() -> Unit = {}): DataSet<Any> = runBlocking{
run(task, block)
}
@ -89,8 +83,8 @@ class SimpleWorkspaceTest {
val linearDep = dependsOn(linear, placement = DataPlacement.into("linear"))
}
transform<Int> { data ->
val squareNode = data["square"].filterIsInstance<Int>() //squareDep()
val linearNode = data["linear"].filterIsInstance<Int>() //linearDep()
val squareNode = data.branch("square").filterIsInstance<Int>() //squareDep()
val linearNode = data.branch("linear").filterIsInstance<Int>() //linearDep()
DataTree.dynamic<Int>(context) {
squareNode.flow().collect {
val newData: Data<Int> = Data {
@ -163,6 +157,7 @@ class SimpleWorkspaceTest {
}
@Test
@Timeout(400)
fun testMetaPropagation() {
val node = workspace.runBlocking("sum") { "testFlag" put true }
val res = node.first()?.value()