From 6a0bfae931e78f636413cbcfe9ecab60f4463e2c Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Tue, 2 Feb 2021 12:09:44 +0300 Subject: [PATCH 1/7] minor data refactor --- .github/workflows/build.yml | 20 +++--- .github/workflows/publish.yml | 70 +++++++++---------- .../hep/dataforge/actions/ReduceAction.kt | 2 +- .../hep/dataforge/data/DataSetBuilder.kt | 10 +-- .../kotlin/hep/dataforge/data/GroupRule.kt | 34 +++------ .../hep/dataforge/data/StaticDataTree.kt | 4 +- .../kotlin/hep/dataforge/data/dataFilter.kt | 2 - .../kotlin/hep/dataforge/data/ActionsTest.kt | 2 +- .../hep/dataforge/data/DataTreeBuilderTest.kt | 26 +++++-- .../workspace/DataPropagationTest.kt | 2 +- .../hep/dataforge/workspace/FileDataTest.kt | 4 +- .../workspace/SimpleWorkspaceTest.kt | 2 +- 12 files changed, 89 insertions(+), 89 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e780ac7e..6a362884 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,6 +1,6 @@ name: Gradle build -on: [push] +on: [ push ] jobs: build: @@ -8,12 +8,12 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - name: Set up JDK 11 - uses: actions/setup-java@v1 - with: - java-version: 11 - - name: Grant execute permission for gradlew - run: chmod +x gradlew - - name: Build with Gradle - run: ./gradlew build + - uses: actions/checkout@v2 + - name: Set up JDK 11 + uses: actions/setup-java@v1 + with: + java-version: 11 + - name: Grant execute permission for gradlew + run: chmod +x gradlew + - name: Build with Gradle + run: ./gradlew build diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 0d472b1b..0fbf9c1e 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -1,40 +1,40 @@ name: Bintray Publish on: - release: - types: - - created + release: + types: + - created jobs: - build-on-windows: - runs-on: windows-latest - steps: - - uses: actions/checkout@v2 - - name: Set up JDK 11 - uses: actions/setup-java@v1 - with: - java-version: 11 - - name: Grant execute permission for gradlew - run: chmod +x gradlew - - name: Gradle clean - run: ./gradlew clean - - name: Gradle build - run: ./gradlew build - - name: Run release task - run: ./gradlew release -PbintrayUser=${{ secrets.BINTRAY_USER }} -PbintrayApiKey=${{ secrets.BINTRAY_KEY }} - build-on-macos: - runs-on: macos-latest - steps: - - uses: actions/checkout@v2 - - name: Set up JDK 11 - uses: actions/setup-java@v1 - with: - java-version: 11 - - name: Grant execute permission for gradlew - run: chmod +x gradlew - - name: Gradle clean - run: ./gradlew clean - - name: Gradle build - run: ./gradlew build - - name: Run release task - run: ./gradlew release -PbintrayUser=${{ secrets.BINTRAY_USER }} -PbintrayApiKey=${{ secrets.BINTRAY_KEY }} + build-on-windows: + runs-on: windows-latest + steps: + - uses: actions/checkout@v2 + - name: Set up JDK 11 + uses: actions/setup-java@v1 + with: + java-version: 11 + - name: Grant execute permission for gradlew + run: chmod +x gradlew + - name: Gradle clean + run: ./gradlew clean + - name: Gradle build + run: ./gradlew build + - name: Run release task + run: ./gradlew release -PbintrayUser=${{ secrets.BINTRAY_USER }} -PbintrayApiKey=${{ secrets.BINTRAY_KEY }} + build-on-macos: + runs-on: macos-latest + steps: + - uses: actions/checkout@v2 + - name: Set up JDK 11 + uses: actions/setup-java@v1 + with: + java-version: 11 + - name: Grant execute permission for gradlew + run: chmod +x gradlew + - name: Gradle clean + run: ./gradlew clean + - name: Gradle build + run: ./gradlew build + - name: Run release task + run: ./gradlew release -PbintrayUser=${{ secrets.BINTRAY_USER }} -PbintrayApiKey=${{ secrets.BINTRAY_KEY }} diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/actions/ReduceAction.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/actions/ReduceAction.kt index 7bc4ab07..505b8040 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/actions/ReduceAction.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/actions/ReduceAction.kt @@ -39,7 +39,7 @@ public class ReduceGroupBuilder( */ public fun byValue(tag: String, defaultTag: String = "@default", action: JoinGroup.() -> Unit) { groupRules += { node -> - GroupRule.byValue(scope, tag, defaultTag).gather(inputType, node).map { + GroupRule.byMetaValue(scope, tag, defaultTag).gather(inputType, node).map { JoinGroup(it.key, it.value).apply(action) } } diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataSetBuilder.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataSetBuilder.kt index 2451dc33..f001c4ad 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataSetBuilder.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataSetBuilder.kt @@ -88,7 +88,7 @@ public suspend fun DataSetBuilder.emit(data: NamedData) { /** * Produce lazy [Data] and emit it into the [DataSetBuilder] */ -public suspend inline fun DataSetBuilder.emitLazy( +public suspend inline fun DataSetBuilder.produce( name: String, meta: Meta = Meta.EMPTY, noinline producer: suspend () -> T, @@ -97,7 +97,7 @@ public suspend inline fun DataSetBuilder.emitLazy( emit(name, data) } -public suspend inline fun DataSetBuilder.emitLazy( +public suspend inline fun DataSetBuilder.produce( name: Name, meta: Meta = Meta.EMPTY, noinline producer: suspend () -> T, @@ -109,13 +109,13 @@ public suspend inline fun DataSetBuilder.emitLazy( /** * Emit a static data with the fixed value */ -public suspend fun DataSetBuilder.emitStatic(name: String, data: T, meta: Meta = Meta.EMPTY): Unit = +public suspend fun DataSetBuilder.static(name: String, data: T, meta: Meta = Meta.EMPTY): Unit = emit(name, Data.static(data, meta)) -public suspend fun DataSetBuilder.emitStatic(name: Name, data: T, meta: Meta = Meta.EMPTY): Unit = +public suspend fun DataSetBuilder.static(name: Name, data: T, meta: Meta = Meta.EMPTY): Unit = emit(name, Data.static(data, meta)) -public suspend fun DataSetBuilder.emitStatic( +public suspend fun DataSetBuilder.static( name: String, data: T, metaBuilder: MetaBuilder.() -> Unit, diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/GroupRule.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/GroupRule.kt index 41e1de53..414fa17e 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/GroupRule.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/GroupRule.kt @@ -19,6 +19,7 @@ import hep.dataforge.meta.get import hep.dataforge.meta.string import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.flow.collect +import kotlinx.coroutines.launch import kotlin.reflect.KClass public interface GroupRule { @@ -33,7 +34,7 @@ public interface GroupRule { * @param defaultTagValue * @return */ - public fun byValue( + public fun byMetaValue( scope: CoroutineScope, key: String, defaultTagValue: String, @@ -50,31 +51,16 @@ public interface GroupRule { map.getOrPut(tagValue) { ActiveDataTree(dataType) }.emit(data.name, data.data) } + scope.launch { + set.updates.collect { name -> + val data = set.getData(name) + val tagValue = data?.meta[key]?.string ?: defaultTagValue + map.getOrPut(tagValue) { ActiveDataTree(dataType) }.emit(name, data) + } + } + return map } } - - - // @ValueDef(key = "byValue", required = true, info = "The name of annotation value by which grouping should be made") -// @ValueDef( -// key = "defaultValue", -// def = "default", -// info = "Default value which should be used for content in which the grouping value is not presented" -// ) -// public fun byMeta(scope: CoroutineScope, config: Meta): GroupRule { -// //TODO expand grouping options -// return config["byValue"]?.string?.let { -// byValue( -// scope, -// it, -// config["defaultValue"]?.string ?: "default" -// ) -// } ?: object : GroupRule { -// override suspend fun gather( -// dataType: KClass, -// source: DataSource, -// ): Map> = mapOf("" to source) -// } -// } } } diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/StaticDataTree.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/StaticDataTree.kt index 05cd85de..4741a464 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/StaticDataTree.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/StaticDataTree.kt @@ -2,7 +2,7 @@ package hep.dataforge.data import hep.dataforge.names.* import kotlinx.coroutines.coroutineScope -import kotlinx.coroutines.flow.* +import kotlinx.coroutines.flow.collect import kotlin.reflect.KClass @PublishedApi @@ -22,7 +22,7 @@ internal class StaticDataTree( } } - fun getOrCreateNode(name: Name): StaticDataTree = when (name.length) { + private fun getOrCreateNode(name: Name): StaticDataTree = when (name.length) { 0 -> this 1 -> { val itemName = name.firstOrNull()!! diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/dataFilter.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/dataFilter.kt index 4d500587..aff59e61 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/dataFilter.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/dataFilter.kt @@ -32,7 +32,6 @@ public fun DataSet.filter( } } - /** * Generate a wrapper data set with a given name prefix appended to all names */ @@ -48,7 +47,6 @@ else object : ActiveDataSet { override val updates: Flow get() = this@withNamePrefix.updates.map { prefix + it } } - /** * Get a subset of data starting with a given [branchName] */ diff --git a/dataforge-data/src/jvmTest/kotlin/hep/dataforge/data/ActionsTest.kt b/dataforge-data/src/jvmTest/kotlin/hep/dataforge/data/ActionsTest.kt index 453dd2f1..81049612 100644 --- a/dataforge-data/src/jvmTest/kotlin/hep/dataforge/data/ActionsTest.kt +++ b/dataforge-data/src/jvmTest/kotlin/hep/dataforge/data/ActionsTest.kt @@ -14,7 +14,7 @@ class ActionsTest { val data: DataTree = runBlocking { DataTree { repeat(10) { - emitStatic(it.toString(), it) + static(it.toString(), it) } } } diff --git a/dataforge-data/src/jvmTest/kotlin/hep/dataforge/data/DataTreeBuilderTest.kt b/dataforge-data/src/jvmTest/kotlin/hep/dataforge/data/DataTreeBuilderTest.kt index 55ed1523..cf34cbb8 100644 --- a/dataforge-data/src/jvmTest/kotlin/hep/dataforge/data/DataTreeBuilderTest.kt +++ b/dataforge-data/src/jvmTest/kotlin/hep/dataforge/data/DataTreeBuilderTest.kt @@ -8,6 +8,22 @@ import kotlin.test.assertEquals internal class DataTreeBuilderTest { + @Test + fun testTreeBuild() = runBlocking { + val node = DataTree { + "primary" put { + static("a", "a") + static("b", "b") + } + static("c.d", "c.d") + static("c.f", "c.f") + } + assertEquals("a", node.getData("primary.a")?.value()) + assertEquals("b", node.getData("primary.b")?.value()) + assertEquals("c.d", node.getData("c.d")?.value()) + assertEquals("c.f", node.getData("c.f")?.value()) + } + @Test fun testDataUpdate() = runBlocking { val updateData: DataTree = DataTree { @@ -18,11 +34,11 @@ internal class DataTreeBuilderTest { } val node = DataTree { - emit("primary") { - emitStatic("a", "a") - emitStatic("b", "b") + "primary" put { + static("a", "a") + static("b", "b") } - emitStatic("root", "root") + static("root", "root") populate(updateData) } @@ -40,7 +56,7 @@ internal class DataTreeBuilderTest { updateJob = launch { repeat(10) { delay(10) - emitStatic("value", it) + static("value", it) } delay(10) } diff --git a/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/DataPropagationTest.kt b/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/DataPropagationTest.kt index 16c1460b..3b70ed5e 100644 --- a/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/DataPropagationTest.kt +++ b/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/DataPropagationTest.kt @@ -49,7 +49,7 @@ class DataPropagationTest { runBlocking { data { repeat(100) { - emitStatic("myData[$it]", it) + static("myData[$it]", it) } } } diff --git a/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/FileDataTest.kt b/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/FileDataTest.kt index 50a4bd0d..e089e448 100644 --- a/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/FileDataTest.kt +++ b/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/FileDataTest.kt @@ -23,11 +23,11 @@ class FileDataTest { val dataNode = runBlocking { DataTree { emit("dir") { - emitStatic("a", "Some string") { + static("a", "Some string") { "content" put "Some string" } } - emitStatic("b", "root data") + static("b", "root data") meta { "content" put "This is root meta node" } diff --git a/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/SimpleWorkspaceTest.kt b/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/SimpleWorkspaceTest.kt index c81621b9..b6bc9a1c 100644 --- a/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/SimpleWorkspaceTest.kt +++ b/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/SimpleWorkspaceTest.kt @@ -56,7 +56,7 @@ class SimpleWorkspaceTest { data { repeat(100) { - emitStatic("myData[$it]", it) + static("myData[$it]", it) } } From fcd99b1ca895fd1dab650fb8104ad16b9d9c8ef3 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Tue, 2 Feb 2021 17:48:25 +0300 Subject: [PATCH 2/7] [WIP] change KClass to KType --- dataforge-data/build.gradle.kts | 4 -- .../kotlin/hep/dataforge/actions/MapAction.kt | 12 ++-- .../hep/dataforge/actions/ReduceAction.kt | 3 +- .../hep/dataforge/actions/SplitAction.kt | 10 +-- .../hep/dataforge/data/ActiveDataTree.kt | 17 ++--- .../hep/dataforge/data/CachingAction.kt | 6 +- .../kotlin/hep/dataforge/data/Data.kt | 25 +++---- .../kotlin/hep/dataforge/data/DataSet.kt | 10 +-- .../hep/dataforge/data/StaticDataTree.kt | 13 ++-- .../hep/dataforge/data/dataTransform.kt | 27 ++++---- .../kotlin/hep/dataforge/data/dataJVM.kt | 42 ------------ .../kotlin/hep/dataforge/data/select.kt | 67 +++++++++++++++---- .../kotlin/hep/dataforge/tables/CastColumn.kt | 4 +- 13 files changed, 123 insertions(+), 117 deletions(-) delete mode 100644 dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/dataJVM.kt diff --git a/dataforge-data/build.gradle.kts b/dataforge-data/build.gradle.kts index 7ae0c747..16d98ced 100644 --- a/dataforge-data/build.gradle.kts +++ b/dataforge-data/build.gradle.kts @@ -12,10 +12,6 @@ kotlin { commonMain{ dependencies { api(project(":dataforge-meta")) - } - } - jvmMain{ - dependencies{ api(kotlin("reflect")) } } diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/actions/MapAction.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/actions/MapAction.kt index 45632b43..77acc6a4 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/actions/MapAction.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/actions/MapAction.kt @@ -7,7 +7,8 @@ import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.flow.collect import kotlinx.coroutines.flow.map import kotlinx.coroutines.launch -import kotlin.reflect.KClass +import kotlin.reflect.KType +import kotlin.reflect.typeOf /** * Action environment includes data name, data meta and action configuration meta @@ -34,8 +35,9 @@ public class MapActionBuilder(public var name: Name, public var meta: Meta } -public class MapAction( - public val outputType: KClass, +@PublishedApi +internal class MapAction( + private val outputType: KType, private val block: MapActionBuilder.() -> Unit, ) : Action { @@ -84,8 +86,8 @@ public class MapAction( @Suppress("FunctionName") -public inline fun MapAction( +public inline fun Action.Companion.map( noinline builder: MapActionBuilder.() -> Unit, -): MapAction = MapAction(R::class, builder) +): Action = MapAction(typeOf(), builder) diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/actions/ReduceAction.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/actions/ReduceAction.kt index 505b8040..f3cf9fdc 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/actions/ReduceAction.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/actions/ReduceAction.kt @@ -11,6 +11,7 @@ import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.flow import kotlinx.coroutines.flow.fold import kotlin.reflect.KClass +import kotlin.reflect.KType @DFExperimental @@ -75,7 +76,7 @@ public class ReduceGroupBuilder( @DFExperimental public class ReduceAction( private val inputType: KClass, - outputType: KClass, + outputType: KType, private val action: ReduceGroupBuilder.() -> Unit, ) : CachingAction(outputType) { //TODO optimize reduction. Currently the whole action recalculates on push diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/actions/SplitAction.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/actions/SplitAction.kt index aaec3836..c659d5a1 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/actions/SplitAction.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/actions/SplitAction.kt @@ -11,7 +11,7 @@ import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.flow.* import kotlinx.coroutines.launch import kotlin.collections.set -import kotlin.reflect.KClass +import kotlin.reflect.KType public class SplitBuilder(public val name: Name, public val meta: Meta) { @@ -39,8 +39,8 @@ public class SplitBuilder(public val name: Name, public val me /** * Action that splits each incoming element into a number of fragments defined in builder */ -public class SplitAction( - private val outputType: KClass, +internal class SplitAction( + private val outputType: KType, private val action: SplitBuilder.() -> Unit, ) : Action { @@ -59,11 +59,11 @@ public class SplitAction( // apply individual fragment rules to result return split.fragments.entries.asFlow().map { (fragmentName, rule) -> val env = SplitBuilder.FragmentRule(fragmentName, laminate.toMutableMeta()).apply(rule) - data.map(outputType, meta = env.meta) { env.result(it) }.named(fragmentName) + data.map(outputType, meta = env.meta) { env.result(it) }.named(fragmentName) } } - return ActiveDataTree(outputType) { + return ActiveDataTree(outputType) { populate(dataSet.flow().flatMapConcat(transform = ::splitOne)) scope?.launch { dataSet.updates.collect { name -> diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/ActiveDataTree.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/ActiveDataTree.kt index eb6e5582..5d197982 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/ActiveDataTree.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/ActiveDataTree.kt @@ -8,13 +8,14 @@ import kotlinx.coroutines.flow.* import kotlinx.coroutines.launch import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock -import kotlin.reflect.KClass +import kotlin.reflect.KType +import kotlin.reflect.typeOf /** * A mutable [DataTree.Companion.active]. It */ public class ActiveDataTree( - override val dataType: KClass, + override val dataType: KType, ) : DataTree, DataSetBuilder, ActiveDataSet { private val mutex = Mutex() private val treeItems = HashMap>() @@ -49,7 +50,7 @@ public class ActiveDataTree( private suspend fun getOrCreateNode(token: NameToken): ActiveDataTree = (treeItems[token] as? DataTreeItem.Node)?.tree as? ActiveDataTree - ?: ActiveDataTree(dataType).also { + ?: ActiveDataTree(dataType).also { mutex.withLock { treeItems[token] = DataTreeItem.Node(it) } @@ -92,10 +93,10 @@ public class ActiveDataTree( */ @Suppress("FunctionName") public suspend fun ActiveDataTree( - type: KClass, + type: KType, block: suspend ActiveDataTree.() -> Unit, ): ActiveDataTree { - val tree = ActiveDataTree(type) + val tree = ActiveDataTree(type) tree.block() return tree } @@ -103,15 +104,15 @@ public suspend fun ActiveDataTree( @Suppress("FunctionName") public suspend inline fun ActiveDataTree( crossinline block: suspend ActiveDataTree.() -> Unit, -): ActiveDataTree = ActiveDataTree(T::class).apply { block() } +): ActiveDataTree = ActiveDataTree(typeOf()).apply { block() } public suspend inline fun ActiveDataTree.emit( name: Name, noinline block: suspend ActiveDataTree.() -> Unit, -): Unit = emit(name, ActiveDataTree(T::class, block)) +): Unit = emit(name, ActiveDataTree(typeOf(), block)) public suspend inline fun ActiveDataTree.emit( name: String, noinline block: suspend ActiveDataTree.() -> Unit, -): Unit = emit(name.toName(), ActiveDataTree(T::class, block)) +): Unit = emit(name.toName(), ActiveDataTree(typeOf(), block)) diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/CachingAction.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/CachingAction.kt index 0e4ddd97..c4fb4f90 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/CachingAction.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/CachingAction.kt @@ -9,7 +9,7 @@ import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.coroutineScope import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.collect -import kotlin.reflect.KClass +import kotlin.reflect.KType /** * Remove all values with keys starting with [name] @@ -23,7 +23,7 @@ internal fun MutableMap.removeWhatStartsWith(name: Name) { * An action that caches results on-demand and recalculates them on source push */ public abstract class CachingAction( - public val outputType: KClass, + public val outputType: KType, ) : Action { protected abstract fun CoroutineScope.transform( @@ -36,7 +36,7 @@ public abstract class CachingAction( dataSet: DataSet, meta: Meta, scope: CoroutineScope?, - ): DataSet = ActiveDataTree(outputType) { + ): DataSet = ActiveDataTree(outputType) { coroutineScope { populate(transform(dataSet, meta)) } diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/Data.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/Data.kt index 6d0c4c38..0a768ee4 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/Data.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/Data.kt @@ -7,7 +7,8 @@ import hep.dataforge.misc.Type import kotlinx.coroutines.* import kotlin.coroutines.CoroutineContext import kotlin.coroutines.EmptyCoroutineContext -import kotlin.reflect.KClass +import kotlin.reflect.KType +import kotlin.reflect.typeOf /** * A data element characterized by its meta @@ -17,7 +18,7 @@ public interface Data : Goal, MetaRepr { /** * Type marker for the data. The type is known before the calculation takes place so it could be checked. */ - public val type: KClass + public val type: KType /** * Meta for the data @@ -25,7 +26,7 @@ public interface Data : Goal, MetaRepr { public val meta: Meta override fun toMeta(): Meta = Meta { - "type" put (type.simpleName ?: "undefined") + "type" put (type.toString()) if (!meta.isEmpty()) { "meta" put meta } @@ -34,16 +35,17 @@ public interface Data : Goal, MetaRepr { public companion object { public const val TYPE: String = "data" - public fun static( + public inline fun static( value: T, meta: Meta = Meta.EMPTY, - ): Data = StaticData(value, meta) + ): Data = StaticData(typeOf(),value, meta) /** * An empty data containing only meta */ public fun empty(meta: Meta): Data = object : Data { - override val type: KClass = Nothing::class + private val nothing: Nothing get() = error("this is nothing") + override val type: KType = this::nothing.returnType override val meta: Meta = meta override val dependencies: Collection> = emptyList() override val deferred: Deferred @@ -58,7 +60,7 @@ public interface Data : Goal, MetaRepr { } public class LazyData( - override val type: KClass, + override val type: KType, override val meta: Meta = Meta.EMPTY, context: CoroutineContext = EmptyCoroutineContext, dependencies: Collection> = emptyList(), @@ -66,15 +68,14 @@ public class LazyData( ) : Data, LazyGoal(context, dependencies, block) public class StaticData( + override val type: KType, value: T, override val meta: Meta = Meta.EMPTY, -) : Data, StaticGoal(value) { - override val type: KClass get() = value::class -} +) : Data, StaticGoal(value) @Suppress("FunctionName") public fun Data( - type: KClass, + type: KType, meta: Meta = Meta.EMPTY, context: CoroutineContext = EmptyCoroutineContext, dependencies: Collection> = emptyList(), @@ -87,4 +88,4 @@ public inline fun Data( context: CoroutineContext = EmptyCoroutineContext, dependencies: Collection> = emptyList(), noinline block: suspend () -> T, -): Data = Data(T::class, meta, context, dependencies, block) +): Data = Data(typeOf(), meta, context, dependencies, block) diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataSet.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataSet.kt index c401b275..129f657f 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataSet.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataSet.kt @@ -7,14 +7,14 @@ import hep.dataforge.meta.set import hep.dataforge.names.* import kotlinx.coroutines.* import kotlinx.coroutines.flow.* -import kotlin.reflect.KClass +import kotlin.reflect.KType public interface DataSet { /** * The minimal common ancestor to all data in the node */ - public val dataType: KClass + public val dataType: KType /** * Traverse this provider or its child. The order is not guaranteed. @@ -43,7 +43,9 @@ public interface DataSet { * An empty [DataSet] that suits all types */ public val EMPTY: DataSet = object : DataSet { - override val dataType: KClass = Nothing::class + override val dataType: KType = this::nothing.returnType + + private val nothing: Nothing get() = error("this is nothing") override fun flow(): Flow> = emptyFlow() @@ -88,7 +90,7 @@ public suspend fun DataSet<*>.toMeta(): Meta = Meta { set(it.name, it.meta) } else { it.name put { - "type" put it.type.simpleName + "type" put it.type.toString() "meta" put it.meta } } diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/StaticDataTree.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/StaticDataTree.kt index 4741a464..950cb1f1 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/StaticDataTree.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/StaticDataTree.kt @@ -3,11 +3,12 @@ package hep.dataforge.data import hep.dataforge.names.* import kotlinx.coroutines.coroutineScope import kotlinx.coroutines.flow.collect -import kotlin.reflect.KClass +import kotlin.reflect.KType +import kotlin.reflect.typeOf @PublishedApi internal class StaticDataTree( - override val dataType: KClass, + override val dataType: KType, ) : DataSetBuilder, DataTree { private val items: MutableMap> = HashMap() @@ -26,7 +27,7 @@ internal class StaticDataTree( 0 -> this 1 -> { val itemName = name.firstOrNull()!! - (items[itemName].tree as? StaticDataTree) ?: StaticDataTree(dataType).also { + (items[itemName].tree as? StaticDataTree) ?: StaticDataTree(dataType).also { items[itemName] = DataTreeItem.Node(it) } } @@ -61,14 +62,14 @@ internal class StaticDataTree( @Suppress("FunctionName") public suspend fun DataTree( - dataType: KClass, + dataType: KType, block: suspend DataSetBuilder.() -> Unit, -): DataTree = StaticDataTree(dataType).apply { block() } +): DataTree = StaticDataTree(dataType).apply { block() } @Suppress("FunctionName") public suspend inline fun DataTree( noinline block: suspend DataSetBuilder.() -> Unit, -): DataTree = DataTree(T::class, block) +): DataTree = DataTree(typeOf(), block) public suspend fun DataSet.seal(): DataTree = DataTree(dataType){ populate(this@seal) diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/dataTransform.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/dataTransform.kt index 31e30fb1..fdc24c1c 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/dataTransform.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/dataTransform.kt @@ -11,7 +11,8 @@ import kotlin.contracts.InvocationKind import kotlin.contracts.contract import kotlin.coroutines.CoroutineContext import kotlin.coroutines.EmptyCoroutineContext -import kotlin.reflect.KClass +import kotlin.reflect.KType +import kotlin.reflect.typeOf /** * Lazily transform this data to another data. By convention [block] should not use external data (be pure). @@ -19,8 +20,8 @@ import kotlin.reflect.KClass * @param meta for the resulting data. By default equals input data. * @param block the transformation itself */ -public fun Data.map( - outputType: KClass, +private fun Data.map( + outputType: KType, coroutineContext: CoroutineContext = EmptyCoroutineContext, meta: Meta = this.meta, block: suspend (T) -> R, @@ -35,7 +36,7 @@ public inline fun Data.map( coroutineContext: CoroutineContext = EmptyCoroutineContext, meta: Meta = this.meta, crossinline block: suspend (T) -> R, -): LazyData = LazyData(R::class, meta, coroutineContext, listOf(this)) { +): LazyData = LazyData(typeOf(), meta, coroutineContext, listOf(this)) { block(await()) } @@ -47,7 +48,7 @@ public inline fun Data.combine( coroutineContext: CoroutineContext = EmptyCoroutineContext, meta: Meta = this.meta, crossinline block: suspend (left: T1, right: T2) -> R, -): LazyData = LazyData(R::class, meta, coroutineContext, listOf(this, other)) { +): LazyData = LazyData(typeOf(), meta, coroutineContext, listOf(this, other)) { block(await(), other.await()) } @@ -62,7 +63,7 @@ public inline fun Collection>.reduceToData( meta: Meta = Meta.EMPTY, crossinline block: suspend (Collection) -> R, ): LazyData = LazyData( - R::class, + typeOf(), meta, coroutineContext, this @@ -71,7 +72,7 @@ public inline fun Collection>.reduceToData( } public fun Map>.reduceToData( - outputType: KClass, + outputType: KType, coroutineContext: CoroutineContext = EmptyCoroutineContext, meta: Meta = Meta.EMPTY, block: suspend (Map) -> R, @@ -96,7 +97,7 @@ public inline fun Map>.reduceToData( meta: Meta = Meta.EMPTY, noinline block: suspend (Map) -> R, ): LazyData = LazyData( - R::class, + typeOf(), meta, coroutineContext, this.values @@ -110,7 +111,7 @@ public inline fun Map>.reduceToData( * Transform a [Flow] of [NamedData] to a single [Data]. */ public suspend fun Flow>.reduceToData( - outputType: KClass, + outputType: KType, coroutineContext: CoroutineContext = EmptyCoroutineContext, meta: Meta = Meta.EMPTY, transformation: suspend (Flow>) -> R, @@ -127,7 +128,7 @@ public suspend inline fun Flow>.reduceTo coroutineContext: CoroutineContext = EmptyCoroutineContext, meta: Meta = Meta.EMPTY, noinline transformation: suspend (Flow>) -> R, -): LazyData = reduceToData(R::class, coroutineContext, meta) { +): LazyData = reduceToData(typeOf(), coroutineContext, meta) { transformation(it) } @@ -148,11 +149,11 @@ public suspend inline fun Flow>.foldToDa //DataSet operations public suspend fun DataSet.map( - outputType: KClass, + outputType: KType, coroutineContext: CoroutineContext = EmptyCoroutineContext, metaTransform: MetaBuilder.() -> Unit = {}, block: suspend (T) -> R, -): DataTree = DataTree(outputType) { +): DataTree = DataTree(outputType) { populate( flow().map { val newMeta = it.meta.toMutableMeta().apply(metaTransform).seal() @@ -165,7 +166,7 @@ public suspend inline fun DataSet.map( coroutineContext: CoroutineContext = EmptyCoroutineContext, noinline metaTransform: MetaBuilder.() -> Unit = {}, noinline block: suspend (T) -> R, -): DataTree = map(R::class, coroutineContext, metaTransform, block) +): DataTree = map(typeOf(), coroutineContext, metaTransform, block) public suspend fun DataSet.forEach(block: suspend (NamedData) -> Unit) { contract { callsInPlace(block, InvocationKind.EXACTLY_ONCE) } diff --git a/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/dataJVM.kt b/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/dataJVM.kt deleted file mode 100644 index 4178766a..00000000 --- a/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/dataJVM.kt +++ /dev/null @@ -1,42 +0,0 @@ -package hep.dataforge.data - -import kotlin.reflect.KClass -import kotlin.reflect.full.isSubclassOf - -/** - * Check if data could be safely cast to given class - */ -internal fun Data<*>.canCast(type: KClass): Boolean = this.type.isSubclassOf(type) - -/** - * Cast the node to given type if the cast is possible or return null - */ -@Suppress("UNCHECKED_CAST") -public fun Data<*>.castOrNull(type: KClass): Data? = - if (!canCast(type)) null else object : Data by (this as Data) { - override val type: KClass = type - } - -/** - * Unsafe cast of data node - */ -public fun Data<*>.cast(type: KClass): Data = - castOrNull(type) ?: error("Can't cast ${this.type} to $type") - -public inline fun Data<*>.cast(): Data = cast(R::class) - -@Suppress("UNCHECKED_CAST") -public fun DataSet<*>.castOrNull(type: KClass): DataSet? = - if (!canCast(type)) null else object : DataSet by (this as DataSet) { - override val dataType: KClass = type - } - - -public fun DataSet<*>.cast(type: KClass): DataSet = - castOrNull(type) ?: error("Can't cast ${this.dataType} to $type") - -/** - * Check that node is compatible with given type meaning that each element could be cast to the type - */ -internal fun DataSet<*>.canCast(type: KClass): Boolean = - type.isSubclassOf(this.dataType) \ No newline at end of file diff --git a/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/select.kt b/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/select.kt index 5217d1f0..def3133a 100644 --- a/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/select.kt +++ b/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/select.kt @@ -3,26 +3,69 @@ package hep.dataforge.data import hep.dataforge.actions.NamedData import hep.dataforge.actions.named import hep.dataforge.meta.DFExperimental -import hep.dataforge.names.* +import hep.dataforge.names.Name +import hep.dataforge.names.matches +import hep.dataforge.names.toName import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.filter import kotlinx.coroutines.flow.map -import kotlin.reflect.KClass +import kotlin.reflect.KType +import kotlin.reflect.full.isSubtypeOf +import kotlin.reflect.typeOf +/** + * Check if data could be safely cast to given class + */ +private fun Data<*>.canCast(type: KType): Boolean = this.type.isSubtypeOf(type) + +/** + * Cast the node to given type if the cast is possible or return null + */ +@Suppress("UNCHECKED_CAST") +private fun Data<*>.castOrNull(type: KType): Data? = + if (!canCast(type)) null else object : Data by (this as Data) { + override val type: KType = type + } + +/** + * Unsafe cast of data node + */ +private fun Data<*>.cast(type: KType): Data = + castOrNull(type) ?: error("Can't cast ${this.type} to $type") + +private inline fun Data<*>.cast(): Data = cast(typeOf()) + +@Suppress("UNCHECKED_CAST") +private fun DataSet<*>.castOrNull(type: KType): DataSet? = + if (!canCast(type)) null else object : DataSet by (this as DataSet) { + override val dataType: KType = type + } + + +private fun DataSet<*>.cast(type: KType): DataSet = + castOrNull(type) ?: error("Can't cast ${this.dataType} to $type") + +/** + * Check that node is compatible with given type meaning that each element could be cast to the type + */ +private fun DataSet<*>.canCast(type: KType): Boolean = + type.isSubtypeOf(this.dataType) + /** * Select all data matching given type and filters. Does not modify paths */ @OptIn(DFExperimental::class) -public fun DataSet<*>.select( - type: KClass, +@PublishedApi +internal fun DataSet<*>.select( + type: KType, namePattern: Name? = null, ): ActiveDataSet = object : ActiveDataSet { - override val dataType: KClass = type + override val dataType = type @Suppress("UNCHECKED_CAST") override fun flow(): Flow> = this@select.flow().filter { - it.canCast(type) && (namePattern == null || it.name.matches(namePattern)) + it.type.isSubtypeOf(type) && (namePattern == null || it.name.matches(namePattern)) }.map { it as NamedData } @@ -31,7 +74,7 @@ public fun DataSet<*>.select( override val updates: Flow = this@select.updates.filter { val datum = this@select.getData(it) - datum?.canCast(type) ?: false + datum?.canCast(type) ?: false } } @@ -40,12 +83,12 @@ public fun DataSet<*>.select( * Select a single datum of the appropriate type */ public inline fun DataSet<*>.select(namePattern: Name? = null): DataSet = - select(R::class, namePattern) + select(typeOf(), namePattern) -public suspend fun DataSet<*>.selectOne(type: KClass, name: Name): NamedData? = - getData(name)?.castOrNull(type)?.named(name) +public suspend fun DataSet<*>.selectOne(type: KType, name: Name): NamedData? = + getData(name)?.castOrNull(type)?.named(name) -public suspend inline fun DataSet<*>.selectOne(name: Name): NamedData? = selectOne(R::class, name) +public suspend inline fun DataSet<*>.selectOne(name: Name): NamedData? = selectOne(typeOf(), name) public suspend inline fun DataSet<*>.selectOne(name: String): NamedData? = - selectOne(R::class, name.toName()) \ No newline at end of file + selectOne(typeOf(), name.toName()) \ No newline at end of file diff --git a/dataforge-tables/src/jvmMain/kotlin/hep/dataforge/tables/CastColumn.kt b/dataforge-tables/src/jvmMain/kotlin/hep/dataforge/tables/CastColumn.kt index 6990f1c6..0ab6c515 100644 --- a/dataforge-tables/src/jvmMain/kotlin/hep/dataforge/tables/CastColumn.kt +++ b/dataforge-tables/src/jvmMain/kotlin/hep/dataforge/tables/CastColumn.kt @@ -4,8 +4,8 @@ import hep.dataforge.meta.Meta import kotlin.properties.ReadOnlyProperty import kotlin.reflect.KClass import kotlin.reflect.KProperty -import kotlin.reflect.full.cast import kotlin.reflect.full.isSubclassOf +import kotlin.reflect.safeCast @Suppress("UNCHECKED_CAST") public fun Column<*>.cast(type: KClass): Column { @@ -22,7 +22,7 @@ public class CastColumn(private val origin: Column<*>, override val typ override val size: Int get() = origin.size - override fun get(index: Int): T? = type.cast(origin[index]) + override fun get(index: Int): T? = type.safeCast(origin[index]) } public class ColumnProperty(public val table: Table, public val type: KClass) : ReadOnlyProperty> { From 730ac69544fb26b8b350c7da28e8414bda4dc649 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Fri, 5 Feb 2021 09:49:35 +0300 Subject: [PATCH 3/7] build tools 0.7.5 and JVM-IR --- .../kotlin/hep/dataforge/properties/bindings.kt | 4 ++-- dataforge-io/dataforge-io-yaml/build.gradle.kts | 15 +++++++++------ .../io/yaml/FrontMatterEnvelopeFormat.kt | 5 ++--- .../hep/dataforge/io/yaml/YamlMetaFormat.kt | 7 +++---- .../hep/dataforge/io/yaml/YamlMetaFormatTest.kt | 0 .../kotlin/hep/dataforge/io/FileBinaryTest.kt | 2 +- .../kotlin/hep/dataforge/io/FileEnvelopeTest.kt | 4 ++-- settings.gradle.kts | 4 ++-- 8 files changed, 21 insertions(+), 20 deletions(-) rename dataforge-io/dataforge-io-yaml/src/{commonMain => jvmMain}/kotlin/hep/dataforge/io/yaml/FrontMatterEnvelopeFormat.kt (97%) rename dataforge-io/dataforge-io-yaml/src/{commonMain => jvmMain}/kotlin/hep/dataforge/io/yaml/YamlMetaFormat.kt (94%) rename dataforge-io/dataforge-io-yaml/src/{commonTest => jvmTest}/kotlin/hep/dataforge/io/yaml/YamlMetaFormatTest.kt (100%) diff --git a/dataforge-context/src/jsMain/kotlin/hep/dataforge/properties/bindings.kt b/dataforge-context/src/jsMain/kotlin/hep/dataforge/properties/bindings.kt index 05818cae..7ef8d72d 100644 --- a/dataforge-context/src/jsMain/kotlin/hep/dataforge/properties/bindings.kt +++ b/dataforge-context/src/jsMain/kotlin/hep/dataforge/properties/bindings.kt @@ -4,7 +4,7 @@ import hep.dataforge.meta.DFExperimental import org.w3c.dom.HTMLInputElement @DFExperimental -fun HTMLInputElement.bindValue(property: Property) { +public fun HTMLInputElement.bindValue(property: Property) { if (this.onchange != null) error("Input element already bound") this.onchange = { property.value = this.value @@ -18,7 +18,7 @@ fun HTMLInputElement.bindValue(property: Property) { } @DFExperimental -fun HTMLInputElement.bindChecked(property: Property) { +public fun HTMLInputElement.bindChecked(property: Property) { if (this.onchange != null) error("Input element already bound") this.onchange = { property.value = this.checked diff --git a/dataforge-io/dataforge-io-yaml/build.gradle.kts b/dataforge-io/dataforge-io-yaml/build.gradle.kts index b8ce9caf..d325a8c8 100644 --- a/dataforge-io/dataforge-io-yaml/build.gradle.kts +++ b/dataforge-io/dataforge-io-yaml/build.gradle.kts @@ -1,14 +1,10 @@ plugins { id("ru.mipt.npm.mpp") - id("ru.mipt.npm.native") +// id("ru.mipt.npm.native") } description = "YAML meta IO" -repositories{ - jcenter() -} - kscience { useSerialization{ yamlKt() @@ -20,8 +16,15 @@ kotlin { commonMain{ dependencies { api(project(":dataforge-io")) -// api("net.mamoe.yamlkt:yamlkt:${ru.mipt.npm.gradle.KScienceVersions.Serialization.yamlKtVersion}") + //api("net.mamoe.yamlkt:yamlkt:${ru.mipt.npm.gradle.KScienceVersions.Serialization.yamlKtVersion}") } } } } + +readme{ + maturity = ru.mipt.npm.gradle.Maturity.PROTOTYPE + description =""" + YAML meta converters and Front Matter envelope format + """.trimIndent() +} diff --git a/dataforge-io/dataforge-io-yaml/src/commonMain/kotlin/hep/dataforge/io/yaml/FrontMatterEnvelopeFormat.kt b/dataforge-io/dataforge-io-yaml/src/jvmMain/kotlin/hep/dataforge/io/yaml/FrontMatterEnvelopeFormat.kt similarity index 97% rename from dataforge-io/dataforge-io-yaml/src/commonMain/kotlin/hep/dataforge/io/yaml/FrontMatterEnvelopeFormat.kt rename to dataforge-io/dataforge-io-yaml/src/jvmMain/kotlin/hep/dataforge/io/yaml/FrontMatterEnvelopeFormat.kt index 254234dd..69100422 100644 --- a/dataforge-io/dataforge-io-yaml/src/commonMain/kotlin/hep/dataforge/io/yaml/FrontMatterEnvelopeFormat.kt +++ b/dataforge-io/dataforge-io-yaml/src/jvmMain/kotlin/hep/dataforge/io/yaml/FrontMatterEnvelopeFormat.kt @@ -17,8 +17,7 @@ public class FrontMatterEnvelopeFormat( ) : EnvelopeFormat { override fun readPartial(input: Input): PartialEnvelope { - @Suppress("VARIABLE_WITH_REDUNDANT_INITIALIZER") - var line = "" + var line: String var offset = 0u do { line = input.readUtf8Line() //?: error("Input does not contain front matter separator") @@ -44,7 +43,7 @@ public class FrontMatterEnvelopeFormat( } override fun readObject(input: Input): Envelope { - var line = "" + var line: String do { line = input.readUtf8Line() //?: error("Input does not contain front matter separator") } while (!line.startsWith(SEPARATOR)) diff --git a/dataforge-io/dataforge-io-yaml/src/commonMain/kotlin/hep/dataforge/io/yaml/YamlMetaFormat.kt b/dataforge-io/dataforge-io-yaml/src/jvmMain/kotlin/hep/dataforge/io/yaml/YamlMetaFormat.kt similarity index 94% rename from dataforge-io/dataforge-io-yaml/src/commonMain/kotlin/hep/dataforge/io/yaml/YamlMetaFormat.kt rename to dataforge-io/dataforge-io-yaml/src/jvmMain/kotlin/hep/dataforge/io/yaml/YamlMetaFormat.kt index e63a3b3e..160683a6 100644 --- a/dataforge-io/dataforge-io-yaml/src/commonMain/kotlin/hep/dataforge/io/yaml/YamlMetaFormat.kt +++ b/dataforge-io/dataforge-io-yaml/src/jvmMain/kotlin/hep/dataforge/io/yaml/YamlMetaFormat.kt @@ -86,16 +86,15 @@ public fun YamlMap.toMeta(): Meta = YamlMeta(this) */ @DFExperimental public class YamlMetaFormat(private val meta: Meta) : MetaFormat { - private val coder = Yaml.default override fun writeMeta(output: Output, meta: Meta, descriptor: NodeDescriptor?) { val yaml = meta.toYaml() - val string = coder.encodeToString(yaml) + val string = Yaml.encodeToString(yaml) output.writeUtf8String(string) } override fun readMeta(input: Input, descriptor: NodeDescriptor?): Meta { - val yaml = coder.decodeYamlMapFromString(input.readUtf8String()) + val yaml = Yaml.decodeYamlMapFromString(input.readUtf8String()) return yaml.toMeta() } @@ -116,7 +115,7 @@ public class YamlMetaFormat(private val meta: Meta) : MetaFormat { override fun writeMeta(output: Output, meta: Meta, descriptor: NodeDescriptor?): Unit = default.writeMeta(output, meta, descriptor) - override fun readMeta(input: kotlinx.io.Input, descriptor: NodeDescriptor?): Meta = + override fun readMeta(input: Input, descriptor: NodeDescriptor?): Meta = default.readMeta(input, descriptor) } } \ No newline at end of file diff --git a/dataforge-io/dataforge-io-yaml/src/commonTest/kotlin/hep/dataforge/io/yaml/YamlMetaFormatTest.kt b/dataforge-io/dataforge-io-yaml/src/jvmTest/kotlin/hep/dataforge/io/yaml/YamlMetaFormatTest.kt similarity index 100% rename from dataforge-io/dataforge-io-yaml/src/commonTest/kotlin/hep/dataforge/io/yaml/YamlMetaFormatTest.kt rename to dataforge-io/dataforge-io-yaml/src/jvmTest/kotlin/hep/dataforge/io/yaml/YamlMetaFormatTest.kt diff --git a/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/FileBinaryTest.kt b/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/FileBinaryTest.kt index 7ccc65e7..4971bf6c 100644 --- a/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/FileBinaryTest.kt +++ b/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/FileBinaryTest.kt @@ -54,7 +54,7 @@ class FileBinaryTest { val tmpPath = Files.createTempFile("dataforge_test", ".df") Global.io.writeEnvelopeFile(tmpPath, envelope) - val binary = Global.io.readEnvelopeFile(tmpPath)?.data!! + val binary = Global.io.readEnvelopeFile(tmpPath).data!! assertEquals(binary.size, binary.toByteArray().size) } } \ No newline at end of file diff --git a/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/FileEnvelopeTest.kt b/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/FileEnvelopeTest.kt index 8db2bf62..ab96c324 100644 --- a/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/FileEnvelopeTest.kt +++ b/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/FileEnvelopeTest.kt @@ -29,7 +29,7 @@ class FileEnvelopeTest { val tmpPath = Files.createTempFile("dataforge_test", ".df") writeEnvelopeFile(tmpPath, envelope) println(tmpPath.toUri()) - val restored: Envelope = readEnvelopeFile(tmpPath)!! + val restored: Envelope = readEnvelopeFile(tmpPath) assertTrue { envelope.contentEquals(restored) } } } @@ -40,7 +40,7 @@ class FileEnvelopeTest { val tmpPath = Files.createTempFile("dataforge_test_tagless", ".df") writeEnvelopeFile(tmpPath, envelope, envelopeFormat = TaglessEnvelopeFormat) println(tmpPath.toUri()) - val restored: Envelope = readEnvelopeFile(tmpPath)!! + val restored: Envelope = readEnvelopeFile(tmpPath) assertTrue { envelope.contentEquals(restored) } } } diff --git a/settings.gradle.kts b/settings.gradle.kts index 96a64daf..c97b21a8 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -10,8 +10,8 @@ pluginManagement { maven("https://dl.bintray.com/mipt-npm/dev") } - val toolsVersion = "0.7.4" - val kotlinVersion = "1.4.30-RC" + val toolsVersion = "0.7.5" + val kotlinVersion = "1.4.30" plugins { id("ru.mipt.npm.project") version toolsVersion From 81fb064d38ff7ef0ae345eba941c2182d53c64e5 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Sun, 7 Feb 2021 12:46:15 +0300 Subject: [PATCH 4/7] Maturity declarations. --- build.gradle.kts | 6 +----- dataforge-context/build.gradle.kts | 4 ++++ dataforge-io/build.gradle.kts | 4 ++++ dataforge-io/dataforge-io-yaml/build.gradle.kts | 7 +++++-- .../hep/dataforge/io/yaml/FrontMatterEnvelopeFormat.kt | 0 .../kotlin/hep/dataforge/io/yaml/YamlMetaFormat.kt | 0 .../kotlin/hep/dataforge/io/yaml/YamlMetaFormatTest.kt | 0 dataforge-meta/build.gradle.kts | 6 +++++- dataforge-scripting/build.gradle.kts | 4 ++++ dataforge-tables/build.gradle.kts | 4 ++++ dataforge-workspace/build.gradle.kts | 4 ++++ settings.gradle.kts | 2 +- 12 files changed, 32 insertions(+), 9 deletions(-) rename dataforge-io/dataforge-io-yaml/src/{jvmMain => commonMain}/kotlin/hep/dataforge/io/yaml/FrontMatterEnvelopeFormat.kt (100%) rename dataforge-io/dataforge-io-yaml/src/{jvmMain => commonMain}/kotlin/hep/dataforge/io/yaml/YamlMetaFormat.kt (100%) rename dataforge-io/dataforge-io-yaml/src/{jvmTest => commonTest}/kotlin/hep/dataforge/io/yaml/YamlMetaFormatTest.kt (100%) diff --git a/build.gradle.kts b/build.gradle.kts index acdfeb81..cc74ad20 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -2,7 +2,7 @@ plugins { id("ru.mipt.npm.project") } -val dataforgeVersion by extra("0.3.0-dev-3") +val dataforgeVersion by extra("0.3.0") val bintrayRepo by extra("dataforge") val githubProject by extra("dataforge-core") @@ -13,10 +13,6 @@ allprojects { version = dataforgeVersion apply() - - repositories { - mavenLocal() - } } apiValidation{ diff --git a/dataforge-context/build.gradle.kts b/dataforge-context/build.gradle.kts index 01a7e3bb..84785ce8 100644 --- a/dataforge-context/build.gradle.kts +++ b/dataforge-context/build.gradle.kts @@ -28,4 +28,8 @@ kotlin { } } } +} + +readme{ + maturity = ru.mipt.npm.gradle.Maturity.DEVELOPMENT } \ No newline at end of file diff --git a/dataforge-io/build.gradle.kts b/dataforge-io/build.gradle.kts index d2e3a41a..01f749cb 100644 --- a/dataforge-io/build.gradle.kts +++ b/dataforge-io/build.gradle.kts @@ -22,4 +22,8 @@ kotlin { } } } +} + +readme{ + maturity = ru.mipt.npm.gradle.Maturity.PROTOTYPE } \ No newline at end of file diff --git a/dataforge-io/dataforge-io-yaml/build.gradle.kts b/dataforge-io/dataforge-io-yaml/build.gradle.kts index d325a8c8..4ee1029a 100644 --- a/dataforge-io/dataforge-io-yaml/build.gradle.kts +++ b/dataforge-io/dataforge-io-yaml/build.gradle.kts @@ -7,16 +7,19 @@ description = "YAML meta IO" kscience { useSerialization{ - yamlKt() + yamlKt("0.9.0-dev-1") } } +repositories{ + maven("https://dl.bintray.com/mamoe/yamlkt") +} + kotlin { sourceSets { commonMain{ dependencies { api(project(":dataforge-io")) - //api("net.mamoe.yamlkt:yamlkt:${ru.mipt.npm.gradle.KScienceVersions.Serialization.yamlKtVersion}") } } } diff --git a/dataforge-io/dataforge-io-yaml/src/jvmMain/kotlin/hep/dataforge/io/yaml/FrontMatterEnvelopeFormat.kt b/dataforge-io/dataforge-io-yaml/src/commonMain/kotlin/hep/dataforge/io/yaml/FrontMatterEnvelopeFormat.kt similarity index 100% rename from dataforge-io/dataforge-io-yaml/src/jvmMain/kotlin/hep/dataforge/io/yaml/FrontMatterEnvelopeFormat.kt rename to dataforge-io/dataforge-io-yaml/src/commonMain/kotlin/hep/dataforge/io/yaml/FrontMatterEnvelopeFormat.kt diff --git a/dataforge-io/dataforge-io-yaml/src/jvmMain/kotlin/hep/dataforge/io/yaml/YamlMetaFormat.kt b/dataforge-io/dataforge-io-yaml/src/commonMain/kotlin/hep/dataforge/io/yaml/YamlMetaFormat.kt similarity index 100% rename from dataforge-io/dataforge-io-yaml/src/jvmMain/kotlin/hep/dataforge/io/yaml/YamlMetaFormat.kt rename to dataforge-io/dataforge-io-yaml/src/commonMain/kotlin/hep/dataforge/io/yaml/YamlMetaFormat.kt diff --git a/dataforge-io/dataforge-io-yaml/src/jvmTest/kotlin/hep/dataforge/io/yaml/YamlMetaFormatTest.kt b/dataforge-io/dataforge-io-yaml/src/commonTest/kotlin/hep/dataforge/io/yaml/YamlMetaFormatTest.kt similarity index 100% rename from dataforge-io/dataforge-io-yaml/src/jvmTest/kotlin/hep/dataforge/io/yaml/YamlMetaFormatTest.kt rename to dataforge-io/dataforge-io-yaml/src/commonTest/kotlin/hep/dataforge/io/yaml/YamlMetaFormatTest.kt diff --git a/dataforge-meta/build.gradle.kts b/dataforge-meta/build.gradle.kts index 980924a7..45ca68f3 100644 --- a/dataforge-meta/build.gradle.kts +++ b/dataforge-meta/build.gradle.kts @@ -9,4 +9,8 @@ kscience { } } -description = "Meta definition and basic operations on meta" \ No newline at end of file +description = "Meta definition and basic operations on meta" + +readme{ + maturity = ru.mipt.npm.gradle.Maturity.DEVELOPMENT +} \ No newline at end of file diff --git a/dataforge-scripting/build.gradle.kts b/dataforge-scripting/build.gradle.kts index cb16b66e..e0c47057 100644 --- a/dataforge-scripting/build.gradle.kts +++ b/dataforge-scripting/build.gradle.kts @@ -22,4 +22,8 @@ kotlin { } } } +} + +readme{ + maturity = ru.mipt.npm.gradle.Maturity.PROTOTYPE } \ No newline at end of file diff --git a/dataforge-tables/build.gradle.kts b/dataforge-tables/build.gradle.kts index e88f70d1..d0f008c3 100644 --- a/dataforge-tables/build.gradle.kts +++ b/dataforge-tables/build.gradle.kts @@ -12,4 +12,8 @@ kotlin { } } } +} + +readme{ + maturity = ru.mipt.npm.gradle.Maturity.PROTOTYPE } \ No newline at end of file diff --git a/dataforge-workspace/build.gradle.kts b/dataforge-workspace/build.gradle.kts index 2bb03cec..ae83b062 100644 --- a/dataforge-workspace/build.gradle.kts +++ b/dataforge-workspace/build.gradle.kts @@ -13,4 +13,8 @@ kotlin { } } } +} + +readme{ + maturity = ru.mipt.npm.gradle.Maturity.EXPERIMENTAL } \ No newline at end of file diff --git a/settings.gradle.kts b/settings.gradle.kts index c97b21a8..defbc0e5 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -10,7 +10,7 @@ pluginManagement { maven("https://dl.bintray.com/mipt-npm/dev") } - val toolsVersion = "0.7.5" + val toolsVersion = "0.7.6" val kotlinVersion = "1.4.30" plugins { From 1970243785e67cb583e7be5b513f3e42089f145e Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Sun, 7 Feb 2021 18:15:54 +0300 Subject: [PATCH 5/7] Remove KClass from dataforge-data --- .../hep/dataforge/context/ContextBuilder.kt | 11 ++++- .../kotlin/hep/dataforge/context/resolve.kt | 2 +- .../dataforge/properties/ConfigProperty.kt | 2 +- .../hep/dataforge/properties/Property.kt | 2 +- .../hep/dataforge/properties/bindings.kt | 2 +- .../hep/dataforge/descriptors/annotations.kt | 1 - .../kotlin/hep/dataforge/provider/dfType.kt | 2 +- .../kotlin/hep/dataforge/actions/Action.kt | 3 +- .../kotlin/hep/dataforge/actions/MapAction.kt | 17 +++++-- .../hep/dataforge/actions/ReduceAction.kt | 34 ++++++++------ .../hep/dataforge/actions/SplitAction.kt | 22 ++++++++- .../hep/dataforge/data/CachingAction.kt | 1 - .../hep/dataforge/data/CoroutineMonitor.kt | 2 +- .../kotlin/hep/dataforge/data/Data.kt | 23 +++++++--- .../kotlin/hep/dataforge/data/DataSet.kt | 5 +- .../hep/dataforge/data/DataSetBuilder.kt | 26 ++++++----- .../kotlin/hep/dataforge/data/DataTree.kt | 15 +++--- .../kotlin/hep/dataforge/data/Goal.kt | 2 +- .../kotlin/hep/dataforge/data/GroupRule.kt | 10 ++-- .../dataforge/{actions => data}/NamedData.kt | 4 +- .../kotlin/hep/dataforge/data/dataFilter.kt | 12 ++--- .../hep/dataforge/data/dataTransform.kt | 46 ++++++++----------- .../kotlin/hep/dataforge/data/select.kt | 4 +- .../kotlin/hep/dataforge/data/ActionsTest.kt | 17 +++---- .../hep/dataforge/data/DataTreeBuilderTest.kt | 19 ++++---- .../io/yaml/FrontMatterEnvelopeFormat.kt | 2 +- .../hep/dataforge/io/yaml/YamlMetaFormat.kt | 1 + .../dataforge/io/yaml/YamlMetaFormatTest.kt | 2 +- .../kotlin/hep/dataforge/io/Consumer.kt | 2 +- .../kotlin/hep/dataforge/io/MultipartTest.kt | 2 +- .../jvmMain/kotlin/hep/dataforge/io/fileIO.kt | 2 +- .../kotlin/hep/dataforge/io/FileBinaryTest.kt | 2 +- .../hep/dataforge/io/FileEnvelopeTest.kt | 2 +- .../kotlin/hep/dataforge/meta/Configurable.kt | 1 + .../kotlin/hep/dataforge/meta/MetaBuilder.kt | 1 + .../kotlin/hep/dataforge/meta/MutableMeta.kt | 1 + .../kotlin/hep/dataforge/meta/annotations.kt | 11 ----- .../meta/descriptors/ItemDescriptor.kt | 1 + .../kotlin/hep/dataforge/meta/mapMeta.kt | 1 + .../transformations/MetaTransformation.kt | 1 + .../kotlin/hep/dataforge/misc/annotations.kt | 21 +++++++++ .../kotlin/hep/dataforge/names/Name.kt | 3 +- .../kotlin/hep/dataforge/names/nameMatcher.kt | 2 +- .../kotlin/hep/dataforge/meta/MetaTest.kt | 1 + .../hep/dataforge/names/NameMatchTest.kt | 2 +- .../dataforge/tables/io/textTableEnvelope.kt | 1 + .../hep/dataforge/tables/io/TextRowsTest.kt | 2 +- .../kotlin/hep/dataforge/workspace/Task.kt | 12 +++-- .../hep/dataforge/workspace/TaskData.kt | 2 +- .../dataforge/workspace/WorkspaceBuilder.kt | 15 ++---- .../hep/dataforge/workspace/envelopeData.kt | 7 ++- .../hep/dataforge/workspace/fileData.kt | 7 +-- .../hep/dataforge/workspace/FileDataTest.kt | 2 +- .../workspace/SimpleWorkspaceTest.kt | 2 +- 54 files changed, 223 insertions(+), 172 deletions(-) rename dataforge-data/src/commonMain/kotlin/hep/dataforge/{actions => data}/NamedData.kt (89%) delete mode 100644 dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/annotations.kt create mode 100644 dataforge-meta/src/commonMain/kotlin/hep/dataforge/misc/annotations.kt diff --git a/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/ContextBuilder.kt b/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/ContextBuilder.kt index 69a68026..f124d3d2 100644 --- a/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/ContextBuilder.kt +++ b/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/ContextBuilder.kt @@ -1,7 +1,16 @@ package hep.dataforge.context -import hep.dataforge.meta.* +import hep.dataforge.meta.Meta +import hep.dataforge.meta.MetaBuilder +import hep.dataforge.meta.seal +import hep.dataforge.misc.DFBuilder +import hep.dataforge.misc.DFExperimental import hep.dataforge.names.toName +import kotlin.collections.HashMap +import kotlin.collections.component1 +import kotlin.collections.component2 +import kotlin.collections.forEach +import kotlin.collections.set /** * A convenience builder for context diff --git a/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/resolve.kt b/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/resolve.kt index bfc4e22d..b41e11fd 100644 --- a/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/resolve.kt +++ b/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/resolve.kt @@ -1,6 +1,6 @@ package hep.dataforge.context -import hep.dataforge.meta.DFExperimental +import hep.dataforge.misc.DFExperimental import hep.dataforge.names.Name import hep.dataforge.names.plus import hep.dataforge.provider.Provider diff --git a/dataforge-context/src/commonMain/kotlin/hep/dataforge/properties/ConfigProperty.kt b/dataforge-context/src/commonMain/kotlin/hep/dataforge/properties/ConfigProperty.kt index 98a43b69..86c6bcde 100644 --- a/dataforge-context/src/commonMain/kotlin/hep/dataforge/properties/ConfigProperty.kt +++ b/dataforge-context/src/commonMain/kotlin/hep/dataforge/properties/ConfigProperty.kt @@ -1,12 +1,12 @@ package hep.dataforge.properties import hep.dataforge.meta.Config -import hep.dataforge.meta.DFExperimental import hep.dataforge.meta.get import hep.dataforge.meta.set import hep.dataforge.meta.transformations.MetaConverter import hep.dataforge.meta.transformations.nullableItemToObject import hep.dataforge.meta.transformations.nullableObjectToMetaItem +import hep.dataforge.misc.DFExperimental import hep.dataforge.names.Name @DFExperimental diff --git a/dataforge-context/src/commonMain/kotlin/hep/dataforge/properties/Property.kt b/dataforge-context/src/commonMain/kotlin/hep/dataforge/properties/Property.kt index 987cfe4c..44e97ea9 100644 --- a/dataforge-context/src/commonMain/kotlin/hep/dataforge/properties/Property.kt +++ b/dataforge-context/src/commonMain/kotlin/hep/dataforge/properties/Property.kt @@ -1,6 +1,6 @@ package hep.dataforge.properties -import hep.dataforge.meta.DFExperimental +import hep.dataforge.misc.DFExperimental import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.ExperimentalCoroutinesApi import kotlinx.coroutines.flow.MutableStateFlow diff --git a/dataforge-context/src/jsMain/kotlin/hep/dataforge/properties/bindings.kt b/dataforge-context/src/jsMain/kotlin/hep/dataforge/properties/bindings.kt index 7ef8d72d..86296e39 100644 --- a/dataforge-context/src/jsMain/kotlin/hep/dataforge/properties/bindings.kt +++ b/dataforge-context/src/jsMain/kotlin/hep/dataforge/properties/bindings.kt @@ -1,6 +1,6 @@ package hep.dataforge.properties -import hep.dataforge.meta.DFExperimental +import hep.dataforge.misc.DFExperimental import org.w3c.dom.HTMLInputElement @DFExperimental diff --git a/dataforge-context/src/jvmMain/kotlin/hep/dataforge/descriptors/annotations.kt b/dataforge-context/src/jvmMain/kotlin/hep/dataforge/descriptors/annotations.kt index cadd4231..f88cee99 100644 --- a/dataforge-context/src/jvmMain/kotlin/hep/dataforge/descriptors/annotations.kt +++ b/dataforge-context/src/jvmMain/kotlin/hep/dataforge/descriptors/annotations.kt @@ -16,7 +16,6 @@ package hep.dataforge.descriptors -import hep.dataforge.meta.DFExperimental import hep.dataforge.values.ValueType import kotlin.reflect.KClass diff --git a/dataforge-context/src/jvmMain/kotlin/hep/dataforge/provider/dfType.kt b/dataforge-context/src/jvmMain/kotlin/hep/dataforge/provider/dfType.kt index 65817b64..6f7855a9 100644 --- a/dataforge-context/src/jvmMain/kotlin/hep/dataforge/provider/dfType.kt +++ b/dataforge-context/src/jvmMain/kotlin/hep/dataforge/provider/dfType.kt @@ -2,7 +2,7 @@ package hep.dataforge.provider import hep.dataforge.context.Context import hep.dataforge.context.gather -import hep.dataforge.meta.DFExperimental +import hep.dataforge.misc.DFExperimental import hep.dataforge.misc.Type import hep.dataforge.names.Name import kotlin.reflect.KClass diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/actions/Action.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/actions/Action.kt index a7c51e8f..85ec2977 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/actions/Action.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/actions/Action.kt @@ -1,10 +1,9 @@ package hep.dataforge.actions import hep.dataforge.data.DataSet -import hep.dataforge.meta.DFExperimental import hep.dataforge.meta.Meta +import hep.dataforge.misc.DFExperimental import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.flow.* /** * A simple data transformation on a data node. Actions should avoid doing actual dependency evaluation in [execute]. diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/actions/MapAction.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/actions/MapAction.kt index 77acc6a4..23731621 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/actions/MapAction.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/actions/MapAction.kt @@ -1,7 +1,13 @@ package hep.dataforge.actions import hep.dataforge.data.* -import hep.dataforge.meta.* +import hep.dataforge.meta.Meta +import hep.dataforge.meta.MetaBuilder +import hep.dataforge.meta.seal +import hep.dataforge.meta.toMutableMeta +import hep.dataforge.misc.DFBuilder +import hep.dataforge.misc.DFExperimental +import hep.dataforge.misc.DFInternal import hep.dataforge.names.Name import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.flow.collect @@ -34,7 +40,6 @@ public class MapActionBuilder(public var name: Name, public var meta: Meta } } - @PublishedApi internal class MapAction( private val outputType: KType, @@ -63,7 +68,9 @@ internal class MapAction( //getting new meta val newMeta = builder.meta.seal() - val newData = data.map(outputType, meta = newMeta) { builder.result(env, it) } + @OptIn(DFInternal::class) val newData = Data(outputType, newMeta, dependencies = listOf(data)) { + builder.result(env, data.await()) + } //setting the data node return newData.named(newName) } @@ -85,6 +92,10 @@ internal class MapAction( } +/** + * A one-to-one mapping action + */ +@DFExperimental @Suppress("FunctionName") public inline fun Action.Companion.map( noinline builder: MapActionBuilder.() -> Unit, diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/actions/ReduceAction.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/actions/ReduceAction.kt index f3cf9fdc..8e2781b5 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/actions/ReduceAction.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/actions/ReduceAction.kt @@ -1,20 +1,21 @@ package hep.dataforge.actions import hep.dataforge.data.* -import hep.dataforge.meta.DFExperimental import hep.dataforge.meta.Meta import hep.dataforge.meta.MetaBuilder +import hep.dataforge.misc.DFBuilder +import hep.dataforge.misc.DFExperimental +import hep.dataforge.misc.DFInternal import hep.dataforge.names.Name import hep.dataforge.names.toName import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.flow import kotlinx.coroutines.flow.fold -import kotlin.reflect.KClass import kotlin.reflect.KType +import kotlin.reflect.typeOf -@DFExperimental public class JoinGroup(public var name: String, internal val set: DataSet) { public var meta: MetaBuilder = MetaBuilder() @@ -27,9 +28,9 @@ public class JoinGroup(public var name: String, internal val s } -@DFExperimental +@DFBuilder public class ReduceGroupBuilder( - private val inputType: KClass, + private val inputType: KType, private val scope: CoroutineScope, public val actionMeta: Meta, ) { @@ -40,7 +41,7 @@ public class ReduceGroupBuilder( */ public fun byValue(tag: String, defaultTag: String = "@default", action: JoinGroup.() -> Unit) { groupRules += { node -> - GroupRule.byMetaValue(scope, tag, defaultTag).gather(inputType, node).map { + GroupRule.byMetaValue(scope, tag, defaultTag).gather(node).map { JoinGroup(it.key, it.value).apply(action) } } @@ -73,16 +74,17 @@ public class ReduceGroupBuilder( } -@DFExperimental -public class ReduceAction( - private val inputType: KClass, +@PublishedApi +internal class ReduceAction( + private val inputType: KType, outputType: KType, private val action: ReduceGroupBuilder.() -> Unit, ) : CachingAction(outputType) { //TODO optimize reduction. Currently the whole action recalculates on push + override fun CoroutineScope.transform(set: DataSet, meta: Meta, key: Name): Flow> = flow { - ReduceGroupBuilder(inputType,this@transform, meta).apply(action).buildGroups(set).forEach { group -> + ReduceGroupBuilder(inputType, this@transform, meta).apply(action).buildGroups(set).forEach { group -> val dataFlow: Map> = group.set.flow().fold(HashMap()) { acc, value -> acc.apply { acc[value.name] = value.data @@ -94,8 +96,7 @@ public class ReduceAction( val groupMeta = group.meta val env = ActionEnv(groupName.toName(), groupMeta, meta) - - val res: LazyData = dataFlow.reduceToData( + @OptIn(DFInternal::class) val res: Data = dataFlow.reduceToData( outputType, meta = groupMeta ) { group.result.invoke(env, it) } @@ -105,4 +106,11 @@ public class ReduceAction( } } -public operator fun Map.get(name: String): T? = get(name.toName()) +/** + * A one-to-one mapping action + */ +@DFExperimental +@Suppress("FunctionName") +public inline fun Action.Companion.reduce( + noinline builder: ReduceGroupBuilder.() -> Unit, +): Action = ReduceAction(typeOf(), typeOf(), builder) diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/actions/SplitAction.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/actions/SplitAction.kt index c659d5a1..37b8f734 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/actions/SplitAction.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/actions/SplitAction.kt @@ -5,13 +5,17 @@ import hep.dataforge.meta.Laminate import hep.dataforge.meta.Meta import hep.dataforge.meta.MetaBuilder import hep.dataforge.meta.toMutableMeta +import hep.dataforge.misc.DFExperimental +import hep.dataforge.misc.DFInternal import hep.dataforge.names.Name import hep.dataforge.names.toName import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.FlowPreview import kotlinx.coroutines.flow.* import kotlinx.coroutines.launch import kotlin.collections.set import kotlin.reflect.KType +import kotlin.reflect.typeOf public class SplitBuilder(public val name: Name, public val meta: Meta) { @@ -39,11 +43,13 @@ public class SplitBuilder(public val name: Name, public val me /** * Action that splits each incoming element into a number of fragments defined in builder */ +@PublishedApi internal class SplitAction( private val outputType: KType, private val action: SplitBuilder.() -> Unit, ) : Action { + @OptIn(FlowPreview::class) override suspend fun execute( dataSet: DataSet, meta: Meta, @@ -59,7 +65,10 @@ internal class SplitAction( // apply individual fragment rules to result return split.fragments.entries.asFlow().map { (fragmentName, rule) -> val env = SplitBuilder.FragmentRule(fragmentName, laminate.toMutableMeta()).apply(rule) - data.map(outputType, meta = env.meta) { env.result(it) }.named(fragmentName) + //data.map(outputType, meta = env.meta) { env.result(it) }.named(fragmentName) + @OptIn(DFInternal::class) Data(outputType, meta = env.meta, dependencies = listOf(data)) { + env.result(data.await()) + }.named(fragmentName) } } @@ -75,4 +84,13 @@ internal class SplitAction( } } } -} \ No newline at end of file +} + +/** + * Action that splits each incoming element into a number of fragments defined in builder + */ +@DFExperimental +@Suppress("FunctionName") +public inline fun Action.Companion.split( + noinline builder: SplitBuilder.() -> Unit, +): Action = SplitAction(typeOf(), builder) \ No newline at end of file diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/CachingAction.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/CachingAction.kt index c4fb4f90..7911cb1f 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/CachingAction.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/CachingAction.kt @@ -1,7 +1,6 @@ package hep.dataforge.data import hep.dataforge.actions.Action -import hep.dataforge.actions.NamedData import hep.dataforge.meta.Meta import hep.dataforge.names.Name import hep.dataforge.names.startsWith diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/CoroutineMonitor.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/CoroutineMonitor.kt index d1c0d55e..60bf5775 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/CoroutineMonitor.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/CoroutineMonitor.kt @@ -1,6 +1,6 @@ package hep.dataforge.data -import hep.dataforge.meta.DFExperimental +import hep.dataforge.misc.DFExperimental import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Job import kotlin.coroutines.CoroutineContext diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/Data.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/Data.kt index 0a768ee4..6702de4d 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/Data.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/Data.kt @@ -3,6 +3,7 @@ package hep.dataforge.data import hep.dataforge.meta.Meta import hep.dataforge.meta.MetaRepr import hep.dataforge.meta.isEmpty +import hep.dataforge.misc.DFInternal import hep.dataforge.misc.Type import kotlinx.coroutines.* import kotlin.coroutines.CoroutineContext @@ -35,17 +36,21 @@ public interface Data : Goal, MetaRepr { public companion object { public const val TYPE: String = "data" + /** + * The type that can't have any subtypes + */ + internal val TYPE_OF_NOTHING: KType = typeOf() + public inline fun static( value: T, meta: Meta = Meta.EMPTY, - ): Data = StaticData(typeOf(),value, meta) + ): Data = StaticData(typeOf(), value, meta) /** * An empty data containing only meta */ public fun empty(meta: Meta): Data = object : Data { - private val nothing: Nothing get() = error("this is nothing") - override val type: KType = this::nothing.returnType + override val type: KType = TYPE_OF_NOTHING override val meta: Meta = meta override val dependencies: Collection> = emptyList() override val deferred: Deferred @@ -59,13 +64,17 @@ public interface Data : Goal, MetaRepr { } } -public class LazyData( +/** + * A lazily computed variant of [Data] based on [LazyGoal] + * One must ensure that proper [type] is used so this method should not be used + */ +private class LazyData( override val type: KType, override val meta: Meta = Meta.EMPTY, - context: CoroutineContext = EmptyCoroutineContext, + additionalContext: CoroutineContext = EmptyCoroutineContext, dependencies: Collection> = emptyList(), block: suspend () -> T, -) : Data, LazyGoal(context, dependencies, block) +) : Data, LazyGoal(additionalContext, dependencies, block) public class StaticData( override val type: KType, @@ -74,6 +83,7 @@ public class StaticData( ) : Data, StaticGoal(value) @Suppress("FunctionName") +@DFInternal public fun Data( type: KType, meta: Meta = Meta.EMPTY, @@ -82,6 +92,7 @@ public fun Data( block: suspend () -> T, ): Data = LazyData(type, meta, context, dependencies, block) +@OptIn(DFInternal::class) @Suppress("FunctionName") public inline fun Data( meta: Meta = Meta.EMPTY, diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataSet.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataSet.kt index 129f657f..f2a4bf88 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataSet.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataSet.kt @@ -1,7 +1,6 @@ package hep.dataforge.data -import hep.dataforge.actions.NamedData -import hep.dataforge.actions.named +import hep.dataforge.data.Data.Companion.TYPE_OF_NOTHING import hep.dataforge.meta.Meta import hep.dataforge.meta.set import hep.dataforge.names.* @@ -43,7 +42,7 @@ public interface DataSet { * An empty [DataSet] that suits all types */ public val EMPTY: DataSet = object : DataSet { - override val dataType: KType = this::nothing.returnType + override val dataType: KType = TYPE_OF_NOTHING private val nothing: Nothing get() = error("this is nothing") diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataSetBuilder.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataSetBuilder.kt index f001c4ad..95e9fd06 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataSetBuilder.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataSetBuilder.kt @@ -1,17 +1,19 @@ package hep.dataforge.data -import hep.dataforge.actions.NamedData -import hep.dataforge.meta.DFExperimental import hep.dataforge.meta.Meta import hep.dataforge.meta.MetaBuilder +import hep.dataforge.misc.DFExperimental import hep.dataforge.names.Name import hep.dataforge.names.plus import hep.dataforge.names.toName import kotlinx.coroutines.coroutineScope import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.collect +import kotlin.reflect.KType public interface DataSetBuilder { + public val dataType: KType + /** * Remove all data items starting with [name] */ @@ -50,8 +52,12 @@ public interface DataSetBuilder { public suspend infix fun String.put(block: suspend DataSetBuilder.() -> Unit): Unit = emit(toName(), block) } -private class SubSetBuilder(private val parent: DataSetBuilder, private val branch: Name) : - DataSetBuilder { +private class SubSetBuilder( + private val parent: DataSetBuilder, + private val branch: Name, +) : DataSetBuilder { + override val dataType: KType get() = parent.dataType + override suspend fun remove(name: Name) { parent.remove(branch + name) } @@ -101,7 +107,7 @@ public suspend inline fun DataSetBuilder.produce( name: Name, meta: Meta = Meta.EMPTY, noinline producer: suspend () -> T, -){ +) { val data = Data(meta, block = producer) emit(name, data) } @@ -109,19 +115,17 @@ public suspend inline fun DataSetBuilder.produce( /** * Emit a static data with the fixed value */ -public suspend fun DataSetBuilder.static(name: String, data: T, meta: Meta = Meta.EMPTY): Unit = +public suspend inline fun DataSetBuilder.static(name: String, data: T, meta: Meta = Meta.EMPTY): Unit = emit(name, Data.static(data, meta)) -public suspend fun DataSetBuilder.static(name: Name, data: T, meta: Meta = Meta.EMPTY): Unit = +public suspend inline fun DataSetBuilder.static(name: Name, data: T, meta: Meta = Meta.EMPTY): Unit = emit(name, Data.static(data, meta)) -public suspend fun DataSetBuilder.static( +public suspend inline fun DataSetBuilder.static( name: String, data: T, metaBuilder: MetaBuilder.() -> Unit, -) { - emit(name.toName(), Data.static(data, Meta(metaBuilder))) -} +): Unit = emit(name.toName(), Data.static(data, Meta(metaBuilder))) /** * Update data with given node data and meta with node meta. diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataTree.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataTree.kt index 0354b161..5676eeed 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataTree.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataTree.kt @@ -1,22 +1,21 @@ package hep.dataforge.data -import hep.dataforge.actions.NamedData -import hep.dataforge.actions.named -import hep.dataforge.meta.* import hep.dataforge.misc.Type import hep.dataforge.names.* -import kotlinx.coroutines.* -import kotlinx.coroutines.flow.* +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.emitAll +import kotlinx.coroutines.flow.flow +import kotlinx.coroutines.flow.map import kotlin.collections.component1 import kotlin.collections.component2 -import kotlin.reflect.KClass +import kotlin.reflect.KType public sealed class DataTreeItem { public class Node(public val tree: DataTree) : DataTreeItem() public class Leaf(public val data: Data) : DataTreeItem() } -public val DataTreeItem.type: KClass +public val DataTreeItem.type: KType get() = when (this) { is DataTreeItem.Node -> tree.dataType is DataTreeItem.Leaf -> data.type @@ -91,7 +90,7 @@ public fun DataTree.itemFlow(): Flow>> = * The difference from similar method for [DataSet] is that internal logic is more simple and the return value is a [DataTree] */ public fun DataTree.branch(branchName: Name): DataTree = object : DataTree { - override val dataType: KClass get() = this@branch.dataType + override val dataType: KType get() = this@branch.dataType override suspend fun items(): Map> = getItem(branchName).tree?.items() ?: emptyMap() } diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/Goal.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/Goal.kt index eef9e7cb..7763c3ce 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/Goal.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/Goal.kt @@ -1,6 +1,6 @@ package hep.dataforge.data -import hep.dataforge.meta.DFExperimental +import hep.dataforge.misc.DFExperimental import kotlinx.coroutines.* import kotlin.coroutines.CoroutineContext import kotlin.coroutines.EmptyCoroutineContext diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/GroupRule.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/GroupRule.kt index 414fa17e..fcaa0658 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/GroupRule.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/GroupRule.kt @@ -20,10 +20,9 @@ import hep.dataforge.meta.string import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.flow.collect import kotlinx.coroutines.launch -import kotlin.reflect.KClass public interface GroupRule { - public suspend fun gather(dataType: KClass, set: DataSet): Map> + public suspend fun gather(set: DataSet): Map> public companion object { /** @@ -41,21 +40,20 @@ public interface GroupRule { ): GroupRule = object : GroupRule { override suspend fun gather( - dataType: KClass, set: DataSet, ): Map> { val map = HashMap>() set.flow().collect { data -> val tagValue = data.meta[key]?.string ?: defaultTagValue - map.getOrPut(tagValue) { ActiveDataTree(dataType) }.emit(data.name, data.data) + map.getOrPut(tagValue) { ActiveDataTree(set.dataType) }.emit(data.name, data.data) } scope.launch { set.updates.collect { name -> val data = set.getData(name) val tagValue = data?.meta[key]?.string ?: defaultTagValue - map.getOrPut(tagValue) { ActiveDataTree(dataType) }.emit(name, data) + map.getOrPut(tagValue) { ActiveDataTree(set.dataType) }.emit(name, data) } } @@ -63,4 +61,4 @@ public interface GroupRule { } } } -} +} \ No newline at end of file diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/actions/NamedData.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/NamedData.kt similarity index 89% rename from dataforge-data/src/commonMain/kotlin/hep/dataforge/actions/NamedData.kt rename to dataforge-data/src/commonMain/kotlin/hep/dataforge/data/NamedData.kt index 126c972b..aa5afcdc 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/actions/NamedData.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/NamedData.kt @@ -1,7 +1,5 @@ -package hep.dataforge.actions +package hep.dataforge.data -import hep.dataforge.data.Data -import hep.dataforge.data.StaticData import hep.dataforge.meta.isEmpty import hep.dataforge.misc.Named import hep.dataforge.names.Name diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/dataFilter.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/dataFilter.kt index aff59e61..8f38d100 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/dataFilter.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/dataFilter.kt @@ -1,14 +1,12 @@ package hep.dataforge.data -import hep.dataforge.actions.NamedData -import hep.dataforge.actions.named -import hep.dataforge.meta.DFExperimental +import hep.dataforge.misc.DFExperimental import hep.dataforge.names.* import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.filter import kotlinx.coroutines.flow.map import kotlinx.coroutines.flow.mapNotNull -import kotlin.reflect.KClass +import kotlin.reflect.KType /** @@ -17,7 +15,7 @@ import kotlin.reflect.KClass public fun DataSet.filter( predicate: suspend (Name, Data) -> Boolean, ): ActiveDataSet = object : ActiveDataSet { - override val dataType: KClass get() = this@filter.dataType + override val dataType: KType get() = this@filter.dataType override fun flow(): Flow> = this@filter.flow().filter { predicate(it.name, it.data) } @@ -37,7 +35,7 @@ public fun DataSet.filter( */ public fun DataSet.withNamePrefix(prefix: Name): DataSet = if (prefix.isEmpty()) this else object : ActiveDataSet { - override val dataType: KClass get() = this@withNamePrefix.dataType + override val dataType: KType get() = this@withNamePrefix.dataType override fun flow(): Flow> = this@withNamePrefix.flow().map { it.data.named(prefix + it.name) } @@ -53,7 +51,7 @@ else object : ActiveDataSet { public fun DataSet.branch(branchName: Name): DataSet = if (branchName.isEmpty()) { this } else object : ActiveDataSet { - override val dataType: KClass get() = this@branch.dataType + override val dataType: KType get() = this@branch.dataType override fun flow(): Flow> = this@branch.flow().mapNotNull { it.name.removeHeadOrNull(branchName)?.let { name -> diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/dataTransform.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/dataTransform.kt index fdc24c1c..8b0c7787 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/dataTransform.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/dataTransform.kt @@ -1,11 +1,10 @@ package hep.dataforge.data -import hep.dataforge.actions.NamedData -import hep.dataforge.actions.named import hep.dataforge.meta.Meta import hep.dataforge.meta.MetaBuilder import hep.dataforge.meta.seal import hep.dataforge.meta.toMutableMeta +import hep.dataforge.misc.DFInternal import kotlinx.coroutines.flow.* import kotlin.contracts.InvocationKind import kotlin.contracts.contract @@ -20,23 +19,11 @@ import kotlin.reflect.typeOf * @param meta for the resulting data. By default equals input data. * @param block the transformation itself */ -private fun Data.map( - outputType: KType, - coroutineContext: CoroutineContext = EmptyCoroutineContext, - meta: Meta = this.meta, - block: suspend (T) -> R, -): LazyData = LazyData(outputType, meta, coroutineContext, listOf(this)) { - block(await()) -} - -/** - * See [map] - */ public inline fun Data.map( coroutineContext: CoroutineContext = EmptyCoroutineContext, meta: Meta = this.meta, crossinline block: suspend (T) -> R, -): LazyData = LazyData(typeOf(), meta, coroutineContext, listOf(this)) { +): Data = Data(meta, coroutineContext, listOf(this)) { block(await()) } @@ -48,7 +35,7 @@ public inline fun Data.combine( coroutineContext: CoroutineContext = EmptyCoroutineContext, meta: Meta = this.meta, crossinline block: suspend (left: T1, right: T2) -> R, -): LazyData = LazyData(typeOf(), meta, coroutineContext, listOf(this, other)) { +): Data = Data(meta, coroutineContext, listOf(this, other)) { block(await(), other.await()) } @@ -62,8 +49,7 @@ public inline fun Collection>.reduceToData( coroutineContext: CoroutineContext = EmptyCoroutineContext, meta: Meta = Meta.EMPTY, crossinline block: suspend (Collection) -> R, -): LazyData = LazyData( - typeOf(), +): Data = Data( meta, coroutineContext, this @@ -71,12 +57,13 @@ public inline fun Collection>.reduceToData( block(map { it.await() }) } +@DFInternal public fun Map>.reduceToData( outputType: KType, coroutineContext: CoroutineContext = EmptyCoroutineContext, meta: Meta = Meta.EMPTY, block: suspend (Map) -> R, -): LazyData = LazyData( +): Data = Data( outputType, meta, coroutineContext, @@ -96,8 +83,7 @@ public inline fun Map>.reduceToData( coroutineContext: CoroutineContext = EmptyCoroutineContext, meta: Meta = Meta.EMPTY, noinline block: suspend (Map) -> R, -): LazyData = LazyData( - typeOf(), +): Data = Data( meta, coroutineContext, this.values @@ -110,12 +96,13 @@ public inline fun Map>.reduceToData( /** * Transform a [Flow] of [NamedData] to a single [Data]. */ +@DFInternal public suspend fun Flow>.reduceToData( outputType: KType, coroutineContext: CoroutineContext = EmptyCoroutineContext, meta: Meta = Meta.EMPTY, transformation: suspend (Flow>) -> R, -): LazyData = LazyData( +): Data = Data( outputType, meta, coroutineContext, @@ -124,11 +111,12 @@ public suspend fun Flow>.reduceToData( transformation(this) } +@OptIn(DFInternal::class) public suspend inline fun Flow>.reduceToData( coroutineContext: CoroutineContext = EmptyCoroutineContext, meta: Meta = Meta.EMPTY, noinline transformation: suspend (Flow>) -> R, -): LazyData = reduceToData(typeOf(), coroutineContext, meta) { +): Data = reduceToData(typeOf(), coroutineContext, meta) { transformation(it) } @@ -140,7 +128,7 @@ public suspend inline fun Flow>.foldToDa coroutineContext: CoroutineContext = EmptyCoroutineContext, meta: Meta = Meta.EMPTY, noinline block: suspend (result: R, data: NamedData) -> R, -): LazyData = reduceToData( +): Data = reduceToData( coroutineContext, meta ) { it.fold(initial, block) @@ -148,6 +136,7 @@ public suspend inline fun Flow>.foldToDa //DataSet operations +@DFInternal public suspend fun DataSet.map( outputType: KType, coroutineContext: CoroutineContext = EmptyCoroutineContext, @@ -157,11 +146,14 @@ public suspend fun DataSet.map( populate( flow().map { val newMeta = it.meta.toMutableMeta().apply(metaTransform).seal() - it.map(outputType, coroutineContext, newMeta, block).named(it.name) + Data(outputType, newMeta, coroutineContext, listOf(it)) { + block(it.await()) + }.named(it.name) } ) } +@OptIn(DFInternal::class) public suspend inline fun DataSet.map( coroutineContext: CoroutineContext = EmptyCoroutineContext, noinline metaTransform: MetaBuilder.() -> Unit = {}, @@ -179,11 +171,11 @@ public suspend inline fun DataSet.reduceToData( coroutineContext: CoroutineContext = EmptyCoroutineContext, meta: Meta = Meta.EMPTY, noinline transformation: suspend (Flow>) -> R, -): LazyData = flow().reduceToData(coroutineContext, meta, transformation) +): Data = flow().reduceToData(coroutineContext, meta, transformation) public suspend inline fun DataSet.foldToData( initial: R, coroutineContext: CoroutineContext = EmptyCoroutineContext, meta: Meta = Meta.EMPTY, noinline block: suspend (result: R, data: NamedData) -> R, -): LazyData = flow().foldToData(initial, coroutineContext, meta, block) \ No newline at end of file +): Data = flow().foldToData(initial, coroutineContext, meta, block) \ No newline at end of file diff --git a/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/select.kt b/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/select.kt index def3133a..9df89f84 100644 --- a/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/select.kt +++ b/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/select.kt @@ -1,8 +1,6 @@ package hep.dataforge.data -import hep.dataforge.actions.NamedData -import hep.dataforge.actions.named -import hep.dataforge.meta.DFExperimental +import hep.dataforge.misc.DFExperimental import hep.dataforge.names.Name import hep.dataforge.names.matches import hep.dataforge.names.toName diff --git a/dataforge-data/src/jvmTest/kotlin/hep/dataforge/data/ActionsTest.kt b/dataforge-data/src/jvmTest/kotlin/hep/dataforge/data/ActionsTest.kt index 81049612..0da9c2c2 100644 --- a/dataforge-data/src/jvmTest/kotlin/hep/dataforge/data/ActionsTest.kt +++ b/dataforge-data/src/jvmTest/kotlin/hep/dataforge/data/ActionsTest.kt @@ -1,15 +1,12 @@ package hep.dataforge.data -import hep.dataforge.actions.MapAction +import hep.dataforge.actions.Action +import hep.dataforge.actions.map import kotlinx.coroutines.runBlocking import org.junit.jupiter.api.Test import kotlin.test.assertEquals -/** - * Block the thread and get data content - */ -public fun Data.value(): T = runBlocking { await() } - +@Suppress("EXPERIMENTAL_API_USAGE") class ActionsTest { val data: DataTree = runBlocking { DataTree { @@ -21,23 +18,23 @@ class ActionsTest { @Test fun testStaticMapAction() { - val plusOne = MapAction { + val plusOne = Action.map { result { it + 1 } } runBlocking { val result = plusOne.execute(data) - assertEquals(2, result.getData("1")?.value()) + assertEquals(2, result.getData("1")?.await()) } } @Test fun testDynamicMapAction() { - val plusOne = MapAction { + val plusOne = Action.map { result { it + 1 } } val datum = runBlocking { val result = plusOne.execute(data, scope = this) - result.getData("1")?.value() + result.getData("1")?.await() } assertEquals(2, datum) } diff --git a/dataforge-data/src/jvmTest/kotlin/hep/dataforge/data/DataTreeBuilderTest.kt b/dataforge-data/src/jvmTest/kotlin/hep/dataforge/data/DataTreeBuilderTest.kt index cf34cbb8..0225e9cf 100644 --- a/dataforge-data/src/jvmTest/kotlin/hep/dataforge/data/DataTreeBuilderTest.kt +++ b/dataforge-data/src/jvmTest/kotlin/hep/dataforge/data/DataTreeBuilderTest.kt @@ -18,10 +18,12 @@ internal class DataTreeBuilderTest { static("c.d", "c.d") static("c.f", "c.f") } - assertEquals("a", node.getData("primary.a")?.value()) - assertEquals("b", node.getData("primary.b")?.value()) - assertEquals("c.d", node.getData("c.d")?.value()) - assertEquals("c.f", node.getData("c.f")?.value()) + runBlocking { + assertEquals("a", node.getData("primary.a")?.await()) + assertEquals("b", node.getData("primary.b")?.await()) + assertEquals("c.d", node.getData("c.d")?.await()) + assertEquals("c.f", node.getData("c.f")?.await()) + } } @Test @@ -42,9 +44,10 @@ internal class DataTreeBuilderTest { populate(updateData) } - - assertEquals("a", node.getData("update.a")?.value()) - assertEquals("a", node.getData("primary.a")?.value()) + runBlocking { + assertEquals("a", node.getData("update.a")?.await()) + assertEquals("a", node.getData("primary.a")?.await()) + } } @Test @@ -76,7 +79,7 @@ internal class DataTreeBuilderTest { } } updateJob.join() - assertEquals(9, rootNode.getData("sub.value")?.value()) + assertEquals(9, rootNode.getData("sub.value")?.await()) cancel() } } catch (t: Throwable) { diff --git a/dataforge-io/dataforge-io-yaml/src/commonMain/kotlin/hep/dataforge/io/yaml/FrontMatterEnvelopeFormat.kt b/dataforge-io/dataforge-io-yaml/src/commonMain/kotlin/hep/dataforge/io/yaml/FrontMatterEnvelopeFormat.kt index 69100422..9a6d38f8 100644 --- a/dataforge-io/dataforge-io-yaml/src/commonMain/kotlin/hep/dataforge/io/yaml/FrontMatterEnvelopeFormat.kt +++ b/dataforge-io/dataforge-io-yaml/src/commonMain/kotlin/hep/dataforge/io/yaml/FrontMatterEnvelopeFormat.kt @@ -4,8 +4,8 @@ import hep.dataforge.context.Context import hep.dataforge.io.* import hep.dataforge.io.IOFormat.Companion.META_KEY import hep.dataforge.io.IOFormat.Companion.NAME_KEY -import hep.dataforge.meta.DFExperimental import hep.dataforge.meta.Meta +import hep.dataforge.misc.DFExperimental import kotlinx.io.* import kotlinx.io.text.readUtf8Line import kotlinx.io.text.writeUtf8String diff --git a/dataforge-io/dataforge-io-yaml/src/commonMain/kotlin/hep/dataforge/io/yaml/YamlMetaFormat.kt b/dataforge-io/dataforge-io-yaml/src/commonMain/kotlin/hep/dataforge/io/yaml/YamlMetaFormat.kt index 160683a6..9f5cdda3 100644 --- a/dataforge-io/dataforge-io-yaml/src/commonMain/kotlin/hep/dataforge/io/yaml/YamlMetaFormat.kt +++ b/dataforge-io/dataforge-io-yaml/src/commonMain/kotlin/hep/dataforge/io/yaml/YamlMetaFormat.kt @@ -8,6 +8,7 @@ import hep.dataforge.io.MetaFormatFactory import hep.dataforge.meta.* import hep.dataforge.meta.descriptors.ItemDescriptor import hep.dataforge.meta.descriptors.NodeDescriptor +import hep.dataforge.misc.DFExperimental import hep.dataforge.names.NameToken import hep.dataforge.names.withIndex import hep.dataforge.values.ListValue diff --git a/dataforge-io/dataforge-io-yaml/src/commonTest/kotlin/hep/dataforge/io/yaml/YamlMetaFormatTest.kt b/dataforge-io/dataforge-io-yaml/src/commonTest/kotlin/hep/dataforge/io/yaml/YamlMetaFormatTest.kt index 83f6d3a6..d63da939 100644 --- a/dataforge-io/dataforge-io-yaml/src/commonTest/kotlin/hep/dataforge/io/yaml/YamlMetaFormatTest.kt +++ b/dataforge-io/dataforge-io-yaml/src/commonTest/kotlin/hep/dataforge/io/yaml/YamlMetaFormatTest.kt @@ -2,10 +2,10 @@ package hep.dataforge.io.yaml import hep.dataforge.io.parse import hep.dataforge.io.toString -import hep.dataforge.meta.DFExperimental import hep.dataforge.meta.Meta import hep.dataforge.meta.get import hep.dataforge.meta.seal +import hep.dataforge.misc.DFExperimental import kotlin.test.Test import kotlin.test.assertEquals diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Consumer.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Consumer.kt index 0ef5d327..51ec6250 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Consumer.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Consumer.kt @@ -1,6 +1,6 @@ package hep.dataforge.io -import hep.dataforge.meta.DFExperimental +import hep.dataforge.misc.DFExperimental /** * A fire-and-forget consumer of messages diff --git a/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/MultipartTest.kt b/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/MultipartTest.kt index 1b239edd..e511a472 100644 --- a/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/MultipartTest.kt +++ b/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/MultipartTest.kt @@ -1,9 +1,9 @@ package hep.dataforge.io import hep.dataforge.context.Global -import hep.dataforge.meta.DFExperimental import hep.dataforge.meta.get import hep.dataforge.meta.int +import hep.dataforge.misc.DFExperimental import kotlinx.io.text.writeUtf8String import kotlin.test.Test import kotlin.test.assertEquals diff --git a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/fileIO.kt b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/fileIO.kt index 2f644878..54cf6b2c 100644 --- a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/fileIO.kt +++ b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/fileIO.kt @@ -1,9 +1,9 @@ package hep.dataforge.io -import hep.dataforge.meta.DFExperimental import hep.dataforge.meta.Meta import hep.dataforge.meta.descriptors.NodeDescriptor import hep.dataforge.meta.isEmpty +import hep.dataforge.misc.DFExperimental import kotlinx.io.* import java.nio.file.Files import java.nio.file.Path diff --git a/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/FileBinaryTest.kt b/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/FileBinaryTest.kt index 4971bf6c..8bfefd6d 100644 --- a/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/FileBinaryTest.kt +++ b/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/FileBinaryTest.kt @@ -1,7 +1,7 @@ package hep.dataforge.io import hep.dataforge.context.Global -import hep.dataforge.meta.DFExperimental +import hep.dataforge.misc.DFExperimental import kotlinx.io.asBinary import kotlinx.io.toByteArray import kotlinx.io.writeDouble diff --git a/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/FileEnvelopeTest.kt b/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/FileEnvelopeTest.kt index ab96c324..fba3ed8e 100644 --- a/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/FileEnvelopeTest.kt +++ b/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/FileEnvelopeTest.kt @@ -1,7 +1,7 @@ package hep.dataforge.io import hep.dataforge.context.Global -import hep.dataforge.meta.DFExperimental +import hep.dataforge.misc.DFExperimental import kotlinx.io.writeDouble import java.nio.file.Files import kotlin.test.Test diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Configurable.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Configurable.kt index 764ee1ea..429ac360 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Configurable.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Configurable.kt @@ -1,5 +1,6 @@ package hep.dataforge.meta +import hep.dataforge.misc.DFBuilder import hep.dataforge.names.Name import kotlin.properties.ReadWriteProperty diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MetaBuilder.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MetaBuilder.kt index f2016b80..4fe5de9b 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MetaBuilder.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MetaBuilder.kt @@ -1,5 +1,6 @@ package hep.dataforge.meta +import hep.dataforge.misc.DFBuilder import hep.dataforge.names.Name import hep.dataforge.names.asName import hep.dataforge.values.EnumValue diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MutableMeta.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MutableMeta.kt index f5353cbd..4524f11e 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MutableMeta.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MutableMeta.kt @@ -1,5 +1,6 @@ package hep.dataforge.meta +import hep.dataforge.misc.DFExperimental import hep.dataforge.names.* public interface MutableMeta> : TypedMeta, MutableItemProvider { diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/annotations.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/annotations.kt deleted file mode 100644 index 69ffa8aa..00000000 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/annotations.kt +++ /dev/null @@ -1,11 +0,0 @@ -package hep.dataforge.meta - -/** - * General marker for dataforge builders - */ -@DslMarker -public annotation class DFBuilder - -@RequiresOptIn(level = RequiresOptIn.Level.WARNING) -@Retention(AnnotationRetention.BINARY) -public annotation class DFExperimental \ No newline at end of file diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/descriptors/ItemDescriptor.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/descriptors/ItemDescriptor.kt index e03c832a..33343fe1 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/descriptors/ItemDescriptor.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/descriptors/ItemDescriptor.kt @@ -1,6 +1,7 @@ package hep.dataforge.meta.descriptors import hep.dataforge.meta.* +import hep.dataforge.misc.DFBuilder import hep.dataforge.names.* import hep.dataforge.values.* diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/mapMeta.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/mapMeta.kt index d223ce99..371a67a0 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/mapMeta.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/mapMeta.kt @@ -1,6 +1,7 @@ package hep.dataforge.meta import hep.dataforge.meta.descriptors.NodeDescriptor +import hep.dataforge.misc.DFExperimental import hep.dataforge.names.toName import hep.dataforge.values.ListValue import hep.dataforge.values.Value diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/transformations/MetaTransformation.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/transformations/MetaTransformation.kt index 710998ef..0e41b016 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/transformations/MetaTransformation.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/transformations/MetaTransformation.kt @@ -1,6 +1,7 @@ package hep.dataforge.meta.transformations import hep.dataforge.meta.* +import hep.dataforge.misc.DFExperimental import hep.dataforge.names.Name /** diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/misc/annotations.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/misc/annotations.kt new file mode 100644 index 00000000..7b271a4b --- /dev/null +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/misc/annotations.kt @@ -0,0 +1,21 @@ +package hep.dataforge.misc + +/** + * General marker for dataforge builders + */ +@DslMarker +public annotation class DFBuilder + +/** + * The declaration is experimental and could be changed in future + */ +@RequiresOptIn(level = RequiresOptIn.Level.WARNING) +@Retention(AnnotationRetention.BINARY) +public annotation class DFExperimental + +/** + * The declaration is internal to the DataForge and could use unsafe or unstable features. + */ +@RequiresOptIn(level = RequiresOptIn.Level.WARNING) +@Retention(AnnotationRetention.BINARY) +public annotation class DFInternal \ No newline at end of file diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/names/Name.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/names/Name.kt index 5d9cc7bb..3bb24ab1 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/names/Name.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/names/Name.kt @@ -1,6 +1,6 @@ package hep.dataforge.names -import hep.dataforge.meta.DFExperimental +import hep.dataforge.misc.DFExperimental import kotlinx.serialization.KSerializer import kotlinx.serialization.Serializable import kotlinx.serialization.descriptors.PrimitiveKind @@ -186,7 +186,6 @@ public fun Name.withIndex(index: String): Name { * Fast [String]-based accessor for item map */ public operator fun Map.get(body: String, query: String? = null): T? = get(NameToken(body, query)) - public operator fun Map.get(name: String): T? = get(name.toName()) public operator fun MutableMap.set(name: String, value: T): Unit = set(name.toName(), value) diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/names/nameMatcher.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/names/nameMatcher.kt index 363109f1..5b786917 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/names/nameMatcher.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/names/nameMatcher.kt @@ -1,6 +1,6 @@ package hep.dataforge.names -import hep.dataforge.meta.DFExperimental +import hep.dataforge.misc.DFExperimental /** diff --git a/dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/MetaTest.kt b/dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/MetaTest.kt index 12569bc3..89786ea1 100644 --- a/dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/MetaTest.kt +++ b/dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/MetaTest.kt @@ -1,5 +1,6 @@ package hep.dataforge.meta +import hep.dataforge.misc.DFExperimental import hep.dataforge.values.NumberValue import hep.dataforge.values.True import hep.dataforge.values.Value diff --git a/dataforge-meta/src/commonTest/kotlin/hep/dataforge/names/NameMatchTest.kt b/dataforge-meta/src/commonTest/kotlin/hep/dataforge/names/NameMatchTest.kt index b91d94a6..1ff4381f 100644 --- a/dataforge-meta/src/commonTest/kotlin/hep/dataforge/names/NameMatchTest.kt +++ b/dataforge-meta/src/commonTest/kotlin/hep/dataforge/names/NameMatchTest.kt @@ -1,6 +1,6 @@ package hep.dataforge.names -import hep.dataforge.meta.DFExperimental +import hep.dataforge.misc.DFExperimental import kotlin.test.Test import kotlin.test.assertFails import kotlin.test.assertFalse diff --git a/dataforge-tables/src/commonMain/kotlin/hep/dataforge/tables/io/textTableEnvelope.kt b/dataforge-tables/src/commonMain/kotlin/hep/dataforge/tables/io/textTableEnvelope.kt index 4deb84e6..b6be63e0 100644 --- a/dataforge-tables/src/commonMain/kotlin/hep/dataforge/tables/io/textTableEnvelope.kt +++ b/dataforge-tables/src/commonMain/kotlin/hep/dataforge/tables/io/textTableEnvelope.kt @@ -2,6 +2,7 @@ package hep.dataforge.tables.io import hep.dataforge.io.Envelope import hep.dataforge.meta.* +import hep.dataforge.misc.DFExperimental import hep.dataforge.tables.SimpleColumnHeader import hep.dataforge.tables.Table import hep.dataforge.values.Value diff --git a/dataforge-tables/src/jvmTest/kotlin/hep/dataforge/tables/io/TextRowsTest.kt b/dataforge-tables/src/jvmTest/kotlin/hep/dataforge/tables/io/TextRowsTest.kt index d97484ed..90fa3a01 100644 --- a/dataforge-tables/src/jvmTest/kotlin/hep/dataforge/tables/io/TextRowsTest.kt +++ b/dataforge-tables/src/jvmTest/kotlin/hep/dataforge/tables/io/TextRowsTest.kt @@ -1,6 +1,6 @@ package hep.dataforge.tables.io -import hep.dataforge.meta.DFExperimental +import hep.dataforge.misc.DFExperimental import hep.dataforge.tables.Table import hep.dataforge.tables.get import hep.dataforge.tables.row diff --git a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/Task.kt b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/Task.kt index df3c16af..6e6c590e 100644 --- a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/Task.kt +++ b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/Task.kt @@ -6,11 +6,13 @@ import hep.dataforge.data.GoalExecutionRestriction import hep.dataforge.meta.Meta import hep.dataforge.meta.descriptors.Described import hep.dataforge.meta.descriptors.ItemDescriptor +import hep.dataforge.misc.DFInternal import hep.dataforge.misc.Type import hep.dataforge.names.Name import hep.dataforge.workspace.Task.Companion.TYPE import kotlinx.coroutines.withContext -import kotlin.reflect.KClass +import kotlin.reflect.KType +import kotlin.reflect.typeOf @Type(TYPE) public interface Task : Described { @@ -42,8 +44,9 @@ public class TaskResultBuilder( * Data dependency cycles are not allowed. */ @Suppress("FunctionName") +@DFInternal public fun Task( - resultType: KClass, + resultType: KType, descriptor: ItemDescriptor? = null, builder: suspend TaskResultBuilder.() -> Unit, ): Task = object : Task { @@ -56,15 +59,16 @@ public fun Task( taskMeta: Meta, ): TaskResult = withContext(GoalExecutionRestriction() + workspace.goalLogger) { //TODO use safe builder and check for external data on add and detects cycles - val dataset = DataTree(resultType) { + val dataset = DataTree(resultType) { TaskResultBuilder(workspace,taskName, taskMeta, this).apply { builder() } } workspace.internalize(dataset, taskName, taskMeta) } } +@OptIn(DFInternal::class) @Suppress("FunctionName") public inline fun Task( descriptor: ItemDescriptor? = null, noinline builder: suspend TaskResultBuilder.() -> Unit, -): Task = Task(T::class, descriptor, builder) \ No newline at end of file +): Task = Task(typeOf(), descriptor, builder) \ No newline at end of file diff --git a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskData.kt b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskData.kt index 37862a48..f317b00a 100644 --- a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskData.kt +++ b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskData.kt @@ -1,7 +1,7 @@ package hep.dataforge.workspace -import hep.dataforge.actions.NamedData import hep.dataforge.data.Data +import hep.dataforge.data.NamedData import hep.dataforge.meta.Meta import hep.dataforge.names.Name diff --git a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/WorkspaceBuilder.kt b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/WorkspaceBuilder.kt index aa3519a3..cd3e6d92 100644 --- a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/WorkspaceBuilder.kt +++ b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/WorkspaceBuilder.kt @@ -7,16 +7,15 @@ import hep.dataforge.data.ActiveDataTree import hep.dataforge.data.DataSet import hep.dataforge.data.DataSetBuilder import hep.dataforge.data.DataTree -import hep.dataforge.meta.DFBuilder -import hep.dataforge.meta.DFExperimental import hep.dataforge.meta.Meta import hep.dataforge.meta.MetaBuilder import hep.dataforge.meta.descriptors.NodeDescriptor +import hep.dataforge.misc.DFBuilder +import hep.dataforge.misc.DFExperimental import hep.dataforge.names.Name import hep.dataforge.names.toName import kotlin.properties.PropertyDelegateProvider import kotlin.properties.ReadOnlyProperty -import kotlin.reflect.KClass public data class TaskReference(public val taskName: Name, public val task: Task) @@ -24,25 +23,19 @@ public interface TaskContainer { public fun registerTask(taskName: Name, task: Task<*>) } -public fun TaskContainer.registerTask( - resultType: KClass, - name: String, - descriptorBuilder: NodeDescriptor.() -> Unit = {}, - builder: suspend TaskResultBuilder.() -> Unit, -): Unit = registerTask(name.toName(), Task(resultType, NodeDescriptor(descriptorBuilder), builder)) public inline fun TaskContainer.registerTask( name: String, noinline descriptorBuilder: NodeDescriptor.() -> Unit = {}, noinline builder: suspend TaskResultBuilder.() -> Unit, -): Unit = registerTask(T::class, name, descriptorBuilder, builder) +): Unit = registerTask(name.toName(), Task(NodeDescriptor(descriptorBuilder), builder)) public inline fun TaskContainer.task( noinline descriptorBuilder: NodeDescriptor.() -> Unit = {}, noinline builder: suspend TaskResultBuilder.() -> Unit, ): PropertyDelegateProvider>> = PropertyDelegateProvider { _, property -> val taskName = property.name.toName() - val task = Task(T::class, NodeDescriptor(descriptorBuilder), builder) + val task = Task(NodeDescriptor(descriptorBuilder), builder) registerTask(taskName, task) ReadOnlyProperty { _, _ -> TaskReference(taskName, task) } } diff --git a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/envelopeData.kt b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/envelopeData.kt index c659d2c5..cebba975 100644 --- a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/envelopeData.kt +++ b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/envelopeData.kt @@ -3,15 +3,14 @@ package hep.dataforge.workspace import hep.dataforge.data.Data import hep.dataforge.data.await import hep.dataforge.io.* -import kotlin.reflect.KClass +import hep.dataforge.misc.DFInternal /** * Convert an [Envelope] to a data via given format. The actual parsing is done lazily. */ +@OptIn(DFInternal::class) public fun Envelope.toData(format: IOFormat): Data { - @Suppress("UNCHECKED_CAST") - val kclass: KClass = format.type.classifier as? KClass ?: error("IOFormat type is not a class") - return Data(kclass, meta) { + return Data(format.type, meta) { data?.readWith(format) ?: error("Can't convert envelope without data to Data") } } diff --git a/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/fileData.kt b/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/fileData.kt index 29ff0c62..a8378046 100644 --- a/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/fileData.kt +++ b/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/fileData.kt @@ -4,6 +4,7 @@ package hep.dataforge.workspace import hep.dataforge.data.* import hep.dataforge.io.* import hep.dataforge.meta.* +import hep.dataforge.misc.DFExperimental import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.runBlocking import kotlinx.coroutines.withContext @@ -15,7 +16,6 @@ import java.nio.file.StandardOpenOption import java.nio.file.spi.FileSystemProvider import java.util.zip.ZipEntry import java.util.zip.ZipOutputStream -import kotlin.reflect.KClass import kotlin.reflect.KType import kotlin.reflect.typeOf import kotlin.streams.toList @@ -36,9 +36,6 @@ internal inline fun IOPlugin.formatResolver(): FileFormatResol resolveIOFormat() ?: error("Can't resolve IO format for ${T::class}") } -private val FileFormatResolver.kClass: KClass - get() = type.classifier as? KClass ?: error("Format resolver actual type does not correspond to type parameter") - private fun newZFS(path: Path): FileSystem { val fsProvider = FileSystemProvider.installedProviders().find { it.scheme == "jar" } ?: error("Zip file system provider not found") @@ -110,7 +107,7 @@ public suspend fun IOPlugin.readDataDirectory( return readDataDirectory(fs.rootDirectories.first(), formatResolver) } if (!Files.isDirectory(path)) error("Provided path $path is not a directory") - return DataTree(formatResolver.kClass) { + return DataTree(formatResolver.type) { Files.list(path).toList().forEach { path -> val fileName = path.fileName.toString() if (fileName.startsWith(IOPlugin.META_FILE_NAME)) { diff --git a/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/FileDataTest.kt b/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/FileDataTest.kt index e089e448..df8e2673 100644 --- a/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/FileDataTest.kt +++ b/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/FileDataTest.kt @@ -4,8 +4,8 @@ import hep.dataforge.context.Global import hep.dataforge.data.* import hep.dataforge.io.IOFormat import hep.dataforge.io.io -import hep.dataforge.meta.DFExperimental import hep.dataforge.meta.Meta +import hep.dataforge.misc.DFExperimental import kotlinx.coroutines.runBlocking import kotlinx.io.Input import kotlinx.io.Output diff --git a/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/SimpleWorkspaceTest.kt b/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/SimpleWorkspaceTest.kt index b6bc9a1c..5ee08ea5 100644 --- a/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/SimpleWorkspaceTest.kt +++ b/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/SimpleWorkspaceTest.kt @@ -1,9 +1,9 @@ package hep.dataforge.workspace -import hep.dataforge.actions.get import hep.dataforge.context.* import hep.dataforge.data.* import hep.dataforge.meta.* +import hep.dataforge.names.get import hep.dataforge.names.plus import kotlinx.coroutines.flow.first import kotlinx.coroutines.flow.single From 7d3df24568688a21d3de3894e6117415b9e24167 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Sun, 7 Feb 2021 19:36:41 +0300 Subject: [PATCH 6/7] Remove KClass from dataforge-data --- .../kotlin/hep/dataforge/data/select.kt | 40 +++---------------- 1 file changed, 6 insertions(+), 34 deletions(-) diff --git a/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/select.kt b/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/select.kt index 9df89f84..02672c37 100644 --- a/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/select.kt +++ b/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/select.kt @@ -12,44 +12,15 @@ import kotlin.reflect.full.isSubtypeOf import kotlin.reflect.typeOf -/** - * Check if data could be safely cast to given class - */ -private fun Data<*>.canCast(type: KType): Boolean = this.type.isSubtypeOf(type) - /** * Cast the node to given type if the cast is possible or return null */ @Suppress("UNCHECKED_CAST") private fun Data<*>.castOrNull(type: KType): Data? = - if (!canCast(type)) null else object : Data by (this as Data) { + if (!this.type.isSubtypeOf(type)) null else object : Data by (this as Data) { override val type: KType = type } -/** - * Unsafe cast of data node - */ -private fun Data<*>.cast(type: KType): Data = - castOrNull(type) ?: error("Can't cast ${this.type} to $type") - -private inline fun Data<*>.cast(): Data = cast(typeOf()) - -@Suppress("UNCHECKED_CAST") -private fun DataSet<*>.castOrNull(type: KType): DataSet? = - if (!canCast(type)) null else object : DataSet by (this as DataSet) { - override val dataType: KType = type - } - - -private fun DataSet<*>.cast(type: KType): DataSet = - castOrNull(type) ?: error("Can't cast ${this.dataType} to $type") - -/** - * Check that node is compatible with given type meaning that each element could be cast to the type - */ -private fun DataSet<*>.canCast(type: KType): Boolean = - type.isSubtypeOf(this.dataType) - /** * Select all data matching given type and filters. Does not modify paths */ @@ -61,10 +32,11 @@ internal fun DataSet<*>.select( ): ActiveDataSet = object : ActiveDataSet { override val dataType = type - @Suppress("UNCHECKED_CAST") - override fun flow(): Flow> = this@select.flow().filter { - it.type.isSubtypeOf(type) && (namePattern == null || it.name.matches(namePattern)) + + override fun flow(): Flow> = this@select.flow().filter { datum -> + datum.type.isSubtypeOf(type) && (namePattern == null || datum.name.matches(namePattern)) }.map { + @Suppress("UNCHECKED_CAST") it as NamedData } @@ -72,7 +44,7 @@ internal fun DataSet<*>.select( override val updates: Flow = this@select.updates.filter { val datum = this@select.getData(it) - datum?.canCast(type) ?: false + datum?.type?.isSubtypeOf(type) ?: false } } From 5d02520904ed5fbc831440a583a9367925231a25 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Sun, 7 Feb 2021 20:23:33 +0300 Subject: [PATCH 7/7] turn on API validation --- CHANGELOG.md | 1 + build.gradle.kts | 8 -------- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0f9f236d..0faac561 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ - \[Major breaking change\] Moved `NodeItem` and `ValueItem` to a top level - Plugins are removed from Context constructor and added lazily in ContextBuilder - \[Major breaking change\] Full refactor of DataTree/DataSource +- \[Major Breaking change\] Replace KClass with KType in data. Remove direct access to constructors with types. ### Deprecated diff --git a/build.gradle.kts b/build.gradle.kts index cc74ad20..27917005 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -15,14 +15,6 @@ allprojects { apply() } -apiValidation{ - validationDisabled = true -} - subprojects { apply(plugin = "ru.mipt.npm.publish") -} - -apiValidation{ - ignoredProjects.add("dataforge-tables") } \ No newline at end of file