From 1970243785e67cb583e7be5b513f3e42089f145e Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Sun, 7 Feb 2021 18:15:54 +0300 Subject: [PATCH] Remove KClass from dataforge-data --- .../hep/dataforge/context/ContextBuilder.kt | 11 ++++- .../kotlin/hep/dataforge/context/resolve.kt | 2 +- .../dataforge/properties/ConfigProperty.kt | 2 +- .../hep/dataforge/properties/Property.kt | 2 +- .../hep/dataforge/properties/bindings.kt | 2 +- .../hep/dataforge/descriptors/annotations.kt | 1 - .../kotlin/hep/dataforge/provider/dfType.kt | 2 +- .../kotlin/hep/dataforge/actions/Action.kt | 3 +- .../kotlin/hep/dataforge/actions/MapAction.kt | 17 +++++-- .../hep/dataforge/actions/ReduceAction.kt | 34 ++++++++------ .../hep/dataforge/actions/SplitAction.kt | 22 ++++++++- .../hep/dataforge/data/CachingAction.kt | 1 - .../hep/dataforge/data/CoroutineMonitor.kt | 2 +- .../kotlin/hep/dataforge/data/Data.kt | 23 +++++++--- .../kotlin/hep/dataforge/data/DataSet.kt | 5 +- .../hep/dataforge/data/DataSetBuilder.kt | 26 ++++++----- .../kotlin/hep/dataforge/data/DataTree.kt | 15 +++--- .../kotlin/hep/dataforge/data/Goal.kt | 2 +- .../kotlin/hep/dataforge/data/GroupRule.kt | 10 ++-- .../dataforge/{actions => data}/NamedData.kt | 4 +- .../kotlin/hep/dataforge/data/dataFilter.kt | 12 ++--- .../hep/dataforge/data/dataTransform.kt | 46 ++++++++----------- .../kotlin/hep/dataforge/data/select.kt | 4 +- .../kotlin/hep/dataforge/data/ActionsTest.kt | 17 +++---- .../hep/dataforge/data/DataTreeBuilderTest.kt | 19 ++++---- .../io/yaml/FrontMatterEnvelopeFormat.kt | 2 +- .../hep/dataforge/io/yaml/YamlMetaFormat.kt | 1 + .../dataforge/io/yaml/YamlMetaFormatTest.kt | 2 +- .../kotlin/hep/dataforge/io/Consumer.kt | 2 +- .../kotlin/hep/dataforge/io/MultipartTest.kt | 2 +- .../jvmMain/kotlin/hep/dataforge/io/fileIO.kt | 2 +- .../kotlin/hep/dataforge/io/FileBinaryTest.kt | 2 +- .../hep/dataforge/io/FileEnvelopeTest.kt | 2 +- .../kotlin/hep/dataforge/meta/Configurable.kt | 1 + .../kotlin/hep/dataforge/meta/MetaBuilder.kt | 1 + .../kotlin/hep/dataforge/meta/MutableMeta.kt | 1 + .../kotlin/hep/dataforge/meta/annotations.kt | 11 ----- .../meta/descriptors/ItemDescriptor.kt | 1 + .../kotlin/hep/dataforge/meta/mapMeta.kt | 1 + .../transformations/MetaTransformation.kt | 1 + .../kotlin/hep/dataforge/misc/annotations.kt | 21 +++++++++ .../kotlin/hep/dataforge/names/Name.kt | 3 +- .../kotlin/hep/dataforge/names/nameMatcher.kt | 2 +- .../kotlin/hep/dataforge/meta/MetaTest.kt | 1 + .../hep/dataforge/names/NameMatchTest.kt | 2 +- .../dataforge/tables/io/textTableEnvelope.kt | 1 + .../hep/dataforge/tables/io/TextRowsTest.kt | 2 +- .../kotlin/hep/dataforge/workspace/Task.kt | 12 +++-- .../hep/dataforge/workspace/TaskData.kt | 2 +- .../dataforge/workspace/WorkspaceBuilder.kt | 15 ++---- .../hep/dataforge/workspace/envelopeData.kt | 7 ++- .../hep/dataforge/workspace/fileData.kt | 7 +-- .../hep/dataforge/workspace/FileDataTest.kt | 2 +- .../workspace/SimpleWorkspaceTest.kt | 2 +- 54 files changed, 223 insertions(+), 172 deletions(-) rename dataforge-data/src/commonMain/kotlin/hep/dataforge/{actions => data}/NamedData.kt (89%) delete mode 100644 dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/annotations.kt create mode 100644 dataforge-meta/src/commonMain/kotlin/hep/dataforge/misc/annotations.kt diff --git a/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/ContextBuilder.kt b/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/ContextBuilder.kt index 69a68026..f124d3d2 100644 --- a/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/ContextBuilder.kt +++ b/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/ContextBuilder.kt @@ -1,7 +1,16 @@ package hep.dataforge.context -import hep.dataforge.meta.* +import hep.dataforge.meta.Meta +import hep.dataforge.meta.MetaBuilder +import hep.dataforge.meta.seal +import hep.dataforge.misc.DFBuilder +import hep.dataforge.misc.DFExperimental import hep.dataforge.names.toName +import kotlin.collections.HashMap +import kotlin.collections.component1 +import kotlin.collections.component2 +import kotlin.collections.forEach +import kotlin.collections.set /** * A convenience builder for context diff --git a/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/resolve.kt b/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/resolve.kt index bfc4e22d..b41e11fd 100644 --- a/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/resolve.kt +++ b/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/resolve.kt @@ -1,6 +1,6 @@ package hep.dataforge.context -import hep.dataforge.meta.DFExperimental +import hep.dataforge.misc.DFExperimental import hep.dataforge.names.Name import hep.dataforge.names.plus import hep.dataforge.provider.Provider diff --git a/dataforge-context/src/commonMain/kotlin/hep/dataforge/properties/ConfigProperty.kt b/dataforge-context/src/commonMain/kotlin/hep/dataforge/properties/ConfigProperty.kt index 98a43b69..86c6bcde 100644 --- a/dataforge-context/src/commonMain/kotlin/hep/dataforge/properties/ConfigProperty.kt +++ b/dataforge-context/src/commonMain/kotlin/hep/dataforge/properties/ConfigProperty.kt @@ -1,12 +1,12 @@ package hep.dataforge.properties import hep.dataforge.meta.Config -import hep.dataforge.meta.DFExperimental import hep.dataforge.meta.get import hep.dataforge.meta.set import hep.dataforge.meta.transformations.MetaConverter import hep.dataforge.meta.transformations.nullableItemToObject import hep.dataforge.meta.transformations.nullableObjectToMetaItem +import hep.dataforge.misc.DFExperimental import hep.dataforge.names.Name @DFExperimental diff --git a/dataforge-context/src/commonMain/kotlin/hep/dataforge/properties/Property.kt b/dataforge-context/src/commonMain/kotlin/hep/dataforge/properties/Property.kt index 987cfe4c..44e97ea9 100644 --- a/dataforge-context/src/commonMain/kotlin/hep/dataforge/properties/Property.kt +++ b/dataforge-context/src/commonMain/kotlin/hep/dataforge/properties/Property.kt @@ -1,6 +1,6 @@ package hep.dataforge.properties -import hep.dataforge.meta.DFExperimental +import hep.dataforge.misc.DFExperimental import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.ExperimentalCoroutinesApi import kotlinx.coroutines.flow.MutableStateFlow diff --git a/dataforge-context/src/jsMain/kotlin/hep/dataforge/properties/bindings.kt b/dataforge-context/src/jsMain/kotlin/hep/dataforge/properties/bindings.kt index 7ef8d72d..86296e39 100644 --- a/dataforge-context/src/jsMain/kotlin/hep/dataforge/properties/bindings.kt +++ b/dataforge-context/src/jsMain/kotlin/hep/dataforge/properties/bindings.kt @@ -1,6 +1,6 @@ package hep.dataforge.properties -import hep.dataforge.meta.DFExperimental +import hep.dataforge.misc.DFExperimental import org.w3c.dom.HTMLInputElement @DFExperimental diff --git a/dataforge-context/src/jvmMain/kotlin/hep/dataforge/descriptors/annotations.kt b/dataforge-context/src/jvmMain/kotlin/hep/dataforge/descriptors/annotations.kt index cadd4231..f88cee99 100644 --- a/dataforge-context/src/jvmMain/kotlin/hep/dataforge/descriptors/annotations.kt +++ b/dataforge-context/src/jvmMain/kotlin/hep/dataforge/descriptors/annotations.kt @@ -16,7 +16,6 @@ package hep.dataforge.descriptors -import hep.dataforge.meta.DFExperimental import hep.dataforge.values.ValueType import kotlin.reflect.KClass diff --git a/dataforge-context/src/jvmMain/kotlin/hep/dataforge/provider/dfType.kt b/dataforge-context/src/jvmMain/kotlin/hep/dataforge/provider/dfType.kt index 65817b64..6f7855a9 100644 --- a/dataforge-context/src/jvmMain/kotlin/hep/dataforge/provider/dfType.kt +++ b/dataforge-context/src/jvmMain/kotlin/hep/dataforge/provider/dfType.kt @@ -2,7 +2,7 @@ package hep.dataforge.provider import hep.dataforge.context.Context import hep.dataforge.context.gather -import hep.dataforge.meta.DFExperimental +import hep.dataforge.misc.DFExperimental import hep.dataforge.misc.Type import hep.dataforge.names.Name import kotlin.reflect.KClass diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/actions/Action.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/actions/Action.kt index a7c51e8f..85ec2977 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/actions/Action.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/actions/Action.kt @@ -1,10 +1,9 @@ package hep.dataforge.actions import hep.dataforge.data.DataSet -import hep.dataforge.meta.DFExperimental import hep.dataforge.meta.Meta +import hep.dataforge.misc.DFExperimental import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.flow.* /** * A simple data transformation on a data node. Actions should avoid doing actual dependency evaluation in [execute]. diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/actions/MapAction.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/actions/MapAction.kt index 77acc6a4..23731621 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/actions/MapAction.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/actions/MapAction.kt @@ -1,7 +1,13 @@ package hep.dataforge.actions import hep.dataforge.data.* -import hep.dataforge.meta.* +import hep.dataforge.meta.Meta +import hep.dataforge.meta.MetaBuilder +import hep.dataforge.meta.seal +import hep.dataforge.meta.toMutableMeta +import hep.dataforge.misc.DFBuilder +import hep.dataforge.misc.DFExperimental +import hep.dataforge.misc.DFInternal import hep.dataforge.names.Name import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.flow.collect @@ -34,7 +40,6 @@ public class MapActionBuilder(public var name: Name, public var meta: Meta } } - @PublishedApi internal class MapAction( private val outputType: KType, @@ -63,7 +68,9 @@ internal class MapAction( //getting new meta val newMeta = builder.meta.seal() - val newData = data.map(outputType, meta = newMeta) { builder.result(env, it) } + @OptIn(DFInternal::class) val newData = Data(outputType, newMeta, dependencies = listOf(data)) { + builder.result(env, data.await()) + } //setting the data node return newData.named(newName) } @@ -85,6 +92,10 @@ internal class MapAction( } +/** + * A one-to-one mapping action + */ +@DFExperimental @Suppress("FunctionName") public inline fun Action.Companion.map( noinline builder: MapActionBuilder.() -> Unit, diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/actions/ReduceAction.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/actions/ReduceAction.kt index f3cf9fdc..8e2781b5 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/actions/ReduceAction.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/actions/ReduceAction.kt @@ -1,20 +1,21 @@ package hep.dataforge.actions import hep.dataforge.data.* -import hep.dataforge.meta.DFExperimental import hep.dataforge.meta.Meta import hep.dataforge.meta.MetaBuilder +import hep.dataforge.misc.DFBuilder +import hep.dataforge.misc.DFExperimental +import hep.dataforge.misc.DFInternal import hep.dataforge.names.Name import hep.dataforge.names.toName import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.flow import kotlinx.coroutines.flow.fold -import kotlin.reflect.KClass import kotlin.reflect.KType +import kotlin.reflect.typeOf -@DFExperimental public class JoinGroup(public var name: String, internal val set: DataSet) { public var meta: MetaBuilder = MetaBuilder() @@ -27,9 +28,9 @@ public class JoinGroup(public var name: String, internal val s } -@DFExperimental +@DFBuilder public class ReduceGroupBuilder( - private val inputType: KClass, + private val inputType: KType, private val scope: CoroutineScope, public val actionMeta: Meta, ) { @@ -40,7 +41,7 @@ public class ReduceGroupBuilder( */ public fun byValue(tag: String, defaultTag: String = "@default", action: JoinGroup.() -> Unit) { groupRules += { node -> - GroupRule.byMetaValue(scope, tag, defaultTag).gather(inputType, node).map { + GroupRule.byMetaValue(scope, tag, defaultTag).gather(node).map { JoinGroup(it.key, it.value).apply(action) } } @@ -73,16 +74,17 @@ public class ReduceGroupBuilder( } -@DFExperimental -public class ReduceAction( - private val inputType: KClass, +@PublishedApi +internal class ReduceAction( + private val inputType: KType, outputType: KType, private val action: ReduceGroupBuilder.() -> Unit, ) : CachingAction(outputType) { //TODO optimize reduction. Currently the whole action recalculates on push + override fun CoroutineScope.transform(set: DataSet, meta: Meta, key: Name): Flow> = flow { - ReduceGroupBuilder(inputType,this@transform, meta).apply(action).buildGroups(set).forEach { group -> + ReduceGroupBuilder(inputType, this@transform, meta).apply(action).buildGroups(set).forEach { group -> val dataFlow: Map> = group.set.flow().fold(HashMap()) { acc, value -> acc.apply { acc[value.name] = value.data @@ -94,8 +96,7 @@ public class ReduceAction( val groupMeta = group.meta val env = ActionEnv(groupName.toName(), groupMeta, meta) - - val res: LazyData = dataFlow.reduceToData( + @OptIn(DFInternal::class) val res: Data = dataFlow.reduceToData( outputType, meta = groupMeta ) { group.result.invoke(env, it) } @@ -105,4 +106,11 @@ public class ReduceAction( } } -public operator fun Map.get(name: String): T? = get(name.toName()) +/** + * A one-to-one mapping action + */ +@DFExperimental +@Suppress("FunctionName") +public inline fun Action.Companion.reduce( + noinline builder: ReduceGroupBuilder.() -> Unit, +): Action = ReduceAction(typeOf(), typeOf(), builder) diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/actions/SplitAction.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/actions/SplitAction.kt index c659d5a1..37b8f734 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/actions/SplitAction.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/actions/SplitAction.kt @@ -5,13 +5,17 @@ import hep.dataforge.meta.Laminate import hep.dataforge.meta.Meta import hep.dataforge.meta.MetaBuilder import hep.dataforge.meta.toMutableMeta +import hep.dataforge.misc.DFExperimental +import hep.dataforge.misc.DFInternal import hep.dataforge.names.Name import hep.dataforge.names.toName import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.FlowPreview import kotlinx.coroutines.flow.* import kotlinx.coroutines.launch import kotlin.collections.set import kotlin.reflect.KType +import kotlin.reflect.typeOf public class SplitBuilder(public val name: Name, public val meta: Meta) { @@ -39,11 +43,13 @@ public class SplitBuilder(public val name: Name, public val me /** * Action that splits each incoming element into a number of fragments defined in builder */ +@PublishedApi internal class SplitAction( private val outputType: KType, private val action: SplitBuilder.() -> Unit, ) : Action { + @OptIn(FlowPreview::class) override suspend fun execute( dataSet: DataSet, meta: Meta, @@ -59,7 +65,10 @@ internal class SplitAction( // apply individual fragment rules to result return split.fragments.entries.asFlow().map { (fragmentName, rule) -> val env = SplitBuilder.FragmentRule(fragmentName, laminate.toMutableMeta()).apply(rule) - data.map(outputType, meta = env.meta) { env.result(it) }.named(fragmentName) + //data.map(outputType, meta = env.meta) { env.result(it) }.named(fragmentName) + @OptIn(DFInternal::class) Data(outputType, meta = env.meta, dependencies = listOf(data)) { + env.result(data.await()) + }.named(fragmentName) } } @@ -75,4 +84,13 @@ internal class SplitAction( } } } -} \ No newline at end of file +} + +/** + * Action that splits each incoming element into a number of fragments defined in builder + */ +@DFExperimental +@Suppress("FunctionName") +public inline fun Action.Companion.split( + noinline builder: SplitBuilder.() -> Unit, +): Action = SplitAction(typeOf(), builder) \ No newline at end of file diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/CachingAction.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/CachingAction.kt index c4fb4f90..7911cb1f 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/CachingAction.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/CachingAction.kt @@ -1,7 +1,6 @@ package hep.dataforge.data import hep.dataforge.actions.Action -import hep.dataforge.actions.NamedData import hep.dataforge.meta.Meta import hep.dataforge.names.Name import hep.dataforge.names.startsWith diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/CoroutineMonitor.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/CoroutineMonitor.kt index d1c0d55e..60bf5775 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/CoroutineMonitor.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/CoroutineMonitor.kt @@ -1,6 +1,6 @@ package hep.dataforge.data -import hep.dataforge.meta.DFExperimental +import hep.dataforge.misc.DFExperimental import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Job import kotlin.coroutines.CoroutineContext diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/Data.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/Data.kt index 0a768ee4..6702de4d 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/Data.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/Data.kt @@ -3,6 +3,7 @@ package hep.dataforge.data import hep.dataforge.meta.Meta import hep.dataforge.meta.MetaRepr import hep.dataforge.meta.isEmpty +import hep.dataforge.misc.DFInternal import hep.dataforge.misc.Type import kotlinx.coroutines.* import kotlin.coroutines.CoroutineContext @@ -35,17 +36,21 @@ public interface Data : Goal, MetaRepr { public companion object { public const val TYPE: String = "data" + /** + * The type that can't have any subtypes + */ + internal val TYPE_OF_NOTHING: KType = typeOf() + public inline fun static( value: T, meta: Meta = Meta.EMPTY, - ): Data = StaticData(typeOf(),value, meta) + ): Data = StaticData(typeOf(), value, meta) /** * An empty data containing only meta */ public fun empty(meta: Meta): Data = object : Data { - private val nothing: Nothing get() = error("this is nothing") - override val type: KType = this::nothing.returnType + override val type: KType = TYPE_OF_NOTHING override val meta: Meta = meta override val dependencies: Collection> = emptyList() override val deferred: Deferred @@ -59,13 +64,17 @@ public interface Data : Goal, MetaRepr { } } -public class LazyData( +/** + * A lazily computed variant of [Data] based on [LazyGoal] + * One must ensure that proper [type] is used so this method should not be used + */ +private class LazyData( override val type: KType, override val meta: Meta = Meta.EMPTY, - context: CoroutineContext = EmptyCoroutineContext, + additionalContext: CoroutineContext = EmptyCoroutineContext, dependencies: Collection> = emptyList(), block: suspend () -> T, -) : Data, LazyGoal(context, dependencies, block) +) : Data, LazyGoal(additionalContext, dependencies, block) public class StaticData( override val type: KType, @@ -74,6 +83,7 @@ public class StaticData( ) : Data, StaticGoal(value) @Suppress("FunctionName") +@DFInternal public fun Data( type: KType, meta: Meta = Meta.EMPTY, @@ -82,6 +92,7 @@ public fun Data( block: suspend () -> T, ): Data = LazyData(type, meta, context, dependencies, block) +@OptIn(DFInternal::class) @Suppress("FunctionName") public inline fun Data( meta: Meta = Meta.EMPTY, diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataSet.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataSet.kt index 129f657f..f2a4bf88 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataSet.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataSet.kt @@ -1,7 +1,6 @@ package hep.dataforge.data -import hep.dataforge.actions.NamedData -import hep.dataforge.actions.named +import hep.dataforge.data.Data.Companion.TYPE_OF_NOTHING import hep.dataforge.meta.Meta import hep.dataforge.meta.set import hep.dataforge.names.* @@ -43,7 +42,7 @@ public interface DataSet { * An empty [DataSet] that suits all types */ public val EMPTY: DataSet = object : DataSet { - override val dataType: KType = this::nothing.returnType + override val dataType: KType = TYPE_OF_NOTHING private val nothing: Nothing get() = error("this is nothing") diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataSetBuilder.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataSetBuilder.kt index f001c4ad..95e9fd06 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataSetBuilder.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataSetBuilder.kt @@ -1,17 +1,19 @@ package hep.dataforge.data -import hep.dataforge.actions.NamedData -import hep.dataforge.meta.DFExperimental import hep.dataforge.meta.Meta import hep.dataforge.meta.MetaBuilder +import hep.dataforge.misc.DFExperimental import hep.dataforge.names.Name import hep.dataforge.names.plus import hep.dataforge.names.toName import kotlinx.coroutines.coroutineScope import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.collect +import kotlin.reflect.KType public interface DataSetBuilder { + public val dataType: KType + /** * Remove all data items starting with [name] */ @@ -50,8 +52,12 @@ public interface DataSetBuilder { public suspend infix fun String.put(block: suspend DataSetBuilder.() -> Unit): Unit = emit(toName(), block) } -private class SubSetBuilder(private val parent: DataSetBuilder, private val branch: Name) : - DataSetBuilder { +private class SubSetBuilder( + private val parent: DataSetBuilder, + private val branch: Name, +) : DataSetBuilder { + override val dataType: KType get() = parent.dataType + override suspend fun remove(name: Name) { parent.remove(branch + name) } @@ -101,7 +107,7 @@ public suspend inline fun DataSetBuilder.produce( name: Name, meta: Meta = Meta.EMPTY, noinline producer: suspend () -> T, -){ +) { val data = Data(meta, block = producer) emit(name, data) } @@ -109,19 +115,17 @@ public suspend inline fun DataSetBuilder.produce( /** * Emit a static data with the fixed value */ -public suspend fun DataSetBuilder.static(name: String, data: T, meta: Meta = Meta.EMPTY): Unit = +public suspend inline fun DataSetBuilder.static(name: String, data: T, meta: Meta = Meta.EMPTY): Unit = emit(name, Data.static(data, meta)) -public suspend fun DataSetBuilder.static(name: Name, data: T, meta: Meta = Meta.EMPTY): Unit = +public suspend inline fun DataSetBuilder.static(name: Name, data: T, meta: Meta = Meta.EMPTY): Unit = emit(name, Data.static(data, meta)) -public suspend fun DataSetBuilder.static( +public suspend inline fun DataSetBuilder.static( name: String, data: T, metaBuilder: MetaBuilder.() -> Unit, -) { - emit(name.toName(), Data.static(data, Meta(metaBuilder))) -} +): Unit = emit(name.toName(), Data.static(data, Meta(metaBuilder))) /** * Update data with given node data and meta with node meta. diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataTree.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataTree.kt index 0354b161..5676eeed 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataTree.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataTree.kt @@ -1,22 +1,21 @@ package hep.dataforge.data -import hep.dataforge.actions.NamedData -import hep.dataforge.actions.named -import hep.dataforge.meta.* import hep.dataforge.misc.Type import hep.dataforge.names.* -import kotlinx.coroutines.* -import kotlinx.coroutines.flow.* +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.emitAll +import kotlinx.coroutines.flow.flow +import kotlinx.coroutines.flow.map import kotlin.collections.component1 import kotlin.collections.component2 -import kotlin.reflect.KClass +import kotlin.reflect.KType public sealed class DataTreeItem { public class Node(public val tree: DataTree) : DataTreeItem() public class Leaf(public val data: Data) : DataTreeItem() } -public val DataTreeItem.type: KClass +public val DataTreeItem.type: KType get() = when (this) { is DataTreeItem.Node -> tree.dataType is DataTreeItem.Leaf -> data.type @@ -91,7 +90,7 @@ public fun DataTree.itemFlow(): Flow>> = * The difference from similar method for [DataSet] is that internal logic is more simple and the return value is a [DataTree] */ public fun DataTree.branch(branchName: Name): DataTree = object : DataTree { - override val dataType: KClass get() = this@branch.dataType + override val dataType: KType get() = this@branch.dataType override suspend fun items(): Map> = getItem(branchName).tree?.items() ?: emptyMap() } diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/Goal.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/Goal.kt index eef9e7cb..7763c3ce 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/Goal.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/Goal.kt @@ -1,6 +1,6 @@ package hep.dataforge.data -import hep.dataforge.meta.DFExperimental +import hep.dataforge.misc.DFExperimental import kotlinx.coroutines.* import kotlin.coroutines.CoroutineContext import kotlin.coroutines.EmptyCoroutineContext diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/GroupRule.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/GroupRule.kt index 414fa17e..fcaa0658 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/GroupRule.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/GroupRule.kt @@ -20,10 +20,9 @@ import hep.dataforge.meta.string import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.flow.collect import kotlinx.coroutines.launch -import kotlin.reflect.KClass public interface GroupRule { - public suspend fun gather(dataType: KClass, set: DataSet): Map> + public suspend fun gather(set: DataSet): Map> public companion object { /** @@ -41,21 +40,20 @@ public interface GroupRule { ): GroupRule = object : GroupRule { override suspend fun gather( - dataType: KClass, set: DataSet, ): Map> { val map = HashMap>() set.flow().collect { data -> val tagValue = data.meta[key]?.string ?: defaultTagValue - map.getOrPut(tagValue) { ActiveDataTree(dataType) }.emit(data.name, data.data) + map.getOrPut(tagValue) { ActiveDataTree(set.dataType) }.emit(data.name, data.data) } scope.launch { set.updates.collect { name -> val data = set.getData(name) val tagValue = data?.meta[key]?.string ?: defaultTagValue - map.getOrPut(tagValue) { ActiveDataTree(dataType) }.emit(name, data) + map.getOrPut(tagValue) { ActiveDataTree(set.dataType) }.emit(name, data) } } @@ -63,4 +61,4 @@ public interface GroupRule { } } } -} +} \ No newline at end of file diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/actions/NamedData.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/NamedData.kt similarity index 89% rename from dataforge-data/src/commonMain/kotlin/hep/dataforge/actions/NamedData.kt rename to dataforge-data/src/commonMain/kotlin/hep/dataforge/data/NamedData.kt index 126c972b..aa5afcdc 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/actions/NamedData.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/NamedData.kt @@ -1,7 +1,5 @@ -package hep.dataforge.actions +package hep.dataforge.data -import hep.dataforge.data.Data -import hep.dataforge.data.StaticData import hep.dataforge.meta.isEmpty import hep.dataforge.misc.Named import hep.dataforge.names.Name diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/dataFilter.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/dataFilter.kt index aff59e61..8f38d100 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/dataFilter.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/dataFilter.kt @@ -1,14 +1,12 @@ package hep.dataforge.data -import hep.dataforge.actions.NamedData -import hep.dataforge.actions.named -import hep.dataforge.meta.DFExperimental +import hep.dataforge.misc.DFExperimental import hep.dataforge.names.* import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.filter import kotlinx.coroutines.flow.map import kotlinx.coroutines.flow.mapNotNull -import kotlin.reflect.KClass +import kotlin.reflect.KType /** @@ -17,7 +15,7 @@ import kotlin.reflect.KClass public fun DataSet.filter( predicate: suspend (Name, Data) -> Boolean, ): ActiveDataSet = object : ActiveDataSet { - override val dataType: KClass get() = this@filter.dataType + override val dataType: KType get() = this@filter.dataType override fun flow(): Flow> = this@filter.flow().filter { predicate(it.name, it.data) } @@ -37,7 +35,7 @@ public fun DataSet.filter( */ public fun DataSet.withNamePrefix(prefix: Name): DataSet = if (prefix.isEmpty()) this else object : ActiveDataSet { - override val dataType: KClass get() = this@withNamePrefix.dataType + override val dataType: KType get() = this@withNamePrefix.dataType override fun flow(): Flow> = this@withNamePrefix.flow().map { it.data.named(prefix + it.name) } @@ -53,7 +51,7 @@ else object : ActiveDataSet { public fun DataSet.branch(branchName: Name): DataSet = if (branchName.isEmpty()) { this } else object : ActiveDataSet { - override val dataType: KClass get() = this@branch.dataType + override val dataType: KType get() = this@branch.dataType override fun flow(): Flow> = this@branch.flow().mapNotNull { it.name.removeHeadOrNull(branchName)?.let { name -> diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/dataTransform.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/dataTransform.kt index fdc24c1c..8b0c7787 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/dataTransform.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/dataTransform.kt @@ -1,11 +1,10 @@ package hep.dataforge.data -import hep.dataforge.actions.NamedData -import hep.dataforge.actions.named import hep.dataforge.meta.Meta import hep.dataforge.meta.MetaBuilder import hep.dataforge.meta.seal import hep.dataforge.meta.toMutableMeta +import hep.dataforge.misc.DFInternal import kotlinx.coroutines.flow.* import kotlin.contracts.InvocationKind import kotlin.contracts.contract @@ -20,23 +19,11 @@ import kotlin.reflect.typeOf * @param meta for the resulting data. By default equals input data. * @param block the transformation itself */ -private fun Data.map( - outputType: KType, - coroutineContext: CoroutineContext = EmptyCoroutineContext, - meta: Meta = this.meta, - block: suspend (T) -> R, -): LazyData = LazyData(outputType, meta, coroutineContext, listOf(this)) { - block(await()) -} - -/** - * See [map] - */ public inline fun Data.map( coroutineContext: CoroutineContext = EmptyCoroutineContext, meta: Meta = this.meta, crossinline block: suspend (T) -> R, -): LazyData = LazyData(typeOf(), meta, coroutineContext, listOf(this)) { +): Data = Data(meta, coroutineContext, listOf(this)) { block(await()) } @@ -48,7 +35,7 @@ public inline fun Data.combine( coroutineContext: CoroutineContext = EmptyCoroutineContext, meta: Meta = this.meta, crossinline block: suspend (left: T1, right: T2) -> R, -): LazyData = LazyData(typeOf(), meta, coroutineContext, listOf(this, other)) { +): Data = Data(meta, coroutineContext, listOf(this, other)) { block(await(), other.await()) } @@ -62,8 +49,7 @@ public inline fun Collection>.reduceToData( coroutineContext: CoroutineContext = EmptyCoroutineContext, meta: Meta = Meta.EMPTY, crossinline block: suspend (Collection) -> R, -): LazyData = LazyData( - typeOf(), +): Data = Data( meta, coroutineContext, this @@ -71,12 +57,13 @@ public inline fun Collection>.reduceToData( block(map { it.await() }) } +@DFInternal public fun Map>.reduceToData( outputType: KType, coroutineContext: CoroutineContext = EmptyCoroutineContext, meta: Meta = Meta.EMPTY, block: suspend (Map) -> R, -): LazyData = LazyData( +): Data = Data( outputType, meta, coroutineContext, @@ -96,8 +83,7 @@ public inline fun Map>.reduceToData( coroutineContext: CoroutineContext = EmptyCoroutineContext, meta: Meta = Meta.EMPTY, noinline block: suspend (Map) -> R, -): LazyData = LazyData( - typeOf(), +): Data = Data( meta, coroutineContext, this.values @@ -110,12 +96,13 @@ public inline fun Map>.reduceToData( /** * Transform a [Flow] of [NamedData] to a single [Data]. */ +@DFInternal public suspend fun Flow>.reduceToData( outputType: KType, coroutineContext: CoroutineContext = EmptyCoroutineContext, meta: Meta = Meta.EMPTY, transformation: suspend (Flow>) -> R, -): LazyData = LazyData( +): Data = Data( outputType, meta, coroutineContext, @@ -124,11 +111,12 @@ public suspend fun Flow>.reduceToData( transformation(this) } +@OptIn(DFInternal::class) public suspend inline fun Flow>.reduceToData( coroutineContext: CoroutineContext = EmptyCoroutineContext, meta: Meta = Meta.EMPTY, noinline transformation: suspend (Flow>) -> R, -): LazyData = reduceToData(typeOf(), coroutineContext, meta) { +): Data = reduceToData(typeOf(), coroutineContext, meta) { transformation(it) } @@ -140,7 +128,7 @@ public suspend inline fun Flow>.foldToDa coroutineContext: CoroutineContext = EmptyCoroutineContext, meta: Meta = Meta.EMPTY, noinline block: suspend (result: R, data: NamedData) -> R, -): LazyData = reduceToData( +): Data = reduceToData( coroutineContext, meta ) { it.fold(initial, block) @@ -148,6 +136,7 @@ public suspend inline fun Flow>.foldToDa //DataSet operations +@DFInternal public suspend fun DataSet.map( outputType: KType, coroutineContext: CoroutineContext = EmptyCoroutineContext, @@ -157,11 +146,14 @@ public suspend fun DataSet.map( populate( flow().map { val newMeta = it.meta.toMutableMeta().apply(metaTransform).seal() - it.map(outputType, coroutineContext, newMeta, block).named(it.name) + Data(outputType, newMeta, coroutineContext, listOf(it)) { + block(it.await()) + }.named(it.name) } ) } +@OptIn(DFInternal::class) public suspend inline fun DataSet.map( coroutineContext: CoroutineContext = EmptyCoroutineContext, noinline metaTransform: MetaBuilder.() -> Unit = {}, @@ -179,11 +171,11 @@ public suspend inline fun DataSet.reduceToData( coroutineContext: CoroutineContext = EmptyCoroutineContext, meta: Meta = Meta.EMPTY, noinline transformation: suspend (Flow>) -> R, -): LazyData = flow().reduceToData(coroutineContext, meta, transformation) +): Data = flow().reduceToData(coroutineContext, meta, transformation) public suspend inline fun DataSet.foldToData( initial: R, coroutineContext: CoroutineContext = EmptyCoroutineContext, meta: Meta = Meta.EMPTY, noinline block: suspend (result: R, data: NamedData) -> R, -): LazyData = flow().foldToData(initial, coroutineContext, meta, block) \ No newline at end of file +): Data = flow().foldToData(initial, coroutineContext, meta, block) \ No newline at end of file diff --git a/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/select.kt b/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/select.kt index def3133a..9df89f84 100644 --- a/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/select.kt +++ b/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/select.kt @@ -1,8 +1,6 @@ package hep.dataforge.data -import hep.dataforge.actions.NamedData -import hep.dataforge.actions.named -import hep.dataforge.meta.DFExperimental +import hep.dataforge.misc.DFExperimental import hep.dataforge.names.Name import hep.dataforge.names.matches import hep.dataforge.names.toName diff --git a/dataforge-data/src/jvmTest/kotlin/hep/dataforge/data/ActionsTest.kt b/dataforge-data/src/jvmTest/kotlin/hep/dataforge/data/ActionsTest.kt index 81049612..0da9c2c2 100644 --- a/dataforge-data/src/jvmTest/kotlin/hep/dataforge/data/ActionsTest.kt +++ b/dataforge-data/src/jvmTest/kotlin/hep/dataforge/data/ActionsTest.kt @@ -1,15 +1,12 @@ package hep.dataforge.data -import hep.dataforge.actions.MapAction +import hep.dataforge.actions.Action +import hep.dataforge.actions.map import kotlinx.coroutines.runBlocking import org.junit.jupiter.api.Test import kotlin.test.assertEquals -/** - * Block the thread and get data content - */ -public fun Data.value(): T = runBlocking { await() } - +@Suppress("EXPERIMENTAL_API_USAGE") class ActionsTest { val data: DataTree = runBlocking { DataTree { @@ -21,23 +18,23 @@ class ActionsTest { @Test fun testStaticMapAction() { - val plusOne = MapAction { + val plusOne = Action.map { result { it + 1 } } runBlocking { val result = plusOne.execute(data) - assertEquals(2, result.getData("1")?.value()) + assertEquals(2, result.getData("1")?.await()) } } @Test fun testDynamicMapAction() { - val plusOne = MapAction { + val plusOne = Action.map { result { it + 1 } } val datum = runBlocking { val result = plusOne.execute(data, scope = this) - result.getData("1")?.value() + result.getData("1")?.await() } assertEquals(2, datum) } diff --git a/dataforge-data/src/jvmTest/kotlin/hep/dataforge/data/DataTreeBuilderTest.kt b/dataforge-data/src/jvmTest/kotlin/hep/dataforge/data/DataTreeBuilderTest.kt index cf34cbb8..0225e9cf 100644 --- a/dataforge-data/src/jvmTest/kotlin/hep/dataforge/data/DataTreeBuilderTest.kt +++ b/dataforge-data/src/jvmTest/kotlin/hep/dataforge/data/DataTreeBuilderTest.kt @@ -18,10 +18,12 @@ internal class DataTreeBuilderTest { static("c.d", "c.d") static("c.f", "c.f") } - assertEquals("a", node.getData("primary.a")?.value()) - assertEquals("b", node.getData("primary.b")?.value()) - assertEquals("c.d", node.getData("c.d")?.value()) - assertEquals("c.f", node.getData("c.f")?.value()) + runBlocking { + assertEquals("a", node.getData("primary.a")?.await()) + assertEquals("b", node.getData("primary.b")?.await()) + assertEquals("c.d", node.getData("c.d")?.await()) + assertEquals("c.f", node.getData("c.f")?.await()) + } } @Test @@ -42,9 +44,10 @@ internal class DataTreeBuilderTest { populate(updateData) } - - assertEquals("a", node.getData("update.a")?.value()) - assertEquals("a", node.getData("primary.a")?.value()) + runBlocking { + assertEquals("a", node.getData("update.a")?.await()) + assertEquals("a", node.getData("primary.a")?.await()) + } } @Test @@ -76,7 +79,7 @@ internal class DataTreeBuilderTest { } } updateJob.join() - assertEquals(9, rootNode.getData("sub.value")?.value()) + assertEquals(9, rootNode.getData("sub.value")?.await()) cancel() } } catch (t: Throwable) { diff --git a/dataforge-io/dataforge-io-yaml/src/commonMain/kotlin/hep/dataforge/io/yaml/FrontMatterEnvelopeFormat.kt b/dataforge-io/dataforge-io-yaml/src/commonMain/kotlin/hep/dataforge/io/yaml/FrontMatterEnvelopeFormat.kt index 69100422..9a6d38f8 100644 --- a/dataforge-io/dataforge-io-yaml/src/commonMain/kotlin/hep/dataforge/io/yaml/FrontMatterEnvelopeFormat.kt +++ b/dataforge-io/dataforge-io-yaml/src/commonMain/kotlin/hep/dataforge/io/yaml/FrontMatterEnvelopeFormat.kt @@ -4,8 +4,8 @@ import hep.dataforge.context.Context import hep.dataforge.io.* import hep.dataforge.io.IOFormat.Companion.META_KEY import hep.dataforge.io.IOFormat.Companion.NAME_KEY -import hep.dataforge.meta.DFExperimental import hep.dataforge.meta.Meta +import hep.dataforge.misc.DFExperimental import kotlinx.io.* import kotlinx.io.text.readUtf8Line import kotlinx.io.text.writeUtf8String diff --git a/dataforge-io/dataforge-io-yaml/src/commonMain/kotlin/hep/dataforge/io/yaml/YamlMetaFormat.kt b/dataforge-io/dataforge-io-yaml/src/commonMain/kotlin/hep/dataforge/io/yaml/YamlMetaFormat.kt index 160683a6..9f5cdda3 100644 --- a/dataforge-io/dataforge-io-yaml/src/commonMain/kotlin/hep/dataforge/io/yaml/YamlMetaFormat.kt +++ b/dataforge-io/dataforge-io-yaml/src/commonMain/kotlin/hep/dataforge/io/yaml/YamlMetaFormat.kt @@ -8,6 +8,7 @@ import hep.dataforge.io.MetaFormatFactory import hep.dataforge.meta.* import hep.dataforge.meta.descriptors.ItemDescriptor import hep.dataforge.meta.descriptors.NodeDescriptor +import hep.dataforge.misc.DFExperimental import hep.dataforge.names.NameToken import hep.dataforge.names.withIndex import hep.dataforge.values.ListValue diff --git a/dataforge-io/dataforge-io-yaml/src/commonTest/kotlin/hep/dataforge/io/yaml/YamlMetaFormatTest.kt b/dataforge-io/dataforge-io-yaml/src/commonTest/kotlin/hep/dataforge/io/yaml/YamlMetaFormatTest.kt index 83f6d3a6..d63da939 100644 --- a/dataforge-io/dataforge-io-yaml/src/commonTest/kotlin/hep/dataforge/io/yaml/YamlMetaFormatTest.kt +++ b/dataforge-io/dataforge-io-yaml/src/commonTest/kotlin/hep/dataforge/io/yaml/YamlMetaFormatTest.kt @@ -2,10 +2,10 @@ package hep.dataforge.io.yaml import hep.dataforge.io.parse import hep.dataforge.io.toString -import hep.dataforge.meta.DFExperimental import hep.dataforge.meta.Meta import hep.dataforge.meta.get import hep.dataforge.meta.seal +import hep.dataforge.misc.DFExperimental import kotlin.test.Test import kotlin.test.assertEquals diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Consumer.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Consumer.kt index 0ef5d327..51ec6250 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Consumer.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Consumer.kt @@ -1,6 +1,6 @@ package hep.dataforge.io -import hep.dataforge.meta.DFExperimental +import hep.dataforge.misc.DFExperimental /** * A fire-and-forget consumer of messages diff --git a/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/MultipartTest.kt b/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/MultipartTest.kt index 1b239edd..e511a472 100644 --- a/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/MultipartTest.kt +++ b/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/MultipartTest.kt @@ -1,9 +1,9 @@ package hep.dataforge.io import hep.dataforge.context.Global -import hep.dataforge.meta.DFExperimental import hep.dataforge.meta.get import hep.dataforge.meta.int +import hep.dataforge.misc.DFExperimental import kotlinx.io.text.writeUtf8String import kotlin.test.Test import kotlin.test.assertEquals diff --git a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/fileIO.kt b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/fileIO.kt index 2f644878..54cf6b2c 100644 --- a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/fileIO.kt +++ b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/fileIO.kt @@ -1,9 +1,9 @@ package hep.dataforge.io -import hep.dataforge.meta.DFExperimental import hep.dataforge.meta.Meta import hep.dataforge.meta.descriptors.NodeDescriptor import hep.dataforge.meta.isEmpty +import hep.dataforge.misc.DFExperimental import kotlinx.io.* import java.nio.file.Files import java.nio.file.Path diff --git a/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/FileBinaryTest.kt b/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/FileBinaryTest.kt index 4971bf6c..8bfefd6d 100644 --- a/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/FileBinaryTest.kt +++ b/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/FileBinaryTest.kt @@ -1,7 +1,7 @@ package hep.dataforge.io import hep.dataforge.context.Global -import hep.dataforge.meta.DFExperimental +import hep.dataforge.misc.DFExperimental import kotlinx.io.asBinary import kotlinx.io.toByteArray import kotlinx.io.writeDouble diff --git a/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/FileEnvelopeTest.kt b/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/FileEnvelopeTest.kt index ab96c324..fba3ed8e 100644 --- a/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/FileEnvelopeTest.kt +++ b/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/FileEnvelopeTest.kt @@ -1,7 +1,7 @@ package hep.dataforge.io import hep.dataforge.context.Global -import hep.dataforge.meta.DFExperimental +import hep.dataforge.misc.DFExperimental import kotlinx.io.writeDouble import java.nio.file.Files import kotlin.test.Test diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Configurable.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Configurable.kt index 764ee1ea..429ac360 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Configurable.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Configurable.kt @@ -1,5 +1,6 @@ package hep.dataforge.meta +import hep.dataforge.misc.DFBuilder import hep.dataforge.names.Name import kotlin.properties.ReadWriteProperty diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MetaBuilder.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MetaBuilder.kt index f2016b80..4fe5de9b 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MetaBuilder.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MetaBuilder.kt @@ -1,5 +1,6 @@ package hep.dataforge.meta +import hep.dataforge.misc.DFBuilder import hep.dataforge.names.Name import hep.dataforge.names.asName import hep.dataforge.values.EnumValue diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MutableMeta.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MutableMeta.kt index f5353cbd..4524f11e 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MutableMeta.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MutableMeta.kt @@ -1,5 +1,6 @@ package hep.dataforge.meta +import hep.dataforge.misc.DFExperimental import hep.dataforge.names.* public interface MutableMeta> : TypedMeta, MutableItemProvider { diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/annotations.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/annotations.kt deleted file mode 100644 index 69ffa8aa..00000000 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/annotations.kt +++ /dev/null @@ -1,11 +0,0 @@ -package hep.dataforge.meta - -/** - * General marker for dataforge builders - */ -@DslMarker -public annotation class DFBuilder - -@RequiresOptIn(level = RequiresOptIn.Level.WARNING) -@Retention(AnnotationRetention.BINARY) -public annotation class DFExperimental \ No newline at end of file diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/descriptors/ItemDescriptor.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/descriptors/ItemDescriptor.kt index e03c832a..33343fe1 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/descriptors/ItemDescriptor.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/descriptors/ItemDescriptor.kt @@ -1,6 +1,7 @@ package hep.dataforge.meta.descriptors import hep.dataforge.meta.* +import hep.dataforge.misc.DFBuilder import hep.dataforge.names.* import hep.dataforge.values.* diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/mapMeta.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/mapMeta.kt index d223ce99..371a67a0 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/mapMeta.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/mapMeta.kt @@ -1,6 +1,7 @@ package hep.dataforge.meta import hep.dataforge.meta.descriptors.NodeDescriptor +import hep.dataforge.misc.DFExperimental import hep.dataforge.names.toName import hep.dataforge.values.ListValue import hep.dataforge.values.Value diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/transformations/MetaTransformation.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/transformations/MetaTransformation.kt index 710998ef..0e41b016 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/transformations/MetaTransformation.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/transformations/MetaTransformation.kt @@ -1,6 +1,7 @@ package hep.dataforge.meta.transformations import hep.dataforge.meta.* +import hep.dataforge.misc.DFExperimental import hep.dataforge.names.Name /** diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/misc/annotations.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/misc/annotations.kt new file mode 100644 index 00000000..7b271a4b --- /dev/null +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/misc/annotations.kt @@ -0,0 +1,21 @@ +package hep.dataforge.misc + +/** + * General marker for dataforge builders + */ +@DslMarker +public annotation class DFBuilder + +/** + * The declaration is experimental and could be changed in future + */ +@RequiresOptIn(level = RequiresOptIn.Level.WARNING) +@Retention(AnnotationRetention.BINARY) +public annotation class DFExperimental + +/** + * The declaration is internal to the DataForge and could use unsafe or unstable features. + */ +@RequiresOptIn(level = RequiresOptIn.Level.WARNING) +@Retention(AnnotationRetention.BINARY) +public annotation class DFInternal \ No newline at end of file diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/names/Name.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/names/Name.kt index 5d9cc7bb..3bb24ab1 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/names/Name.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/names/Name.kt @@ -1,6 +1,6 @@ package hep.dataforge.names -import hep.dataforge.meta.DFExperimental +import hep.dataforge.misc.DFExperimental import kotlinx.serialization.KSerializer import kotlinx.serialization.Serializable import kotlinx.serialization.descriptors.PrimitiveKind @@ -186,7 +186,6 @@ public fun Name.withIndex(index: String): Name { * Fast [String]-based accessor for item map */ public operator fun Map.get(body: String, query: String? = null): T? = get(NameToken(body, query)) - public operator fun Map.get(name: String): T? = get(name.toName()) public operator fun MutableMap.set(name: String, value: T): Unit = set(name.toName(), value) diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/names/nameMatcher.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/names/nameMatcher.kt index 363109f1..5b786917 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/names/nameMatcher.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/names/nameMatcher.kt @@ -1,6 +1,6 @@ package hep.dataforge.names -import hep.dataforge.meta.DFExperimental +import hep.dataforge.misc.DFExperimental /** diff --git a/dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/MetaTest.kt b/dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/MetaTest.kt index 12569bc3..89786ea1 100644 --- a/dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/MetaTest.kt +++ b/dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/MetaTest.kt @@ -1,5 +1,6 @@ package hep.dataforge.meta +import hep.dataforge.misc.DFExperimental import hep.dataforge.values.NumberValue import hep.dataforge.values.True import hep.dataforge.values.Value diff --git a/dataforge-meta/src/commonTest/kotlin/hep/dataforge/names/NameMatchTest.kt b/dataforge-meta/src/commonTest/kotlin/hep/dataforge/names/NameMatchTest.kt index b91d94a6..1ff4381f 100644 --- a/dataforge-meta/src/commonTest/kotlin/hep/dataforge/names/NameMatchTest.kt +++ b/dataforge-meta/src/commonTest/kotlin/hep/dataforge/names/NameMatchTest.kt @@ -1,6 +1,6 @@ package hep.dataforge.names -import hep.dataforge.meta.DFExperimental +import hep.dataforge.misc.DFExperimental import kotlin.test.Test import kotlin.test.assertFails import kotlin.test.assertFalse diff --git a/dataforge-tables/src/commonMain/kotlin/hep/dataforge/tables/io/textTableEnvelope.kt b/dataforge-tables/src/commonMain/kotlin/hep/dataforge/tables/io/textTableEnvelope.kt index 4deb84e6..b6be63e0 100644 --- a/dataforge-tables/src/commonMain/kotlin/hep/dataforge/tables/io/textTableEnvelope.kt +++ b/dataforge-tables/src/commonMain/kotlin/hep/dataforge/tables/io/textTableEnvelope.kt @@ -2,6 +2,7 @@ package hep.dataforge.tables.io import hep.dataforge.io.Envelope import hep.dataforge.meta.* +import hep.dataforge.misc.DFExperimental import hep.dataforge.tables.SimpleColumnHeader import hep.dataforge.tables.Table import hep.dataforge.values.Value diff --git a/dataforge-tables/src/jvmTest/kotlin/hep/dataforge/tables/io/TextRowsTest.kt b/dataforge-tables/src/jvmTest/kotlin/hep/dataforge/tables/io/TextRowsTest.kt index d97484ed..90fa3a01 100644 --- a/dataforge-tables/src/jvmTest/kotlin/hep/dataforge/tables/io/TextRowsTest.kt +++ b/dataforge-tables/src/jvmTest/kotlin/hep/dataforge/tables/io/TextRowsTest.kt @@ -1,6 +1,6 @@ package hep.dataforge.tables.io -import hep.dataforge.meta.DFExperimental +import hep.dataforge.misc.DFExperimental import hep.dataforge.tables.Table import hep.dataforge.tables.get import hep.dataforge.tables.row diff --git a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/Task.kt b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/Task.kt index df3c16af..6e6c590e 100644 --- a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/Task.kt +++ b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/Task.kt @@ -6,11 +6,13 @@ import hep.dataforge.data.GoalExecutionRestriction import hep.dataforge.meta.Meta import hep.dataforge.meta.descriptors.Described import hep.dataforge.meta.descriptors.ItemDescriptor +import hep.dataforge.misc.DFInternal import hep.dataforge.misc.Type import hep.dataforge.names.Name import hep.dataforge.workspace.Task.Companion.TYPE import kotlinx.coroutines.withContext -import kotlin.reflect.KClass +import kotlin.reflect.KType +import kotlin.reflect.typeOf @Type(TYPE) public interface Task : Described { @@ -42,8 +44,9 @@ public class TaskResultBuilder( * Data dependency cycles are not allowed. */ @Suppress("FunctionName") +@DFInternal public fun Task( - resultType: KClass, + resultType: KType, descriptor: ItemDescriptor? = null, builder: suspend TaskResultBuilder.() -> Unit, ): Task = object : Task { @@ -56,15 +59,16 @@ public fun Task( taskMeta: Meta, ): TaskResult = withContext(GoalExecutionRestriction() + workspace.goalLogger) { //TODO use safe builder and check for external data on add and detects cycles - val dataset = DataTree(resultType) { + val dataset = DataTree(resultType) { TaskResultBuilder(workspace,taskName, taskMeta, this).apply { builder() } } workspace.internalize(dataset, taskName, taskMeta) } } +@OptIn(DFInternal::class) @Suppress("FunctionName") public inline fun Task( descriptor: ItemDescriptor? = null, noinline builder: suspend TaskResultBuilder.() -> Unit, -): Task = Task(T::class, descriptor, builder) \ No newline at end of file +): Task = Task(typeOf(), descriptor, builder) \ No newline at end of file diff --git a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskData.kt b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskData.kt index 37862a48..f317b00a 100644 --- a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskData.kt +++ b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskData.kt @@ -1,7 +1,7 @@ package hep.dataforge.workspace -import hep.dataforge.actions.NamedData import hep.dataforge.data.Data +import hep.dataforge.data.NamedData import hep.dataforge.meta.Meta import hep.dataforge.names.Name diff --git a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/WorkspaceBuilder.kt b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/WorkspaceBuilder.kt index aa3519a3..cd3e6d92 100644 --- a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/WorkspaceBuilder.kt +++ b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/WorkspaceBuilder.kt @@ -7,16 +7,15 @@ import hep.dataforge.data.ActiveDataTree import hep.dataforge.data.DataSet import hep.dataforge.data.DataSetBuilder import hep.dataforge.data.DataTree -import hep.dataforge.meta.DFBuilder -import hep.dataforge.meta.DFExperimental import hep.dataforge.meta.Meta import hep.dataforge.meta.MetaBuilder import hep.dataforge.meta.descriptors.NodeDescriptor +import hep.dataforge.misc.DFBuilder +import hep.dataforge.misc.DFExperimental import hep.dataforge.names.Name import hep.dataforge.names.toName import kotlin.properties.PropertyDelegateProvider import kotlin.properties.ReadOnlyProperty -import kotlin.reflect.KClass public data class TaskReference(public val taskName: Name, public val task: Task) @@ -24,25 +23,19 @@ public interface TaskContainer { public fun registerTask(taskName: Name, task: Task<*>) } -public fun TaskContainer.registerTask( - resultType: KClass, - name: String, - descriptorBuilder: NodeDescriptor.() -> Unit = {}, - builder: suspend TaskResultBuilder.() -> Unit, -): Unit = registerTask(name.toName(), Task(resultType, NodeDescriptor(descriptorBuilder), builder)) public inline fun TaskContainer.registerTask( name: String, noinline descriptorBuilder: NodeDescriptor.() -> Unit = {}, noinline builder: suspend TaskResultBuilder.() -> Unit, -): Unit = registerTask(T::class, name, descriptorBuilder, builder) +): Unit = registerTask(name.toName(), Task(NodeDescriptor(descriptorBuilder), builder)) public inline fun TaskContainer.task( noinline descriptorBuilder: NodeDescriptor.() -> Unit = {}, noinline builder: suspend TaskResultBuilder.() -> Unit, ): PropertyDelegateProvider>> = PropertyDelegateProvider { _, property -> val taskName = property.name.toName() - val task = Task(T::class, NodeDescriptor(descriptorBuilder), builder) + val task = Task(NodeDescriptor(descriptorBuilder), builder) registerTask(taskName, task) ReadOnlyProperty { _, _ -> TaskReference(taskName, task) } } diff --git a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/envelopeData.kt b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/envelopeData.kt index c659d2c5..cebba975 100644 --- a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/envelopeData.kt +++ b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/envelopeData.kt @@ -3,15 +3,14 @@ package hep.dataforge.workspace import hep.dataforge.data.Data import hep.dataforge.data.await import hep.dataforge.io.* -import kotlin.reflect.KClass +import hep.dataforge.misc.DFInternal /** * Convert an [Envelope] to a data via given format. The actual parsing is done lazily. */ +@OptIn(DFInternal::class) public fun Envelope.toData(format: IOFormat): Data { - @Suppress("UNCHECKED_CAST") - val kclass: KClass = format.type.classifier as? KClass ?: error("IOFormat type is not a class") - return Data(kclass, meta) { + return Data(format.type, meta) { data?.readWith(format) ?: error("Can't convert envelope without data to Data") } } diff --git a/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/fileData.kt b/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/fileData.kt index 29ff0c62..a8378046 100644 --- a/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/fileData.kt +++ b/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/fileData.kt @@ -4,6 +4,7 @@ package hep.dataforge.workspace import hep.dataforge.data.* import hep.dataforge.io.* import hep.dataforge.meta.* +import hep.dataforge.misc.DFExperimental import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.runBlocking import kotlinx.coroutines.withContext @@ -15,7 +16,6 @@ import java.nio.file.StandardOpenOption import java.nio.file.spi.FileSystemProvider import java.util.zip.ZipEntry import java.util.zip.ZipOutputStream -import kotlin.reflect.KClass import kotlin.reflect.KType import kotlin.reflect.typeOf import kotlin.streams.toList @@ -36,9 +36,6 @@ internal inline fun IOPlugin.formatResolver(): FileFormatResol resolveIOFormat() ?: error("Can't resolve IO format for ${T::class}") } -private val FileFormatResolver.kClass: KClass - get() = type.classifier as? KClass ?: error("Format resolver actual type does not correspond to type parameter") - private fun newZFS(path: Path): FileSystem { val fsProvider = FileSystemProvider.installedProviders().find { it.scheme == "jar" } ?: error("Zip file system provider not found") @@ -110,7 +107,7 @@ public suspend fun IOPlugin.readDataDirectory( return readDataDirectory(fs.rootDirectories.first(), formatResolver) } if (!Files.isDirectory(path)) error("Provided path $path is not a directory") - return DataTree(formatResolver.kClass) { + return DataTree(formatResolver.type) { Files.list(path).toList().forEach { path -> val fileName = path.fileName.toString() if (fileName.startsWith(IOPlugin.META_FILE_NAME)) { diff --git a/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/FileDataTest.kt b/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/FileDataTest.kt index e089e448..df8e2673 100644 --- a/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/FileDataTest.kt +++ b/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/FileDataTest.kt @@ -4,8 +4,8 @@ import hep.dataforge.context.Global import hep.dataforge.data.* import hep.dataforge.io.IOFormat import hep.dataforge.io.io -import hep.dataforge.meta.DFExperimental import hep.dataforge.meta.Meta +import hep.dataforge.misc.DFExperimental import kotlinx.coroutines.runBlocking import kotlinx.io.Input import kotlinx.io.Output diff --git a/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/SimpleWorkspaceTest.kt b/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/SimpleWorkspaceTest.kt index b6bc9a1c..5ee08ea5 100644 --- a/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/SimpleWorkspaceTest.kt +++ b/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/SimpleWorkspaceTest.kt @@ -1,9 +1,9 @@ package hep.dataforge.workspace -import hep.dataforge.actions.get import hep.dataforge.context.* import hep.dataforge.data.* import hep.dataforge.meta.* +import hep.dataforge.names.get import hep.dataforge.names.plus import kotlinx.coroutines.flow.first import kotlinx.coroutines.flow.single