From 0a232451175811ecba11cecb07c5b3519e31f9ba Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Sun, 26 Nov 2023 12:02:54 +0300 Subject: [PATCH 01/62] fix sonatype deploy address --- build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle.kts b/build.gradle.kts index 4b5fe48c..68b1aba6 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -31,7 +31,7 @@ ksciencePublish { useSPCTeam() } repository("spc","https://maven.sciprog.center/kscience") - sonatype() + sonatype("https://oss.sonatype.org") } apiValidation { -- 2.34.1 From 5461a83417f397795332a0eb0de289bd792e6545 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Mon, 4 Dec 2023 09:55:04 +0300 Subject: [PATCH 02/62] Name refactoring. Meta.stringList accepts nullable receiver. --- build.gradle.kts | 2 +- .../kscience/dataforge/context/Plugin.kt | 4 +-- .../dataforge/context/PluginFactory.kt | 4 +-- .../kscience/dataforge/provider/dfType.kt | 6 ++-- .../kscience/dataforge/actions/MapAction.kt | 3 +- .../dataforge/actions/ReduceAction.kt | 2 +- .../kscience/dataforge/actions/SplitAction.kt | 2 +- .../space/kscience/dataforge/data/Data.kt | 4 +-- .../space/kscience/dataforge/data/DataTree.kt | 4 +-- .../dataforge/data/actionInContext.kt | 2 -- .../kscience/dataforge/io/EnvelopeFormat.kt | 4 +-- .../space/kscience/dataforge/io/IOFormat.kt | 4 +-- .../space/kscience/dataforge/io/MetaFormat.kt | 4 +-- .../space/kscience/dataforge/meta/Meta.kt | 6 ++-- .../meta/descriptors/MetaDescriptorBuilder.kt | 10 ++++-- .../dataforge/misc/{DfId.kt => DfType.kt} | 5 ++- .../meta/descriptors/DescriptorTest.kt | 6 ++-- .../kscience/dataforge/workspace/Task.kt | 4 +-- .../kscience/dataforge/workspace/Workspace.kt | 4 +-- .../kscience/dataforge/workspace/fileData.kt | 33 +++++++++++++++---- 20 files changed, 68 insertions(+), 45 deletions(-) delete mode 100644 dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/actionInContext.kt rename dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/misc/{DfId.kt => DfType.kt} (56%) diff --git a/build.gradle.kts b/build.gradle.kts index 68b1aba6..edeae557 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -8,7 +8,7 @@ plugins { allprojects { group = "space.kscience" - version = "0.7.0" + version = "0.7.1" } subprojects { diff --git a/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/context/Plugin.kt b/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/context/Plugin.kt index 152f5a76..6c5648a6 100644 --- a/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/context/Plugin.kt +++ b/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/context/Plugin.kt @@ -3,7 +3,7 @@ package space.kscience.dataforge.context import space.kscience.dataforge.context.Plugin.Companion.TARGET import space.kscience.dataforge.meta.Meta import space.kscience.dataforge.meta.MetaRepr -import space.kscience.dataforge.misc.DfId +import space.kscience.dataforge.misc.DfType import space.kscience.dataforge.misc.Named import space.kscience.dataforge.names.Name import space.kscience.dataforge.names.parseAsName @@ -18,7 +18,7 @@ import space.kscience.dataforge.provider.Provider * * create - configure - attach - detach - destroy */ -@DfId(TARGET) +@DfType(TARGET) public interface Plugin : Named, ContextAware, Provider, MetaRepr { /** diff --git a/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/context/PluginFactory.kt b/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/context/PluginFactory.kt index 0273d327..9cc67168 100644 --- a/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/context/PluginFactory.kt +++ b/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/context/PluginFactory.kt @@ -1,9 +1,9 @@ package space.kscience.dataforge.context import space.kscience.dataforge.meta.Meta -import space.kscience.dataforge.misc.DfId +import space.kscience.dataforge.misc.DfType -@DfId(PluginFactory.TYPE) +@DfType(PluginFactory.TYPE) public interface PluginFactory : Factory { public val tag: PluginTag diff --git a/dataforge-context/src/jvmMain/kotlin/space/kscience/dataforge/provider/dfType.kt b/dataforge-context/src/jvmMain/kotlin/space/kscience/dataforge/provider/dfType.kt index ab34ea50..04e681da 100644 --- a/dataforge-context/src/jvmMain/kotlin/space/kscience/dataforge/provider/dfType.kt +++ b/dataforge-context/src/jvmMain/kotlin/space/kscience/dataforge/provider/dfType.kt @@ -4,7 +4,7 @@ import space.kscience.dataforge.context.Context import space.kscience.dataforge.context.PluginBuilder import space.kscience.dataforge.context.gather import space.kscience.dataforge.misc.DFExperimental -import space.kscience.dataforge.misc.DfId +import space.kscience.dataforge.misc.DfType import space.kscience.dataforge.misc.Named import space.kscience.dataforge.names.Name import kotlin.reflect.KClass @@ -13,10 +13,10 @@ import kotlin.reflect.full.findAnnotation @DFExperimental public val KClass<*>.dfId: String - get() = findAnnotation()?.id ?: simpleName ?: "" + get() = findAnnotation()?.id ?: simpleName ?: "" /** - * Provide an object with given name inferring target from its type using [DfId] annotation + * Provide an object with given name inferring target from its type using [DfType] annotation */ @DFExperimental public inline fun Provider.provideByType(name: String): T? { diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/MapAction.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/MapAction.kt index 883b3928..7b2c94f5 100644 --- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/MapAction.kt +++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/MapAction.kt @@ -98,8 +98,7 @@ internal class MapAction( * A one-to-one mapping action */ @DFExperimental -@Suppress("FunctionName") -public inline fun Action.Companion.map( +public inline fun Action.Companion.mapping( noinline builder: MapActionBuilder.() -> Unit, ): Action = MapAction(typeOf(), builder) diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/ReduceAction.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/ReduceAction.kt index fe823bd7..a74cfad9 100644 --- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/ReduceAction.kt +++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/ReduceAction.kt @@ -112,6 +112,6 @@ internal class ReduceAction( * A one-to-one mapping action */ @DFExperimental -public inline fun Action.Companion.reduce( +public inline fun Action.Companion.reducing( noinline builder: ReduceGroupBuilder.() -> Unit, ): Action = ReduceAction(typeOf(), builder) diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/SplitAction.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/SplitAction.kt index 24745929..0ecde319 100644 --- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/SplitAction.kt +++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/SplitAction.kt @@ -87,6 +87,6 @@ internal class SplitAction( * Action that splits each incoming element into a number of fragments defined in builder */ @DFExperimental -public inline fun Action.Companion.split( +public inline fun Action.Companion.splitting( noinline builder: SplitBuilder.() -> Unit, ): Action = SplitAction(typeOf(), builder) \ No newline at end of file diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/Data.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/Data.kt index 984582e5..4d883795 100644 --- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/Data.kt +++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/Data.kt @@ -5,7 +5,7 @@ import space.kscience.dataforge.meta.Meta import space.kscience.dataforge.meta.MetaRepr import space.kscience.dataforge.meta.isEmpty import space.kscience.dataforge.misc.DFInternal -import space.kscience.dataforge.misc.DfId +import space.kscience.dataforge.misc.DfType import kotlin.coroutines.CoroutineContext import kotlin.coroutines.EmptyCoroutineContext import kotlin.reflect.KType @@ -14,7 +14,7 @@ import kotlin.reflect.typeOf /** * A data element characterized by its meta */ -@DfId(Data.TYPE) +@DfType(Data.TYPE) public interface Data : Goal, MetaRepr { /** * Type marker for the data. The type is known before the calculation takes place so it could be checked. diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataTree.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataTree.kt index bafcbea2..b9273c07 100644 --- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataTree.kt +++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataTree.kt @@ -2,7 +2,7 @@ package space.kscience.dataforge.data import space.kscience.dataforge.meta.Meta import space.kscience.dataforge.misc.DFInternal -import space.kscience.dataforge.misc.DfId +import space.kscience.dataforge.misc.DfType import space.kscience.dataforge.names.* import kotlin.collections.component1 import kotlin.collections.component2 @@ -31,7 +31,7 @@ public val DataTreeItem.type: KType /** * A tree-like [DataSet] grouped into the node. All data inside the node must inherit its type */ -@DfId(DataTree.TYPE) +@DfType(DataTree.TYPE) public interface DataTree : DataSet { /** diff --git a/dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/actionInContext.kt b/dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/actionInContext.kt deleted file mode 100644 index 33731a95..00000000 --- a/dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/actionInContext.kt +++ /dev/null @@ -1,2 +0,0 @@ -package space.kscience.dataforge.data - diff --git a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/EnvelopeFormat.kt b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/EnvelopeFormat.kt index 0df5ab27..0e998760 100644 --- a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/EnvelopeFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/EnvelopeFormat.kt @@ -4,7 +4,7 @@ import kotlinx.io.Source import space.kscience.dataforge.context.Context import space.kscience.dataforge.io.EnvelopeFormatFactory.Companion.ENVELOPE_FORMAT_TYPE import space.kscience.dataforge.meta.Meta -import space.kscience.dataforge.misc.DfId +import space.kscience.dataforge.misc.DfType import space.kscience.dataforge.names.Name import space.kscience.dataforge.names.asName import kotlin.reflect.KType @@ -17,7 +17,7 @@ public interface EnvelopeFormat : IOFormat { public fun EnvelopeFormat.read(input: Source): Envelope = readFrom(input) -@DfId(ENVELOPE_FORMAT_TYPE) +@DfType(ENVELOPE_FORMAT_TYPE) public interface EnvelopeFormatFactory : IOFormatFactory, EnvelopeFormat { override val type: KType get() = typeOf() diff --git a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/IOFormat.kt b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/IOFormat.kt index ffcadf1a..390a8bf4 100644 --- a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/IOFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/IOFormat.kt @@ -7,7 +7,7 @@ import space.kscience.dataforge.context.Context import space.kscience.dataforge.context.Factory import space.kscience.dataforge.io.IOFormatFactory.Companion.IO_FORMAT_TYPE import space.kscience.dataforge.meta.Meta -import space.kscience.dataforge.misc.DfId +import space.kscience.dataforge.misc.DfType import space.kscience.dataforge.misc.Named import space.kscience.dataforge.names.Name import space.kscience.dataforge.names.asName @@ -72,7 +72,7 @@ public fun Sink.writeWith(format: IOWriter, obj: T): Unit = format.writeTo(this, obj) -@DfId(IO_FORMAT_TYPE) +@DfType(IO_FORMAT_TYPE) public interface IOFormatFactory : Factory>, Named { /** * Explicit type for dynamic type checks diff --git a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/MetaFormat.kt b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/MetaFormat.kt index cadf87ca..f864dd2f 100644 --- a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/MetaFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/MetaFormat.kt @@ -9,7 +9,7 @@ import space.kscience.dataforge.context.Global import space.kscience.dataforge.io.MetaFormatFactory.Companion.META_FORMAT_TYPE import space.kscience.dataforge.meta.Meta import space.kscience.dataforge.meta.descriptors.MetaDescriptor -import space.kscience.dataforge.misc.DfId +import space.kscience.dataforge.misc.DfType import space.kscience.dataforge.names.Name import space.kscience.dataforge.names.asName import space.kscience.dataforge.names.plus @@ -38,7 +38,7 @@ public interface MetaFormat : IOFormat { public fun readMeta(source: Source, descriptor: MetaDescriptor? = null): Meta } -@DfId(META_FORMAT_TYPE) +@DfType(META_FORMAT_TYPE) public interface MetaFormatFactory : IOFormatFactory, MetaFormat { public val shortName: String diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/Meta.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/Meta.kt index 420625ca..979c8782 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/Meta.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/Meta.kt @@ -2,7 +2,7 @@ package space.kscience.dataforge.meta import kotlinx.serialization.Serializable import kotlinx.serialization.json.Json -import space.kscience.dataforge.misc.DfId +import space.kscience.dataforge.misc.DfType import space.kscience.dataforge.misc.unsafeCast import space.kscience.dataforge.names.* import kotlin.jvm.JvmName @@ -31,7 +31,7 @@ public fun interface MetaProvider : ValueProvider { * TODO add documentation * Same name siblings are supported via elements with the same [Name] but different indices. */ -@DfId(Meta.TYPE) +@DfType(Meta.TYPE) @Serializable(MetaSerializer::class) public interface Meta : MetaRepr, MetaProvider { public val value: Value? @@ -248,7 +248,7 @@ public inline fun > Meta?.enum(): E? = this?.value?.let { } } -public val Meta.stringList: List? get() = value?.list?.map { it.string } +public val Meta?.stringList: List? get() = this?.value?.list?.map { it.string } /** * Create a provider that uses given provider for default values if those are not found in this provider diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/descriptors/MetaDescriptorBuilder.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/descriptors/MetaDescriptorBuilder.kt index ae6c171a..95949d03 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/descriptors/MetaDescriptorBuilder.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/descriptors/MetaDescriptorBuilder.kt @@ -9,7 +9,11 @@ import space.kscience.dataforge.names.length import kotlin.collections.set public class MetaDescriptorBuilder @PublishedApi internal constructor() { - public var info: String? = null + public var description: String? = null + + @Deprecated("Replace by description", ReplaceWith("description")) + public var info: String? by ::description + public var children: MutableMap = linkedMapOf() public var multiple: Boolean = false public var valueRestriction: ValueRestriction = ValueRestriction.NONE @@ -87,7 +91,7 @@ public class MetaDescriptorBuilder @PublishedApi internal constructor() { @PublishedApi internal fun build(): MetaDescriptor = MetaDescriptor( - description = info, + description = description, children = children.mapValues { it.value.build() }, multiple = multiple, valueRestriction = valueRestriction, @@ -165,7 +169,7 @@ public inline fun > MetaDescriptorBuilder.enum( } private fun MetaDescriptor.toBuilder(): MetaDescriptorBuilder = MetaDescriptorBuilder().apply { - info = this@toBuilder.description + description = this@toBuilder.description children = this@toBuilder.children.mapValuesTo(LinkedHashMap()) { it.value.toBuilder() } multiple = this@toBuilder.multiple valueRestriction = this@toBuilder.valueRestriction diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/misc/DfId.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/misc/DfType.kt similarity index 56% rename from dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/misc/DfId.kt rename to dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/misc/DfType.kt index 5d485e23..11f548ae 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/misc/DfId.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/misc/DfType.kt @@ -5,4 +5,7 @@ package space.kscience.dataforge.misc */ @MustBeDocumented @Target(AnnotationTarget.CLASS) -public annotation class DfId(val id: String) +public annotation class DfType(val id: String) + +@Deprecated("Replace with DfType", replaceWith = ReplaceWith("DfType")) +public typealias DfId = DfType diff --git a/dataforge-meta/src/commonTest/kotlin/space/kscience/dataforge/meta/descriptors/DescriptorTest.kt b/dataforge-meta/src/commonTest/kotlin/space/kscience/dataforge/meta/descriptors/DescriptorTest.kt index e8c321fc..1a08ce34 100644 --- a/dataforge-meta/src/commonTest/kotlin/space/kscience/dataforge/meta/descriptors/DescriptorTest.kt +++ b/dataforge-meta/src/commonTest/kotlin/space/kscience/dataforge/meta/descriptors/DescriptorTest.kt @@ -11,14 +11,14 @@ class DescriptorTest { val descriptor = MetaDescriptor { node("aNode") { - info = "A root demo node" + description = "A root demo node" value("b", ValueType.NUMBER) { - info = "b number value" + description = "b number value" } node("otherNode") { value("otherValue", ValueType.BOOLEAN) { default(false) - info = "default value" + description = "default value" } } } diff --git a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/Task.kt b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/Task.kt index a1ef7be2..329d9c5a 100644 --- a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/Task.kt +++ b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/Task.kt @@ -9,7 +9,7 @@ import space.kscience.dataforge.meta.MetaRepr import space.kscience.dataforge.meta.Specification import space.kscience.dataforge.meta.descriptors.Described import space.kscience.dataforge.meta.descriptors.MetaDescriptor -import space.kscience.dataforge.misc.DfId +import space.kscience.dataforge.misc.DfType import space.kscience.dataforge.names.Name import space.kscience.dataforge.workspace.Task.Companion.TYPE import kotlin.reflect.KType @@ -19,7 +19,7 @@ import kotlin.reflect.typeOf * A configurable task that could be executed on a workspace. The [TaskResult] represents a lazy result of the task. * In general no computations should be made until the result is called. */ -@DfId(TYPE) +@DfType(TYPE) public interface Task : Described { /** diff --git a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/Workspace.kt b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/Workspace.kt index ee00f539..37b473db 100644 --- a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/Workspace.kt +++ b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/Workspace.kt @@ -6,7 +6,7 @@ import space.kscience.dataforge.data.DataSet import space.kscience.dataforge.data.asSequence import space.kscience.dataforge.meta.Meta import space.kscience.dataforge.meta.MutableMeta -import space.kscience.dataforge.misc.DfId +import space.kscience.dataforge.misc.DfType import space.kscience.dataforge.names.Name import space.kscience.dataforge.provider.Provider @@ -18,7 +18,7 @@ public interface DataSelector{ /** * An environment for pull-mode computation */ -@DfId(Workspace.TYPE) +@DfType(Workspace.TYPE) public interface Workspace : ContextAware, Provider { /** * The whole data node for current workspace diff --git a/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/fileData.kt b/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/fileData.kt index d9f678b3..ce1b5152 100644 --- a/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/fileData.kt +++ b/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/fileData.kt @@ -11,7 +11,6 @@ import space.kscience.dataforge.data.* import space.kscience.dataforge.io.* import space.kscience.dataforge.meta.Meta import space.kscience.dataforge.meta.copy -import space.kscience.dataforge.meta.get import space.kscience.dataforge.meta.string import space.kscience.dataforge.misc.DFExperimental import space.kscience.dataforge.misc.DFInternal @@ -27,10 +26,7 @@ import java.nio.file.WatchEvent import java.nio.file.attribute.BasicFileAttributes import java.nio.file.spi.FileSystemProvider import java.time.Instant -import kotlin.io.path.extension -import kotlin.io.path.name -import kotlin.io.path.nameWithoutExtension -import kotlin.io.path.readAttributes +import kotlin.io.path.* import kotlin.reflect.KType import kotlin.reflect.typeOf @@ -92,7 +88,7 @@ public fun IOPlugin.readDataFile( context(IOPlugin) @DFExperimental -private fun DataSetBuilder.directory( +public fun DataSetBuilder.directory( path: Path, ignoreExtensions: Set, formatResolver: FileFormatResolver, @@ -145,7 +141,7 @@ public inline fun IOPlugin.readDataDirectory( ): DataTree = readDataDirectory(typeOf(), path, ignoreExtensions, formatResolver) /** - * Read raw binary data tree from the directory. All files are read as-is (save for meta files). + * Read a raw binary data tree from the directory. All files are read as-is (save for meta files). */ @DFExperimental public fun IOPlugin.readRawDirectory( @@ -260,6 +256,29 @@ public suspend fun IOPlugin.writeDataDirectory( } } +/** + * Reads the specified resources and returns a [DataTree] containing the data. + * + * @param resources The names of the resources to read. + * @param classLoader The class loader to use for loading the resources. By default, it uses the current thread's context class loader. + * @return A DataTree containing the data read from the resources. + */ +@DFExperimental +private fun IOPlugin.readResources( + vararg resources: String, + classLoader: ClassLoader = Thread.currentThread().contextClassLoader, +): DataTree { +// require(resource.isNotBlank()) {"Can't mount root resource tree as data root"} + return DataTree { + resources.forEach { resource -> + val path = classLoader.getResource(resource)?.toURI()?.toPath() ?: error( + "Resource with name $resource is not resolved" + ) + node(resource, readRawDirectory(path)) + } + } +} + /** * Add file/directory-based data tree item * -- 2.34.1 From 946ac88480342b0b96205fcec182c75435953ecc Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Tue, 5 Dec 2023 15:13:50 +0300 Subject: [PATCH 03/62] Partially fixed a bug with `MutableMeta` observable wrappers. --- CHANGELOG.md | 2 + dataforge-context/build.gradle.kts | 6 ++- .../kscience/dataforge/context/loggingWasm.kt | 3 ++ dataforge-data/build.gradle.kts | 4 +- .../kscience/dataforge/data/ActionsTest.kt | 6 +-- dataforge-io/build.gradle.kts | 3 +- dataforge-meta/api/dataforge-meta.api | 6 ++- dataforge-meta/build.gradle.kts | 1 + .../dataforge/meta/ObservableMetaWrapper.kt | 41 ++++++++++------ .../space/kscience/dataforge/names/Name.kt | 4 ++ .../dataforge/meta/ObservableMetaTest.kt | 49 +++++++++++++++++++ .../space/kscience/dataforge/misc/castJs.kt | 4 +- .../kscience/dataforge/misc/castNative.kt | 2 +- .../space/kscience/dataforge/misc/castWasm.kt | 4 ++ dataforge-scripting/build.gradle.kts | 6 +-- dataforge-workspace/build.gradle.kts | 26 +++++----- gradle.properties | 2 +- 17 files changed, 125 insertions(+), 44 deletions(-) create mode 100644 dataforge-context/src/wasmJsMain/kotlin/space/kscience/dataforge/context/loggingWasm.kt create mode 100644 dataforge-meta/src/commonTest/kotlin/space/kscience/dataforge/meta/ObservableMetaTest.kt create mode 100644 dataforge-meta/src/wasmJsMain/kotlin/space/kscience/dataforge/misc/castWasm.kt diff --git a/CHANGELOG.md b/CHANGELOG.md index e32cc6bf..87ccad00 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## Unreleased ### Added +- Wasm artifacts ### Changed @@ -11,6 +12,7 @@ ### Removed ### Fixed +- Partially fixed a bug with `MutableMeta` observable wrappers. ### Security diff --git a/dataforge-context/build.gradle.kts b/dataforge-context/build.gradle.kts index be9036d0..b59abed0 100644 --- a/dataforge-context/build.gradle.kts +++ b/dataforge-context/build.gradle.kts @@ -8,12 +8,14 @@ kscience { jvm() js() native() + wasm() useCoroutines() useSerialization() - dependencies { + commonMain { api(project(":dataforge-meta")) + api(spclibs.atomicfu) } - dependencies(jvmMain){ + jvmMain{ api(kotlin("reflect")) api("org.slf4j:slf4j-api:1.7.30") } diff --git a/dataforge-context/src/wasmJsMain/kotlin/space/kscience/dataforge/context/loggingWasm.kt b/dataforge-context/src/wasmJsMain/kotlin/space/kscience/dataforge/context/loggingWasm.kt new file mode 100644 index 00000000..740957b4 --- /dev/null +++ b/dataforge-context/src/wasmJsMain/kotlin/space/kscience/dataforge/context/loggingWasm.kt @@ -0,0 +1,3 @@ +package space.kscience.dataforge.context + +internal actual fun getGlobalLoggerFactory(): PluginFactory = DefaultLogManager \ No newline at end of file diff --git a/dataforge-data/build.gradle.kts b/dataforge-data/build.gradle.kts index 9f96604a..ea542290 100644 --- a/dataforge-data/build.gradle.kts +++ b/dataforge-data/build.gradle.kts @@ -6,9 +6,11 @@ kscience{ jvm() js() native() + wasm() useCoroutines() dependencies { - api(project(":dataforge-meta")) + api(spclibs.atomicfu) + api(projects.dataforgeMeta) api(kotlin("reflect")) } } diff --git a/dataforge-data/src/jvmTest/kotlin/space/kscience/dataforge/data/ActionsTest.kt b/dataforge-data/src/jvmTest/kotlin/space/kscience/dataforge/data/ActionsTest.kt index 3987cd19..b24c4f27 100644 --- a/dataforge-data/src/jvmTest/kotlin/space/kscience/dataforge/data/ActionsTest.kt +++ b/dataforge-data/src/jvmTest/kotlin/space/kscience/dataforge/data/ActionsTest.kt @@ -6,7 +6,7 @@ import kotlinx.coroutines.test.runTest import org.junit.jupiter.api.Test import space.kscience.dataforge.actions.Action import space.kscience.dataforge.actions.invoke -import space.kscience.dataforge.actions.map +import space.kscience.dataforge.actions.mapping import space.kscience.dataforge.misc.DFExperimental import kotlin.test.assertEquals @@ -20,7 +20,7 @@ internal class ActionsTest { } } - val plusOne = Action.map { + val plusOne = Action.mapping { result { it + 1 } } val result = plusOne(data) @@ -31,7 +31,7 @@ internal class ActionsTest { fun testDynamicMapAction() = runTest { val data: DataSourceBuilder = DataSource() - val plusOne = Action.map { + val plusOne = Action.mapping { result { it + 1 } } diff --git a/dataforge-io/build.gradle.kts b/dataforge-io/build.gradle.kts index 6d3c888c..f7197197 100644 --- a/dataforge-io/build.gradle.kts +++ b/dataforge-io/build.gradle.kts @@ -4,12 +4,13 @@ plugins { description = "IO module" -val ioVersion = "0.2.1" +val ioVersion = "0.3.0" kscience { jvm() js() native() + wasm() useSerialization() useSerialization(sourceSet = space.kscience.gradle.DependencySourceSet.TEST) { cbor() diff --git a/dataforge-meta/api/dataforge-meta.api b/dataforge-meta/api/dataforge-meta.api index 7daa7540..1700ca7d 100644 --- a/dataforge-meta/api/dataforge-meta.api +++ b/dataforge-meta/api/dataforge-meta.api @@ -724,6 +724,7 @@ public final class space/kscience/dataforge/meta/descriptors/MetaDescriptorBuild public final fun getAttributes ()Lspace/kscience/dataforge/meta/MutableMeta; public final fun getChildren ()Ljava/util/Map; public final fun getDefault ()Lspace/kscience/dataforge/meta/Value; + public final fun getDescription ()Ljava/lang/String; public final fun getIndexKey ()Ljava/lang/String; public final fun getInfo ()Ljava/lang/String; public final fun getMultiple ()Z @@ -737,6 +738,7 @@ public final class space/kscience/dataforge/meta/descriptors/MetaDescriptorBuild public final fun setAttributes (Lspace/kscience/dataforge/meta/MutableMeta;)V public final fun setChildren (Ljava/util/Map;)V public final fun setDefault (Lspace/kscience/dataforge/meta/Value;)V + public final fun setDescription (Ljava/lang/String;)V public final fun setIndexKey (Ljava/lang/String;)V public final fun setInfo (Ljava/lang/String;)V public final fun setMultiple (Z)V @@ -903,7 +905,7 @@ public abstract interface annotation class space/kscience/dataforge/misc/DFExper public abstract interface annotation class space/kscience/dataforge/misc/DFInternal : java/lang/annotation/Annotation { } -public abstract interface annotation class space/kscience/dataforge/misc/DfId : java/lang/annotation/Annotation { +public abstract interface annotation class space/kscience/dataforge/misc/DfType : java/lang/annotation/Annotation { public abstract fun id ()Ljava/lang/String; } @@ -944,6 +946,7 @@ public final class space/kscience/dataforge/names/NameKt { public static final fun asName (Lspace/kscience/dataforge/names/NameToken;)Lspace/kscience/dataforge/names/Name; public static final fun cutFirst (Lspace/kscience/dataforge/names/Name;)Lspace/kscience/dataforge/names/Name; public static final fun cutLast (Lspace/kscience/dataforge/names/Name;)Lspace/kscience/dataforge/names/Name; + public static final fun endsWith (Lspace/kscience/dataforge/names/Name;Ljava/lang/String;)Z public static final fun endsWith (Lspace/kscience/dataforge/names/Name;Lspace/kscience/dataforge/names/Name;)Z public static final fun endsWith (Lspace/kscience/dataforge/names/Name;Lspace/kscience/dataforge/names/NameToken;)Z public static final fun first (Lspace/kscience/dataforge/names/Name;)Lspace/kscience/dataforge/names/NameToken; @@ -966,6 +969,7 @@ public final class space/kscience/dataforge/names/NameKt { public static final fun removeHeadOrNull (Lspace/kscience/dataforge/names/Name;Lspace/kscience/dataforge/names/Name;)Lspace/kscience/dataforge/names/Name; public static final fun replaceLast (Lspace/kscience/dataforge/names/Name;Lkotlin/jvm/functions/Function1;)Lspace/kscience/dataforge/names/Name; public static final fun set (Ljava/util/Map;Ljava/lang/String;Ljava/lang/Object;)V + public static final fun startsWith (Lspace/kscience/dataforge/names/Name;Ljava/lang/String;)Z public static final fun startsWith (Lspace/kscience/dataforge/names/Name;Lspace/kscience/dataforge/names/Name;)Z public static final fun startsWith (Lspace/kscience/dataforge/names/Name;Lspace/kscience/dataforge/names/NameToken;)Z public static final fun withIndex (Lspace/kscience/dataforge/names/Name;Ljava/lang/String;)Lspace/kscience/dataforge/names/Name; diff --git a/dataforge-meta/build.gradle.kts b/dataforge-meta/build.gradle.kts index 51b07113..d150ef98 100644 --- a/dataforge-meta/build.gradle.kts +++ b/dataforge-meta/build.gradle.kts @@ -6,6 +6,7 @@ kscience { jvm() js() native() + wasm() useSerialization{ json() } diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/ObservableMetaWrapper.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/ObservableMetaWrapper.kt index 76645d83..71e15aa9 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/ObservableMetaWrapper.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/ObservableMetaWrapper.kt @@ -6,60 +6,71 @@ import space.kscience.dataforge.names.* /** * A class that takes [MutableMeta] provider and adds obsevability on top of that + * + * TODO rewrite to properly work with detached nodes */ private class ObservableMetaWrapper( val root: MutableMeta, - val absoluteName: Name, + val nodeName: Name, val listeners: MutableSet, ) : ObservableMutableMeta { override val items: Map get() = root.items.keys.associateWith { - ObservableMetaWrapper(root, absoluteName + it, listeners) + ObservableMetaWrapper(root, nodeName + it, listeners) } - override fun get(name: Name): ObservableMutableMeta? = - root.get(name)?.let { ObservableMetaWrapper(root, this.absoluteName + name, listeners) } + override fun get(name: Name): ObservableMutableMeta? = if (root[nodeName + name] == null) { + null + } else { + ObservableMetaWrapper(root, nodeName + name, listeners) + } @ThreadSafe override fun onChange(owner: Any?, callback: Meta.(name: Name) -> Unit) { listeners.add( - MetaListener(Pair(owner, absoluteName)) { name -> - if (name.startsWith(absoluteName)) { - (this[absoluteName] ?: Meta.EMPTY).callback(name.removeFirstOrNull(absoluteName)!!) + MetaListener(Pair(owner, nodeName)) { fullName -> + if (fullName.startsWith(nodeName)) { + root[nodeName]?.callback(fullName.removeFirstOrNull(nodeName)!!) } } ) } override fun removeListener(owner: Any?) { - listeners.removeAll { it.owner === Pair(owner, absoluteName) } + listeners.removeAll { it.owner === Pair(owner, nodeName) } } override fun invalidate(name: Name) { - listeners.forEach { it.callback(this, name) } + listeners.forEach { it.callback(this, nodeName + name) } } override var value: Value? - get() = root.value + get() = root[nodeName]?.value set(value) { - root.value = value + root.getOrCreate(nodeName).value = value invalidate(Name.EMPTY) } override fun getOrCreate(name: Name): ObservableMutableMeta = - ObservableMetaWrapper(root, this.absoluteName + name, listeners) + ObservableMetaWrapper(root, nodeName + name, listeners) - override fun set(name: Name, node: Meta?) { + fun removeNode(name: Name): Meta? { val oldMeta = get(name) //don't forget to remove listener oldMeta?.removeListener(this) - root.set(absoluteName + name, node) + + return oldMeta + } + + override fun set(name: Name, node: Meta?) { + val oldMeta = removeNode(name) + root[nodeName + name] = node if (oldMeta != node) { invalidate(name) } } - override fun toMeta(): Meta = root[absoluteName]?.toMeta() ?: Meta.EMPTY + override fun toMeta(): Meta = root[nodeName]?.toMeta() ?: Meta.EMPTY override fun toString(): String = Meta.toString(this) override fun equals(other: Any?): Boolean = Meta.equals(this, other as? Meta) diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/names/Name.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/names/Name.kt index 160ea3a1..1c9a9cf3 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/names/Name.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/names/Name.kt @@ -216,9 +216,13 @@ public fun Name.endsWith(token: NameToken): Boolean = lastOrNull() == token public fun Name.startsWith(name: Name): Boolean = this.length >= name.length && (this == name || tokens.subList(0, name.length) == name.tokens) +public fun Name.startsWith(name: String): Boolean = startsWith(name.parseAsName()) + public fun Name.endsWith(name: Name): Boolean = this.length >= name.length && (this == name || tokens.subList(length - name.length, length) == name.tokens) +public fun Name.endsWith(name: String): Boolean = endsWith(name.parseAsName()) + /** * if [this] starts with given [head] name, returns the reminder of the name (could be empty). Otherwise, returns null */ diff --git a/dataforge-meta/src/commonTest/kotlin/space/kscience/dataforge/meta/ObservableMetaTest.kt b/dataforge-meta/src/commonTest/kotlin/space/kscience/dataforge/meta/ObservableMetaTest.kt new file mode 100644 index 00000000..4681ec12 --- /dev/null +++ b/dataforge-meta/src/commonTest/kotlin/space/kscience/dataforge/meta/ObservableMetaTest.kt @@ -0,0 +1,49 @@ +package space.kscience.dataforge.meta + +import space.kscience.dataforge.names.startsWith +import kotlin.test.Ignore +import kotlin.test.Test +import kotlin.test.assertEquals + +class ObservableMetaTest { + + @Test + fun asObservable() { + val meta = MutableMeta { + "data" put { + "x" put ListValue(1, 2, 3) + "y" put ListValue(5, 6, 7) + "type" put "scatter" + } + }.asObservable() + + assertEquals("scatter", meta["data.type"].string) + } + + @Test + @Ignore + fun detachNode() { + val meta = MutableMeta { + "data" put { + "x" put ListValue(1, 2, 3) + "y" put ListValue(5, 6, 7) + "type" put "scatter" + } + }.asObservable() + + var collector: Value? = null + + meta.onChange(null) { name -> + if (name.startsWith("data")) { + collector = get("data.z")?.value + } + } + + val data = meta["data"]!! + + meta.remove("data") + + data["z"] = ListValue(2, 5, 7) + assertEquals(null, collector) + } +} \ No newline at end of file diff --git a/dataforge-meta/src/jsMain/kotlin/space/kscience/dataforge/misc/castJs.kt b/dataforge-meta/src/jsMain/kotlin/space/kscience/dataforge/misc/castJs.kt index b404ebb4..b057bcbe 100644 --- a/dataforge-meta/src/jsMain/kotlin/space/kscience/dataforge/misc/castJs.kt +++ b/dataforge-meta/src/jsMain/kotlin/space/kscience/dataforge/misc/castJs.kt @@ -1,5 +1,5 @@ package space.kscience.dataforge.misc import kotlin.js.unsafeCast as unsafeCastJs -@Suppress("UNCHECKED_CAST", "NOTHING_TO_INLINE") -public actual inline fun Any?.unsafeCast(): T = this.unsafeCastJs() \ No newline at end of file +@Suppress("NOTHING_TO_INLINE") +public actual inline fun Any?.unsafeCast(): T = unsafeCastJs() \ No newline at end of file diff --git a/dataforge-meta/src/nativeMain/kotlin/space/kscience/dataforge/misc/castNative.kt b/dataforge-meta/src/nativeMain/kotlin/space/kscience/dataforge/misc/castNative.kt index 27d399fe..4d9aa758 100644 --- a/dataforge-meta/src/nativeMain/kotlin/space/kscience/dataforge/misc/castNative.kt +++ b/dataforge-meta/src/nativeMain/kotlin/space/kscience/dataforge/misc/castNative.kt @@ -1,4 +1,4 @@ package space.kscience.dataforge.misc -@Suppress("UNCHECKED_CAST", "NOTHING_TO_INLINE") +@Suppress("UNCHECKED_CAST") public actual inline fun Any?.unsafeCast(): T = this as T \ No newline at end of file diff --git a/dataforge-meta/src/wasmJsMain/kotlin/space/kscience/dataforge/misc/castWasm.kt b/dataforge-meta/src/wasmJsMain/kotlin/space/kscience/dataforge/misc/castWasm.kt new file mode 100644 index 00000000..27d399fe --- /dev/null +++ b/dataforge-meta/src/wasmJsMain/kotlin/space/kscience/dataforge/misc/castWasm.kt @@ -0,0 +1,4 @@ +package space.kscience.dataforge.misc + +@Suppress("UNCHECKED_CAST", "NOTHING_TO_INLINE") +public actual inline fun Any?.unsafeCast(): T = this as T \ No newline at end of file diff --git a/dataforge-scripting/build.gradle.kts b/dataforge-scripting/build.gradle.kts index be81fe70..d9d87742 100644 --- a/dataforge-scripting/build.gradle.kts +++ b/dataforge-scripting/build.gradle.kts @@ -4,15 +4,15 @@ plugins { kscience{ jvm() - dependencies { + commonMain { api(projects.dataforgeWorkspace) implementation(kotlin("scripting-common")) } - dependencies(jvmMain){ + jvmMain{ implementation(kotlin("scripting-jvm-host")) implementation(kotlin("scripting-jvm")) } - dependencies(jvmTest){ + jvmTest{ implementation(spclibs.logback.classic) } } diff --git a/dataforge-workspace/build.gradle.kts b/dataforge-workspace/build.gradle.kts index ec117865..5fa555eb 100644 --- a/dataforge-workspace/build.gradle.kts +++ b/dataforge-workspace/build.gradle.kts @@ -2,29 +2,27 @@ plugins { id("space.kscience.gradle.mpp") } -kscience{ +kscience { jvm() js() native() + wasm() useCoroutines() - useSerialization{ + useSerialization { protobuf() } - commonMain{ - dependencies { - api(projects.dataforgeContext) - api(projects.dataforgeData) - api(projects.dataforgeIo) - } + commonMain { + api(projects.dataforgeContext) + api(projects.dataforgeData) + api(projects.dataforgeIo) + } - jvmTest{ - dependencies { - implementation(spclibs.logback.classic) - implementation(projects.dataforgeIo.dataforgeIoYaml) - } + jvmTest { + implementation(spclibs.logback.classic) + implementation(projects.dataforgeIo.dataforgeIoYaml) } } -readme{ +readme { maturity = space.kscience.gradle.Maturity.EXPERIMENTAL } \ No newline at end of file diff --git a/gradle.properties b/gradle.properties index 31ef2f9a..3734d13e 100644 --- a/gradle.properties +++ b/gradle.properties @@ -6,5 +6,5 @@ kotlin.mpp.stability.nowarn=true kotlin.incremental.js.ir=true kotlin.native.ignoreDisabledTargets=true -toolsVersion=0.15.1-kotlin-1.9.21 +toolsVersion=0.15.2-kotlin-1.9.21 #kotlin.experimental.tryK2=true \ No newline at end of file -- 2.34.1 From a32cbe95dc955e2c2e9b1c69a42ac3f29a6ea98d Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Tue, 19 Dec 2023 17:53:01 +0300 Subject: [PATCH 04/62] Add listOfSpec delegate to Scheme --- .../space/kscience/dataforge/meta/Specification.kt | 9 ++++++++- .../kotlin/space/kscience/dataforge/names/Name.kt | 10 ++++------ 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/Specification.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/Specification.kt index 6d3afbea..7f3dd2a6 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/Specification.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/Specification.kt @@ -7,7 +7,7 @@ import space.kscience.dataforge.names.asName import kotlin.properties.ReadWriteProperty import kotlin.reflect.KProperty -public interface ReadOnlySpecification: Described { +public interface ReadOnlySpecification : Described { /** * Read generic read-only meta with this [Specification] producing instance of desired type. @@ -128,3 +128,10 @@ public fun MutableMeta.listOfSpec( setIndexed(name, value.map { it.toMeta() }) } } + + +@DFExperimental +public fun Scheme.listOfSpec( + spec: Specification, + key: Name? = null, +): ReadWriteProperty> = meta.listOfSpec(spec, key) diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/names/Name.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/names/Name.kt index 1c9a9cf3..5e4b98a8 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/names/Name.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/names/Name.kt @@ -16,12 +16,10 @@ public class Name(public val tokens: List) { override fun toString(): String = tokens.joinToString(separator = NAME_SEPARATOR) { it.toString() } - override fun equals(other: Any?): Boolean { - return when (other) { - is Name -> this.tokens == other.tokens - is NameToken -> this.length == 1 && this.tokens.first() == other - else -> false - } + override fun equals(other: Any?): Boolean = when (other) { + is Name -> this.tokens == other.tokens + is NameToken -> this.length == 1 && this.tokens.first() == other + else -> false } private val cachedHashCode = if (tokens.size == 1) { -- 2.34.1 From 4de9e1865c3458a2cb41436dabb137c38d8a7740 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Tue, 19 Dec 2023 17:53:17 +0300 Subject: [PATCH 05/62] Add listOfSpec delegate to Scheme --- build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle.kts b/build.gradle.kts index edeae557..1fa05824 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -8,7 +8,7 @@ plugins { allprojects { group = "space.kscience" - version = "0.7.1" + version = "0.7.2-dev-1" } subprojects { -- 2.34.1 From f7dec52438b71cabf6d09d1525d026fa022ddee3 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Tue, 19 Dec 2023 18:21:42 +0300 Subject: [PATCH 06/62] Add automatic MetaConverter for serializeable objects --- CHANGELOG.md | 1 + .../meta/transformations/MetaConverter.kt | 46 +++++++++++++++---- 2 files changed, 37 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 87ccad00..0d89307a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ ### Added - Wasm artifacts +- Add automatic MetaConverter for serializeable objects ### Changed diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/transformations/MetaConverter.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/transformations/MetaConverter.kt index 0dfb63d7..17774913 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/transformations/MetaConverter.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/transformations/MetaConverter.kt @@ -1,7 +1,12 @@ package space.kscience.dataforge.meta.transformations +import kotlinx.serialization.KSerializer +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.encodeToJsonElement +import kotlinx.serialization.serializer import space.kscience.dataforge.meta.* import space.kscience.dataforge.meta.descriptors.MetaDescriptor +import space.kscience.dataforge.misc.DFExperimental import kotlin.reflect.KType import kotlin.reflect.typeOf @@ -141,19 +146,40 @@ public interface MetaConverter { public fun valueList( writer: (T) -> Value = { Value.of(it) }, reader: (Value) -> T, - ): MetaConverter> = - object : MetaConverter> { - override val type: KType = typeOf>() + ): MetaConverter> = object : MetaConverter> { + override val type: KType = typeOf>() - override val descriptor: MetaDescriptor = MetaDescriptor { - valueType(ValueType.LIST) - } - - override fun metaToObjectOrNull(meta: Meta): List? = meta.value?.list?.map(reader) - - override fun objectToMeta(obj: List): Meta = Meta(obj.map(writer).asValue()) + override val descriptor: MetaDescriptor = MetaDescriptor { + valueType(ValueType.LIST) } + override fun metaToObjectOrNull(meta: Meta): List? = meta.value?.list?.map(reader) + + override fun objectToMeta(obj: List): Meta = Meta(obj.map(writer).asValue()) + } + + /** + * Automatically generate [MetaConverter] for a class using its serializer and optional [descriptor] + */ + @DFExperimental + public inline fun serializable( + descriptor: MetaDescriptor? = null, + ): MetaConverter = object : MetaConverter { + override val type: KType = typeOf() + private val serializer: KSerializer = serializer() + + override fun metaToObjectOrNull(meta: Meta): T? { + val json = meta.toJson(descriptor) + return Json.decodeFromJsonElement(serializer, json) + } + + override fun objectToMeta(obj: T): Meta { + val json = Json.encodeToJsonElement(obj) + return json.toMeta(descriptor) + } + + } + } } -- 2.34.1 From f9e7d0098feda048fa886a169cf6b62c1cf3ae71 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Wed, 20 Dec 2023 11:07:30 +0300 Subject: [PATCH 07/62] Add convertable and serializable extensions to Meta delegates --- CHANGELOG.md | 2 + .../kscience/dataforge/meta/MetaDelegate.kt | 43 +++++++++++++- .../kscience/dataforge/meta/MutableMeta.kt | 2 +- .../dataforge/meta/MutableMetaDelegate.kt | 56 ++++++++++++++++++- 4 files changed, 96 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0d89307a..7cc1c21d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,10 +5,12 @@ ### Added - Wasm artifacts - Add automatic MetaConverter for serializeable objects +- Add Meta and MutableMeta delegates for convertable and serializeable ### Changed ### Deprecated +- `node(key,converter)` in favor of `serializable` delegate ### Removed diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MetaDelegate.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MetaDelegate.kt index 73923d56..42e4fe14 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MetaDelegate.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MetaDelegate.kt @@ -1,6 +1,8 @@ package space.kscience.dataforge.meta +import space.kscience.dataforge.meta.descriptors.MetaDescriptor import space.kscience.dataforge.meta.transformations.MetaConverter +import space.kscience.dataforge.misc.DFExperimental import space.kscience.dataforge.names.Name import space.kscience.dataforge.names.asName import kotlin.properties.ReadOnlyProperty @@ -11,13 +13,48 @@ public fun MetaProvider.node(key: Name? = null): ReadOnlyProperty = get(key ?: property.name.asName()) } -public fun MetaProvider.node( +/** + * Use [converter] to read the Meta node + */ +public fun MetaProvider.convertable( + converter: MetaConverter, key: Name? = null, - converter: MetaConverter ): ReadOnlyProperty = ReadOnlyProperty { _, property -> get(key ?: property.name.asName())?.let { converter.metaToObject(it) } } +/** + * Use object serializer to transform it to Meta and back + */ +@DFExperimental +public inline fun MetaProvider.serializable( + descriptor: MetaDescriptor? = null, + key: Name? = null, +): ReadOnlyProperty = convertable(MetaConverter.serializable(descriptor), key) + +@Deprecated("Use convertable", ReplaceWith("convertable(converter, key)")) +public fun MetaProvider.node( + key: Name? = null, + converter: MetaConverter, +): ReadOnlyProperty = convertable(converter, key) + +/** + * Use [converter] to convert a list of same name siblings meta to object + */ +public fun Meta.listOfConvertable( + converter: MetaConverter, + key: Name? = null, +): ReadOnlyProperty> = ReadOnlyProperty{_, property -> + val name = key ?: property.name.asName() + getIndexed(name).values.map { converter.metaToObject(it) } +} + +@DFExperimental +public inline fun Meta.listOfSerializable( + descriptor: MetaDescriptor? = null, + key: Name? = null, +): ReadOnlyProperty> = listOfConvertable(MetaConverter.serializable(descriptor), key) + /** * A property delegate that uses custom key */ @@ -27,7 +64,7 @@ public fun MetaProvider.value(key: Name? = null): ReadOnlyProperty public fun MetaProvider.value( key: Name? = null, - reader: (Value?) -> R + reader: (Value?) -> R, ): ReadOnlyProperty = ReadOnlyProperty { _, property -> reader(get(key ?: property.name.asName())?.value) } diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MutableMeta.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MutableMeta.kt index 7e05d215..24a09240 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MutableMeta.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MutableMeta.kt @@ -165,7 +165,7 @@ public fun MutableMetaProvider.remove(key: String) { // node setters -public operator fun MutableMetaProvider.set(Key: NameToken, value: Meta): Unit = set(Key.asName(), value) +public operator fun MutableMetaProvider.set(key: NameToken, value: Meta): Unit = set(key.asName(), value) public operator fun MutableMetaProvider.set(key: String, value: Meta): Unit = set(Name.parse(key), value) diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MutableMetaDelegate.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MutableMetaDelegate.kt index 0f28523c..d77348dd 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MutableMetaDelegate.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MutableMetaDelegate.kt @@ -1,6 +1,8 @@ package space.kscience.dataforge.meta +import space.kscience.dataforge.meta.descriptors.MetaDescriptor import space.kscience.dataforge.meta.transformations.MetaConverter +import space.kscience.dataforge.misc.DFExperimental import space.kscience.dataforge.names.Name import space.kscience.dataforge.names.asName import kotlin.properties.ReadWriteProperty @@ -20,10 +22,18 @@ public fun MutableMetaProvider.node(key: Name? = null): ReadWriteProperty MutableMetaProvider.node(key: Name? = null, converter: MetaConverter): ReadWriteProperty = +/** + * Use [converter] to transform an object to Meta and back. + * Note that mutation of the object does not change Meta. + */ +public fun MutableMetaProvider.convertable( + converter: MetaConverter, + key: Name? = null, +): ReadWriteProperty = object : ReadWriteProperty { override fun getValue(thisRef: Any?, property: KProperty<*>): T? { - return get(key ?: property.name.asName())?.let { converter.metaToObject(it) } + val name = key ?: property.name.asName() + return get(name)?.let { converter.metaToObject(it) } } override fun setValue(thisRef: Any?, property: KProperty<*>, value: T?) { @@ -32,6 +42,46 @@ public fun MutableMetaProvider.node(key: Name? = null, converter: MetaConver } } +@Deprecated("Use convertable", ReplaceWith("convertable(converter, key)")) +public fun MutableMetaProvider.node(key: Name? = null, converter: MetaConverter): ReadWriteProperty = + convertable(converter, key) + +/** + * Use object serializer to transform it to Meta and back. + * Note that mutation of the object does not change Meta. + */ +@DFExperimental +public inline fun MutableMetaProvider.serializable( + descriptor: MetaDescriptor? = null, + key: Name? = null, +): ReadWriteProperty = convertable(MetaConverter.serializable(descriptor), key) + +/** + * Use [converter] to convert a list of same name siblings meta to object and back. + * Note that mutation of the object does not change Meta. + */ +public fun MutableMeta.listOfConvertable( + converter: MetaConverter, + key: Name? = null, +): ReadWriteProperty> = object : ReadWriteProperty> { + override fun getValue(thisRef: Any?, property: KProperty<*>): List { + val name = key ?: property.name.asName() + return getIndexed(name).values.map { converter.metaToObject(it) } + } + + override fun setValue(thisRef: Any?, property: KProperty<*>, value: List) { + val name = key ?: property.name.asName() + setIndexed(name, value.map { converter.objectToMeta(it) }) + } +} + +@DFExperimental +public inline fun MutableMeta.listOfSerializable( + descriptor: MetaDescriptor? = null, + key: Name? = null, +): ReadWriteProperty> = listOfConvertable(MetaConverter.serializable(descriptor), key) + + public fun MutableMetaProvider.value(key: Name? = null): ReadWriteProperty = object : ReadWriteProperty { override fun getValue(thisRef: Any?, property: KProperty<*>): Value? = @@ -45,7 +95,7 @@ public fun MutableMetaProvider.value(key: Name? = null): ReadWriteProperty MutableMetaProvider.value( key: Name? = null, writer: (T) -> Value? = { Value.of(it) }, - reader: (Value?) -> T + reader: (Value?) -> T, ): ReadWriteProperty = object : ReadWriteProperty { override fun getValue(thisRef: Any?, property: KProperty<*>): T = reader(get(key ?: property.name.asName())?.value) -- 2.34.1 From 6e20fc39299aeeb0af58691edd4c2a2a36b2e69e Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Thu, 28 Dec 2023 22:14:15 +0300 Subject: [PATCH 08/62] fix valueSequence --- CHANGELOG.md | 1 + build.gradle.kts | 2 +- .../commonMain/kotlin/space/kscience/dataforge/meta/Meta.kt | 4 +++- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7cc1c21d..8e2861e2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ ### Fixed - Partially fixed a bug with `MutableMeta` observable wrappers. +- `valueSequence` now include root value. So `meta.update` works properly. ### Security diff --git a/build.gradle.kts b/build.gradle.kts index 1fa05824..33be577c 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -8,7 +8,7 @@ plugins { allprojects { group = "space.kscience" - version = "0.7.2-dev-1" + version = "0.7.2-dev-2" } subprojects { diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/Meta.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/Meta.kt index 979c8782..5cf53c75 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/Meta.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/Meta.kt @@ -188,10 +188,12 @@ public operator fun > M?.get(key: String): M? = this?.get(key.p /** - * Get a sequence of [Name]-[Value] pairs using top-down traversal of the tree + * Get a sequence of [Name]-[Value] pairs using top-down traversal of the tree. + * The sequence includes root value with empty name */ public fun Meta.valueSequence(): Sequence> = sequence { items.forEach { (key, item) -> + value?.let { yield(Name.EMPTY to it) } item.value?.let { itemValue -> yield(key.asName() to itemValue) } -- 2.34.1 From 282b43a05a03fcfb9c78398ac31ebfce35ad64f0 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Sun, 31 Dec 2023 11:57:54 +0300 Subject: [PATCH 09/62] Clean up scheme and descriptor logic --- build.gradle.kts | 2 +- .../dataforge/descriptors/annotations.kt | 126 ------------- .../descriptors/reflectiveDescriptors.kt | 169 +++++++++++++----- .../descriptors/TestAutoDescriptors.kt | 29 +++ .../space/kscience/dataforge/io/Binary.kt | 5 + .../space/kscience/dataforge/meta/Scheme.kt | 69 ++++--- .../meta/descriptors/MetaDescriptorBuilder.kt | 143 ++++++++------- .../meta/descriptors/schemeDescriptor.kt | 4 +- .../kscience/dataforge/meta/JsonMetaTest.kt | 4 +- 9 files changed, 274 insertions(+), 277 deletions(-) delete mode 100644 dataforge-context/src/jvmMain/kotlin/space/kscience/dataforge/descriptors/annotations.kt create mode 100644 dataforge-context/src/jvmTest/kotlin/space/kscience/dataforge/descriptors/TestAutoDescriptors.kt diff --git a/build.gradle.kts b/build.gradle.kts index 33be577c..4a336e25 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -8,7 +8,7 @@ plugins { allprojects { group = "space.kscience" - version = "0.7.2-dev-2" + version = "0.8.0-dev-1" } subprojects { diff --git a/dataforge-context/src/jvmMain/kotlin/space/kscience/dataforge/descriptors/annotations.kt b/dataforge-context/src/jvmMain/kotlin/space/kscience/dataforge/descriptors/annotations.kt deleted file mode 100644 index f97cb28d..00000000 --- a/dataforge-context/src/jvmMain/kotlin/space/kscience/dataforge/descriptors/annotations.kt +++ /dev/null @@ -1,126 +0,0 @@ -/* - * Copyright 2018 Alexander Nozik. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package space.kscience.dataforge.descriptors - -//@MustBeDocumented -//annotation class Attribute( -// val key: String, -// val value: String -//) -// -//@MustBeDocumented -//annotation class Attributes( -// val attrs: Array -//) -// -//@MustBeDocumented -//annotation class ItemDef( -// val info: String = "", -// val multiple: Boolean = false, -// val required: Boolean = false -//) -// -//@Target(AnnotationTarget.PROPERTY) -//@MustBeDocumented -//annotation class ValueDef( -// val type: Array = [ValueType.STRING], -// val def: String = "", -// val allowed: Array = [], -// val enumeration: KClass<*> = Any::class -//) - -///** -// * Description text for meta property, node or whole object -// */ -//@Target(AnnotationTarget.CLASS, AnnotationTarget.PROPERTY) -//@Retention(AnnotationRetention.RUNTIME) -//@MustBeDocumented -//annotation class Description(val value: String) -// -///** -// * Annotation for value property which states that lists are expected -// */ -//@Target(AnnotationTarget.PROPERTY) -//@Retention(AnnotationRetention.RUNTIME) -//@MustBeDocumented -//annotation class Multiple -// -///** -// * Descriptor target -// * The DataForge path to the resource containing the description. Following targets are supported: -// * 1. resource -// * 1. file -// * 1. class -// * 1. method -// * 1. property -// * -// * -// * Does not work if [type] is provided -// */ -//@Target(AnnotationTarget.CLASS, AnnotationTarget.PROPERTY, AnnotationTarget.VALUE_PARAMETER) -//@Retention(AnnotationRetention.RUNTIME) -//@MustBeDocumented -//annotation class Descriptor(val value: String) -// -// -///** -// * Aggregator class for descriptor nodes -// */ -//@Target(AnnotationTarget.CLASS, AnnotationTarget.PROPERTY, AnnotationTarget.FUNCTION, AnnotationTarget.VALUE_PARAMETER) -//@Retention(AnnotationRetention.RUNTIME) -//@MustBeDocumented -//annotation class DescriptorNodes(vararg val nodes: NodeDef) -// -///** -// * Aggregator class for descriptor values -// */ -//@Target(AnnotationTarget.CLASS, AnnotationTarget.PROPERTY, AnnotationTarget.FUNCTION, AnnotationTarget.VALUE_PARAMETER) -//@Retention(AnnotationRetention.RUNTIME) -//@MustBeDocumented -//annotation class DescriptorValues(vararg val nodes: ValueDef) -// -///** -// * Alternative name for property descriptor declaration -// */ -//@Target(AnnotationTarget.PROPERTY, AnnotationTarget.VALUE_PARAMETER) -//@Retention(AnnotationRetention.RUNTIME) -//@MustBeDocumented -//annotation class DescriptorName(val name: String) -// -//@Target(AnnotationTarget.PROPERTY) -//@Retention(AnnotationRetention.RUNTIME) -//@MustBeDocumented -//annotation class DescriptorValue(val def: ValueDef) -////TODO enter fields directly? -// -//@Target(AnnotationTarget.PROPERTY) -//@Retention(AnnotationRetention.RUNTIME) -//@MustBeDocumented -//annotation class ValueProperty( -// val name: String = "", -// val type: Array = arrayOf(ValueType.STRING), -// val multiple: Boolean = false, -// val def: String = "", -// val enumeration: KClass<*> = Any::class, -// val tags: Array = emptyArray() -//) -// -// -//@Target(AnnotationTarget.PROPERTY) -//@Retention(AnnotationRetention.RUNTIME) -//@MustBeDocumented -//annotation class NodeProperty(val name: String = "") diff --git a/dataforge-context/src/jvmMain/kotlin/space/kscience/dataforge/descriptors/reflectiveDescriptors.kt b/dataforge-context/src/jvmMain/kotlin/space/kscience/dataforge/descriptors/reflectiveDescriptors.kt index 9de5e280..1f2db7fc 100644 --- a/dataforge-context/src/jvmMain/kotlin/space/kscience/dataforge/descriptors/reflectiveDescriptors.kt +++ b/dataforge-context/src/jvmMain/kotlin/space/kscience/dataforge/descriptors/reflectiveDescriptors.kt @@ -1,53 +1,124 @@ package space.kscience.dataforge.descriptors +import kotlinx.serialization.ExperimentalSerializationApi +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.decodeFromStream +import org.slf4j.LoggerFactory +import space.kscience.dataforge.meta.Scheme +import space.kscience.dataforge.meta.SchemeSpec +import space.kscience.dataforge.meta.ValueType +import space.kscience.dataforge.meta.descriptors.MetaDescriptor +import space.kscience.dataforge.meta.descriptors.MetaDescriptorBuilder +import space.kscience.dataforge.meta.descriptors.node +import java.net.URL +import kotlin.reflect.KClass +import kotlin.reflect.full.isSubclassOf +import kotlin.reflect.full.memberProperties +import kotlin.reflect.typeOf -//inline fun T.buildDescriptor(): NodeDescriptor = NodeDescriptor { -// T::class.apply { -// findAnnotation()?.let { def -> -// info = def.info -// required = def.required -// multiple = def.multiple -// } -// findAnnotation()?.let { attr -> -// attributes { -// this[attr.key] = attr.value.parseValue() -// } -// } -// findAnnotation()?.attrs?.forEach { attr -> -// attributes { -// this[attr.key] = attr.value.parseValue() -// } -// } -// } -// T::class.memberProperties.forEach { property -> -// val delegate = property.getDelegate(this@buildDescriptor) -// -// val descriptor: ItemDescriptor = when (delegate) { -// is ConfigurableDelegate -> buildPropertyDescriptor(property, delegate) -// is ReadWriteDelegateWrapper<*, *> -> { -// if (delegate.delegate is ConfigurableDelegate) { -// buildPropertyDescriptor(property, delegate.delegate as ConfigurableDelegate) -// } else { -// return@forEach -// } -// } -// else -> return@forEach -// } -// defineItem(property.name, descriptor) -// } -//} -//inline fun buildPropertyDescriptor( -// property: KProperty1, -// delegate: ConfigurableDelegate -//): ItemDescriptor { -// when { -// V::class.isSubclassOf(Scheme::class) -> NodeDescriptor { -// default = delegate.default.node -// } -// V::class.isSubclassOf(Meta::class) -> NodeDescriptor { -// default = delegate.default.node -// } -// -// } -//} +/** + * Description text for meta property, node or whole object + */ +@Target(AnnotationTarget.CLASS, AnnotationTarget.PROPERTY) +@Retention(AnnotationRetention.RUNTIME) +@MustBeDocumented +public annotation class Description(val value: String) + +@Target(AnnotationTarget.PROPERTY) +@Retention(AnnotationRetention.RUNTIME) +@MustBeDocumented +public annotation class Multiple() + +@Target(AnnotationTarget.CLASS, AnnotationTarget.PROPERTY) +@Retention(AnnotationRetention.RUNTIME) +@MustBeDocumented +public annotation class DescriptorResource(val resourceName: String) + +@Target(AnnotationTarget.CLASS, AnnotationTarget.PROPERTY) +@Retention(AnnotationRetention.RUNTIME) +@MustBeDocumented +public annotation class DescriptorUrl(val url: String) + + +@OptIn(ExperimentalSerializationApi::class) +private fun MetaDescriptorBuilder.loadDescriptorFromUrl(url: URL) { + url.openStream().use { + from(Json.decodeFromStream(MetaDescriptor.serializer(), it)) + } +} + +private fun MetaDescriptorBuilder.loadDescriptorFromResource(resource: DescriptorResource) { + val url = {}.javaClass.getResource(resource.resourceName) + if (url != null) { + loadDescriptorFromUrl(url) + } else { + LoggerFactory.getLogger("System") + .error("Can't find descriptor resource with name ${resource.resourceName}") + } +} + + +public fun MetaDescriptor.Companion.forClass( + kClass: KClass, + mod: MetaDescriptorBuilder.() -> Unit = {}, +): MetaDescriptor = MetaDescriptor { + when { + kClass.isSubclassOf(Number::class) -> valueType(ValueType.NUMBER) + kClass == String::class -> ValueType.STRING + kClass == Boolean::class -> ValueType.BOOLEAN + kClass == DoubleArray::class -> ValueType.LIST + } + + kClass.annotations.forEach { + when (it) { + is Description -> description = it.value + + is DescriptorResource -> loadDescriptorFromResource(it) + + is DescriptorUrl -> loadDescriptorFromUrl(URL(it.url)) + } + } + kClass.memberProperties.forEach { property -> + + var flag = false + + val descriptor = MetaDescriptor { + //use base type descriptor as a base + (property.returnType.classifier as? KClass<*>)?.let { + from(forClass(it)) + } + property.annotations.forEach { + when (it) { + is Description -> { + description = it.value + flag = true + } + + is Multiple -> { + multiple = true + flag = true + } + + is DescriptorResource -> { + loadDescriptorFromResource(it) + flag = true + } + + is DescriptorUrl -> { + loadDescriptorFromUrl(URL(it.url)) + flag = true + } + } + } + } + if (flag) { + node(property.name, descriptor) + } + } + mod() +} + +@Suppress("UNCHECKED_CAST") +public inline fun SchemeSpec.autoDescriptor( noinline mod: MetaDescriptorBuilder.() -> Unit = {}): MetaDescriptor = + MetaDescriptor.forClass(typeOf().classifier as KClass, mod) \ No newline at end of file diff --git a/dataforge-context/src/jvmTest/kotlin/space/kscience/dataforge/descriptors/TestAutoDescriptors.kt b/dataforge-context/src/jvmTest/kotlin/space/kscience/dataforge/descriptors/TestAutoDescriptors.kt new file mode 100644 index 00000000..c1271537 --- /dev/null +++ b/dataforge-context/src/jvmTest/kotlin/space/kscience/dataforge/descriptors/TestAutoDescriptors.kt @@ -0,0 +1,29 @@ +package space.kscience.dataforge.descriptors + +import kotlinx.serialization.encodeToString +import kotlinx.serialization.json.Json +import org.junit.jupiter.api.Test +import space.kscience.dataforge.meta.* +import space.kscience.dataforge.meta.descriptors.MetaDescriptor +import space.kscience.dataforge.meta.transformations.MetaConverter + +private class TestScheme: Scheme(){ + + @Description("A") + val a by string() + + @Description("B") + val b by int() + + companion object: SchemeSpec(::TestScheme){ + override val descriptor: MetaDescriptor = autoDescriptor() + } +} + +class TestAutoDescriptors { + @Test + fun autoDescriptor(){ + val autoDescriptor = MetaDescriptor.forClass(TestScheme::class) + println(Json{prettyPrint = true}.encodeToString(autoDescriptor)) + } +} \ No newline at end of file diff --git a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/Binary.kt b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/Binary.kt index f85dd98e..de365519 100644 --- a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/Binary.kt +++ b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/Binary.kt @@ -71,6 +71,11 @@ internal class ByteArrayBinary( override fun view(offset: Int, binarySize: Int): ByteArrayBinary = ByteArrayBinary(array, start + offset, binarySize) + + override fun toString(): String = + "ByteArrayBinary(array=$array, start=$start, size=$size)" + + } public fun ByteArray.asBinary(): Binary = ByteArrayBinary(this) diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/Scheme.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/Scheme.kt index 90473286..ab504733 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/Scheme.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/Scheme.kt @@ -15,28 +15,38 @@ import space.kscience.dataforge.names.* public open class Scheme : Described, MetaRepr, MutableMetaProvider, Configurable { /** - * Meta to be mutated by this schme + * Meta to be mutated by this scheme */ - private var targetMeta: MutableMeta = MutableMeta() + private var target: MutableMeta? = null + get() { + // automatic initialization of target if it is missing + if (field == null) { + field = MutableMeta() + } + return field + } /** * Default values provided by this scheme */ - private var defaultMeta: Meta? = null + private var prototype: Meta? = null final override val meta: ObservableMutableMeta = SchemeMeta(Name.EMPTY) final override var descriptor: MetaDescriptor? = null - internal set + private set - internal fun wrap( - newMeta: MutableMeta, - preserveDefault: Boolean = false, + /** + * This method must be called before the scheme could be used + */ + internal fun initialize( + target: MutableMeta, + prototype: Meta, + descriptor: MetaDescriptor?, ) { - if (preserveDefault) { - defaultMeta = targetMeta.seal() - } - targetMeta = newMeta + this.target = target + this.prototype = prototype + this.descriptor = descriptor } /** @@ -47,11 +57,11 @@ public open class Scheme : Described, MetaRepr, MutableMetaProvider, Configurabl return descriptor?.validate(meta) ?: true } - override fun get(name: Name): MutableMeta? = meta.get(name) + override fun get(name: Name): MutableMeta? = meta[name] override fun set(name: Name, node: Meta?) { if (validate(name, meta)) { - meta.set(name, node) + meta[name] = node } else { error("Validation failed for node $node at $name") } @@ -68,14 +78,16 @@ public open class Scheme : Described, MetaRepr, MutableMetaProvider, Configurabl private val listeners: MutableList = mutableListOf() + override fun toString(): String = meta.toString() + private inner class SchemeMeta(val pathName: Name) : ObservableMutableMeta { override var value: Value? - get() = targetMeta[pathName]?.value - ?: defaultMeta?.get(pathName)?.value + get() = target[pathName]?.value + ?: prototype?.get(pathName)?.value ?: descriptor?.get(pathName)?.defaultValue set(value) { - val oldValue = targetMeta[pathName]?.value - targetMeta[pathName] = value + val oldValue = target[pathName]?.value + target!![pathName] = value if (oldValue != value) { invalidate(Name.EMPTY) } @@ -83,8 +95,8 @@ public open class Scheme : Described, MetaRepr, MutableMetaProvider, Configurabl override val items: Map get() { - val targetKeys = targetMeta[pathName]?.items?.keys ?: emptySet() - val defaultKeys = defaultMeta?.get(pathName)?.items?.keys ?: emptySet() + val targetKeys = target[pathName]?.items?.keys ?: emptySet() + val defaultKeys = prototype?.get(pathName)?.items?.keys ?: emptySet() return (targetKeys + defaultKeys).associateWith { SchemeMeta(pathName + it) } } @@ -111,7 +123,7 @@ public open class Scheme : Described, MetaRepr, MutableMetaProvider, Configurabl override fun hashCode(): Int = Meta.hashCode(this) override fun set(name: Name, node: Meta?) { - targetMeta.set(name, node) + target!![name] = node invalidate(name) } @@ -119,7 +131,6 @@ public open class Scheme : Described, MetaRepr, MutableMetaProvider, Configurabl @DFExperimental override fun attach(name: Name, node: ObservableMutableMeta) { - //TODO implement zero-copy attachment set(name, node) node.onChange(this) { changeName -> set(name + changeName, this[changeName]) @@ -131,10 +142,11 @@ public open class Scheme : Described, MetaRepr, MutableMetaProvider, Configurabl /** * Relocate scheme target onto given [MutableMeta]. Old provider does not get updates anymore. - * Current state of the scheme used as a default. + * The Current state of the scheme that os used as a default. */ +@DFExperimental public fun T.retarget(provider: MutableMeta): T = apply { - wrap(provider, true) + initialize(provider, meta.seal(), descriptor) } /** @@ -155,19 +167,18 @@ public open class SchemeSpec( private val builder: () -> T, ) : Specification { + override val descriptor: MetaDescriptor? get() = null + override fun read(source: Meta): T = builder().also { - it.wrap(MutableMeta().withDefault(source)) + it.initialize(MutableMeta(), source, descriptor) } override fun write(target: MutableMeta): T = empty().also { - it.wrap(target) + it.initialize(target, Meta.EMPTY, descriptor) } - //TODO Generate descriptor from Scheme class - override val descriptor: MetaDescriptor? get() = null - override fun empty(): T = builder().also { - it.descriptor = descriptor + it.initialize(MutableMeta(), Meta.EMPTY, descriptor) } @Suppress("OVERRIDE_BY_INLINE") diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/descriptors/MetaDescriptorBuilder.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/descriptors/MetaDescriptorBuilder.kt index 95949d03..751d2525 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/descriptors/MetaDescriptorBuilder.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/descriptors/MetaDescriptorBuilder.kt @@ -44,38 +44,27 @@ public class MetaDescriptorBuilder @PublishedApi internal constructor() { attributes.apply(block) } - public fun item(name: Name, block: MetaDescriptorBuilder.() -> Unit = {}): MetaDescriptorBuilder { - return when (name.length) { - 0 -> apply(block) + internal fun node( + name: Name, + descriptorBuilder: MetaDescriptorBuilder, + ): Unit { + when (name.length) { + 0 -> error("Can't set descriptor to root") 1 -> { - val target = MetaDescriptorBuilder().apply(block) - children[name.first().body] = target - target + children[name.first().body] = descriptorBuilder } - else -> { - children.getOrPut(name.first().body) { MetaDescriptorBuilder() }.item(name.cutFirst(), block) - } + else -> children.getOrPut(name.first().body) { + MetaDescriptorBuilder() + }.node(name.cutFirst(), descriptorBuilder) } } - public fun node( + internal fun node( name: Name, - descriptor: MetaDescriptor, - block: MetaDescriptorBuilder.() -> Unit = {}, - ): MetaDescriptorBuilder = when (name.length) { - 0 -> error("Can't set descriptor to root") - 1 -> { - val item = descriptor.toBuilder().apply { - valueRestriction = ValueRestriction.ABSENT - }.apply(block) - children[name.first().body] = item - item - } - - else -> children.getOrPut(name.first().body) { - MetaDescriptorBuilder() - }.node(name.cutFirst(), descriptor, block) + descriptorBuilder: MetaDescriptor, + ): Unit { + node(name, descriptorBuilder.toBuilder()) } public var allowedValues: List @@ -89,6 +78,17 @@ public class MetaDescriptorBuilder @PublishedApi internal constructor() { allowedValues = values.map { Value.of(it) } } + public fun from(descriptor: MetaDescriptor) { + description = descriptor.description + children.putAll(descriptor.children.mapValues { it.value.toBuilder() }) + multiple = descriptor.multiple + valueRestriction = descriptor.valueRestriction + valueTypes = descriptor.valueTypes + indexKey = descriptor.indexKey + default = descriptor.defaultValue + attributes.update(descriptor.attributes) + } + @PublishedApi internal fun build(): MetaDescriptor = MetaDescriptor( description = description, @@ -102,40 +102,27 @@ public class MetaDescriptorBuilder @PublishedApi internal constructor() { ) } -public fun MetaDescriptorBuilder.item(name: String, block: MetaDescriptorBuilder.() -> Unit): MetaDescriptorBuilder = - item(Name.parse(name), block) +//public fun MetaDescriptorBuilder.item(name: String, block: MetaDescriptorBuilder.() -> Unit): MetaDescriptorBuilder = +// item(Name.parse(name), block) public inline fun MetaDescriptor(block: MetaDescriptorBuilder.() -> Unit): MetaDescriptor = MetaDescriptorBuilder().apply(block).build() /** - * Create and configure child value descriptor - */ -public fun MetaDescriptorBuilder.value( - name: Name, - type: ValueType, - vararg additionalTypes: ValueType, - block: MetaDescriptorBuilder.() -> Unit = {}, -): MetaDescriptorBuilder = item(name) { - valueType(type, *additionalTypes) - block() -} - -public fun MetaDescriptorBuilder.value( - name: String, - type: ValueType, - vararg additionalTypes: ValueType, - block: MetaDescriptorBuilder.() -> Unit = {}, -): MetaDescriptorBuilder = value(Name.parse(name), type, additionalTypes = additionalTypes, block) - -/** - * Create and configure child value descriptor + * Create and configure child node descriptor */ public fun MetaDescriptorBuilder.node( - name: Name, block: MetaDescriptorBuilder.() -> Unit, -): MetaDescriptorBuilder = item(name) { - valueRestriction = ValueRestriction.ABSENT - block() + name: Name, + block: MetaDescriptorBuilder.() -> Unit, +) { + node( + name, + MetaDescriptorBuilder().apply(block) + ) +} + +public fun MetaDescriptorBuilder.node(name: String, descriptor: MetaDescriptor) { + node(Name.parse(name), descriptor) } public fun MetaDescriptorBuilder.node(name: String, block: MetaDescriptorBuilder.() -> Unit) { @@ -144,30 +131,16 @@ public fun MetaDescriptorBuilder.node(name: String, block: MetaDescriptorBuilder public fun MetaDescriptorBuilder.node( key: String, - described: Described, + base: Described, block: MetaDescriptorBuilder.() -> Unit = {}, ) { - described.descriptor?.let { - node(Name.parse(key), it, block) - } + node(Name.parse(key), base.descriptor?.toBuilder()?.apply(block) ?: MetaDescriptorBuilder()) } public fun MetaDescriptorBuilder.required() { valueRestriction = ValueRestriction.REQUIRED } -public inline fun > MetaDescriptorBuilder.enum( - key: Name, - default: E?, - crossinline modifier: MetaDescriptorBuilder.() -> Unit = {}, -): MetaDescriptorBuilder = value(key, ValueType.STRING) { - default?.let { - this.default = default.asValue() - } - allowedValues = enumValues().map { it.asValue() } - modifier() -} - private fun MetaDescriptor.toBuilder(): MetaDescriptorBuilder = MetaDescriptorBuilder().apply { description = this@toBuilder.description children = this@toBuilder.children.mapValuesTo(LinkedHashMap()) { it.value.toBuilder() } @@ -179,6 +152,40 @@ private fun MetaDescriptor.toBuilder(): MetaDescriptorBuilder = MetaDescriptorBu attributes = this@toBuilder.attributes.toMutableMeta() } + +/** + * Create and configure child value descriptor + */ +public fun MetaDescriptorBuilder.value( + name: Name, + type: ValueType, + vararg additionalTypes: ValueType, + block: MetaDescriptorBuilder.() -> Unit = {}, +): Unit = node(name) { + valueType(type, *additionalTypes) + block() +} + +public fun MetaDescriptorBuilder.value( + name: String, + type: ValueType, + vararg additionalTypes: ValueType, + block: MetaDescriptorBuilder.() -> Unit = {}, +): Unit = value(Name.parse(name), type, additionalTypes = additionalTypes, block) + + +public inline fun > MetaDescriptorBuilder.enum( + key: Name, + default: E?, + crossinline modifier: MetaDescriptorBuilder.() -> Unit = {}, +): Unit = value(key, ValueType.STRING) { + default?.let { + this.default = default.asValue() + } + allowedValues = enumValues().map { it.asValue() } + modifier() +} + /** * Make a deep copy of this descriptor applying given transformation [block] */ diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/descriptors/schemeDescriptor.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/descriptors/schemeDescriptor.kt index 79fb6fdb..803659a6 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/descriptors/schemeDescriptor.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/descriptors/schemeDescriptor.kt @@ -9,7 +9,7 @@ import kotlin.reflect.typeOf public inline fun MetaDescriptorBuilder.value( property: KProperty1, noinline block: MetaDescriptorBuilder.() -> Unit = {}, -): MetaDescriptorBuilder = when (typeOf()) { +): Unit = when (typeOf()) { typeOf(), typeOf(), typeOf(), typeOf(), typeOf(), typeOf() -> value(property.name, ValueType.NUMBER) { block() @@ -34,7 +34,7 @@ public inline fun MetaDescriptorBuilder.value( multiple = true block() } - else -> item(property.name, block) + else -> node(property.name, block) } public inline fun MetaDescriptorBuilder.scheme( diff --git a/dataforge-meta/src/commonTest/kotlin/space/kscience/dataforge/meta/JsonMetaTest.kt b/dataforge-meta/src/commonTest/kotlin/space/kscience/dataforge/meta/JsonMetaTest.kt index fba9b596..2ada3ade 100644 --- a/dataforge-meta/src/commonTest/kotlin/space/kscience/dataforge/meta/JsonMetaTest.kt +++ b/dataforge-meta/src/commonTest/kotlin/space/kscience/dataforge/meta/JsonMetaTest.kt @@ -2,7 +2,7 @@ package space.kscience.dataforge.meta import kotlinx.serialization.json.* import space.kscience.dataforge.meta.descriptors.MetaDescriptor -import space.kscience.dataforge.meta.descriptors.item +import space.kscience.dataforge.meta.descriptors.node import kotlin.test.Test import kotlin.test.assertEquals @@ -32,7 +32,7 @@ class JsonMetaTest { } val descriptor = MetaDescriptor { - item("nodeArray") { + node("nodeArray") { indexKey = "index" } } -- 2.34.1 From fd1d98aa87ff1f7b013df876e2a0ac36a87a39c0 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Sun, 31 Dec 2023 12:54:38 +0300 Subject: [PATCH 10/62] Merge MetaConverter and MetaSpec --- CHANGELOG.md | 1 + .../dataforge/properties/MetaProperty.kt | 12 +- .../descriptors/TestAutoDescriptors.kt | 6 +- .../space/kscience/dataforge/meta/JsonMeta.kt | 2 +- .../{transformations => }/MetaConverter.kt | 74 +++++----- .../kscience/dataforge/meta/MetaDelegate.kt | 5 +- .../space/kscience/dataforge/meta/MetaSpec.kt | 18 +++ .../MetaTransformation.kt | 3 +- .../dataforge/meta/MutableMetaDelegate.kt | 9 +- .../space/kscience/dataforge/meta/Scheme.kt | 119 ++++++++++++++- .../kscience/dataforge/meta/Specification.kt | 137 ------------------ .../meta/descriptors/MetaDescriptor.kt | 15 +- .../meta/descriptors/MetaDescriptorBuilder.kt | 6 +- .../dataforge/meta/MetaDelegateTest.kt | 2 +- .../kscience/dataforge/workspace/Task.kt | 25 ++-- .../dataforge/workspace/WorkspaceBuilder.kt | 4 +- 16 files changed, 209 insertions(+), 229 deletions(-) rename dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/{transformations => }/MetaConverter.kt (61%) create mode 100644 dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MetaSpec.kt rename dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/{transformations => }/MetaTransformation.kt (98%) delete mode 100644 dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/Specification.kt diff --git a/CHANGELOG.md b/CHANGELOG.md index 8e2861e2..d6daac6c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ - Add Meta and MutableMeta delegates for convertable and serializeable ### Changed +- Descriptor `children` renamed to `nodes` ### Deprecated - `node(key,converter)` in favor of `serializable` delegate diff --git a/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/properties/MetaProperty.kt b/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/properties/MetaProperty.kt index e79ce931..85435602 100644 --- a/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/properties/MetaProperty.kt +++ b/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/properties/MetaProperty.kt @@ -1,11 +1,7 @@ package space.kscience.dataforge.properties -import space.kscience.dataforge.meta.Meta -import space.kscience.dataforge.meta.ObservableMutableMeta -import space.kscience.dataforge.meta.transformations.MetaConverter -import space.kscience.dataforge.meta.transformations.nullableMetaToObject -import space.kscience.dataforge.meta.transformations.nullableObjectToMeta +import space.kscience.dataforge.meta.* import space.kscience.dataforge.misc.DFExperimental import space.kscience.dataforge.names.Name import space.kscience.dataforge.names.startsWith @@ -18,14 +14,14 @@ public class MetaProperty( ) : Property { override var value: T? - get() = converter.nullableMetaToObject(meta[name]) + get() = converter.readNullable(meta[name]) set(value) { - meta[name] = converter.nullableObjectToMeta(value) ?: Meta.EMPTY + meta[name] = converter.convertNullable(value) ?: Meta.EMPTY } override fun onChange(owner: Any?, callback: (T?) -> Unit) { meta.onChange(owner) { name -> - if (name.startsWith(this@MetaProperty.name)) callback(converter.nullableMetaToObject(this[name])) + if (name.startsWith(this@MetaProperty.name)) callback(converter.readNullable(this[name])) } } diff --git a/dataforge-context/src/jvmTest/kotlin/space/kscience/dataforge/descriptors/TestAutoDescriptors.kt b/dataforge-context/src/jvmTest/kotlin/space/kscience/dataforge/descriptors/TestAutoDescriptors.kt index c1271537..617e85cc 100644 --- a/dataforge-context/src/jvmTest/kotlin/space/kscience/dataforge/descriptors/TestAutoDescriptors.kt +++ b/dataforge-context/src/jvmTest/kotlin/space/kscience/dataforge/descriptors/TestAutoDescriptors.kt @@ -3,9 +3,11 @@ package space.kscience.dataforge.descriptors import kotlinx.serialization.encodeToString import kotlinx.serialization.json.Json import org.junit.jupiter.api.Test -import space.kscience.dataforge.meta.* +import space.kscience.dataforge.meta.Scheme +import space.kscience.dataforge.meta.SchemeSpec import space.kscience.dataforge.meta.descriptors.MetaDescriptor -import space.kscience.dataforge.meta.transformations.MetaConverter +import space.kscience.dataforge.meta.int +import space.kscience.dataforge.meta.string private class TestScheme: Scheme(){ diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/JsonMeta.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/JsonMeta.kt index 36373582..8da8b2d3 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/JsonMeta.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/JsonMeta.kt @@ -31,7 +31,7 @@ private fun Meta.toJsonWithIndex(descriptor: MetaDescriptor?, index: String?): J val pairs: MutableList> = items.entries.groupBy { it.key.body }.mapTo(ArrayList()) { (body, list) -> - val childDescriptor = descriptor?.children?.get(body) + val childDescriptor = descriptor?.nodes?.get(body) if (list.size == 1) { val (token, element) = list.first() //do not add an empty element diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/transformations/MetaConverter.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MetaConverter.kt similarity index 61% rename from dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/transformations/MetaConverter.kt rename to dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MetaConverter.kt index 17774913..0da36d17 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/transformations/MetaConverter.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MetaConverter.kt @@ -1,19 +1,19 @@ -package space.kscience.dataforge.meta.transformations +package space.kscience.dataforge.meta import kotlinx.serialization.KSerializer import kotlinx.serialization.json.Json import kotlinx.serialization.json.encodeToJsonElement import kotlinx.serialization.serializer -import space.kscience.dataforge.meta.* import space.kscience.dataforge.meta.descriptors.MetaDescriptor import space.kscience.dataforge.misc.DFExperimental import kotlin.reflect.KType import kotlin.reflect.typeOf + /** * A converter of generic object to and from [Meta] */ -public interface MetaConverter { +public interface MetaConverter: MetaSpec { /** * Runtime type of [T] @@ -23,32 +23,32 @@ public interface MetaConverter { /** * A descriptor for resulting meta */ - public val descriptor: MetaDescriptor get() = MetaDescriptor.EMPTY + override val descriptor: MetaDescriptor get() = MetaDescriptor.EMPTY /** - * Attempt conversion of [meta] to an object or return null if conversion failed + * Attempt conversion of [source] to an object or return null if conversion failed */ - public fun metaToObjectOrNull(meta: Meta): T? + override fun readOrNull(source: Meta): T? - public fun metaToObject(meta: Meta): T = - metaToObjectOrNull(meta) ?: error("Meta $meta could not be interpreted by $this") + override fun read(source: Meta): T = + readOrNull(source) ?: error("Meta $source could not be interpreted by $this") - public fun objectToMeta(obj: T): Meta + public fun convert(obj: T): Meta public companion object { public val meta: MetaConverter = object : MetaConverter { override val type: KType = typeOf() - override fun metaToObjectOrNull(meta: Meta): Meta = meta - override fun objectToMeta(obj: Meta): Meta = obj + override fun readOrNull(source: Meta): Meta = source + override fun convert(obj: Meta): Meta = obj } public val value: MetaConverter = object : MetaConverter { override val type: KType = typeOf() - override fun metaToObjectOrNull(meta: Meta): Value? = meta.value - override fun objectToMeta(obj: Value): Meta = Meta(obj) + override fun readOrNull(source: Meta): Value? = source.value + override fun convert(obj: Value): Meta = Meta(obj) } public val string: MetaConverter = object : MetaConverter { @@ -59,8 +59,8 @@ public interface MetaConverter { } - override fun metaToObjectOrNull(meta: Meta): String? = meta.string - override fun objectToMeta(obj: String): Meta = Meta(obj.asValue()) + override fun readOrNull(source: Meta): String? = source.string + override fun convert(obj: String): Meta = Meta(obj.asValue()) } public val boolean: MetaConverter = object : MetaConverter { @@ -70,8 +70,8 @@ public interface MetaConverter { valueType(ValueType.BOOLEAN) } - override fun metaToObjectOrNull(meta: Meta): Boolean? = meta.boolean - override fun objectToMeta(obj: Boolean): Meta = Meta(obj.asValue()) + override fun readOrNull(source: Meta): Boolean? = source.boolean + override fun convert(obj: Boolean): Meta = Meta(obj.asValue()) } public val number: MetaConverter = object : MetaConverter { @@ -81,8 +81,8 @@ public interface MetaConverter { valueType(ValueType.NUMBER) } - override fun metaToObjectOrNull(meta: Meta): Number? = meta.number - override fun objectToMeta(obj: Number): Meta = Meta(obj.asValue()) + override fun readOrNull(source: Meta): Number? = source.number + override fun convert(obj: Number): Meta = Meta(obj.asValue()) } public val double: MetaConverter = object : MetaConverter { @@ -92,8 +92,8 @@ public interface MetaConverter { valueType(ValueType.NUMBER) } - override fun metaToObjectOrNull(meta: Meta): Double? = meta.double - override fun objectToMeta(obj: Double): Meta = Meta(obj.asValue()) + override fun readOrNull(source: Meta): Double? = source.double + override fun convert(obj: Double): Meta = Meta(obj.asValue()) } public val float: MetaConverter = object : MetaConverter { @@ -103,8 +103,8 @@ public interface MetaConverter { valueType(ValueType.NUMBER) } - override fun metaToObjectOrNull(meta: Meta): Float? = meta.float - override fun objectToMeta(obj: Float): Meta = Meta(obj.asValue()) + override fun readOrNull(source: Meta): Float? = source.float + override fun convert(obj: Float): Meta = Meta(obj.asValue()) } public val int: MetaConverter = object : MetaConverter { @@ -114,8 +114,8 @@ public interface MetaConverter { valueType(ValueType.NUMBER) } - override fun metaToObjectOrNull(meta: Meta): Int? = meta.int - override fun objectToMeta(obj: Int): Meta = Meta(obj.asValue()) + override fun readOrNull(source: Meta): Int? = source.int + override fun convert(obj: Int): Meta = Meta(obj.asValue()) } public val long: MetaConverter = object : MetaConverter { @@ -125,8 +125,8 @@ public interface MetaConverter { valueType(ValueType.NUMBER) } - override fun metaToObjectOrNull(meta: Meta): Long? = meta.long - override fun objectToMeta(obj: Long): Meta = Meta(obj.asValue()) + override fun readOrNull(source: Meta): Long? = source.long + override fun convert(obj: Long): Meta = Meta(obj.asValue()) } public inline fun > enum(): MetaConverter = object : MetaConverter { @@ -138,9 +138,9 @@ public interface MetaConverter { } @Suppress("USELESS_CAST") - override fun metaToObjectOrNull(meta: Meta): E = meta.enum() as? E ?: error("The Item is not a Enum") + override fun readOrNull(source: Meta): E = source.enum() as? E ?: error("The Item is not a Enum") - override fun objectToMeta(obj: E): Meta = Meta(obj.asValue()) + override fun convert(obj: E): Meta = Meta(obj.asValue()) } public fun valueList( @@ -153,9 +153,9 @@ public interface MetaConverter { valueType(ValueType.LIST) } - override fun metaToObjectOrNull(meta: Meta): List? = meta.value?.list?.map(reader) + override fun readOrNull(source: Meta): List? = source.value?.list?.map(reader) - override fun objectToMeta(obj: List): Meta = Meta(obj.map(writer).asValue()) + override fun convert(obj: List): Meta = Meta(obj.map(writer).asValue()) } /** @@ -168,12 +168,12 @@ public interface MetaConverter { override val type: KType = typeOf() private val serializer: KSerializer = serializer() - override fun metaToObjectOrNull(meta: Meta): T? { - val json = meta.toJson(descriptor) + override fun readOrNull(source: Meta): T? { + val json = source.toJson(descriptor) return Json.decodeFromJsonElement(serializer, json) } - override fun objectToMeta(obj: T): Meta { + override fun convert(obj: T): Meta { val json = Json.encodeToJsonElement(obj) return json.toMeta(descriptor) } @@ -183,7 +183,7 @@ public interface MetaConverter { } } -public fun MetaConverter.nullableMetaToObject(item: Meta?): T? = item?.let { metaToObject(it) } -public fun MetaConverter.nullableObjectToMeta(obj: T?): Meta? = obj?.let { objectToMeta(it) } +public fun MetaConverter.readNullable(item: Meta?): T? = item?.let { read(it) } +public fun MetaConverter.convertNullable(obj: T?): Meta? = obj?.let { convert(it) } -public fun MetaConverter.valueToObject(value: Value): T? = metaToObject(Meta(value)) +public fun MetaConverter.readValue(value: Value): T? = read(Meta(value)) diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MetaDelegate.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MetaDelegate.kt index 42e4fe14..70ff460d 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MetaDelegate.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MetaDelegate.kt @@ -1,7 +1,6 @@ package space.kscience.dataforge.meta import space.kscience.dataforge.meta.descriptors.MetaDescriptor -import space.kscience.dataforge.meta.transformations.MetaConverter import space.kscience.dataforge.misc.DFExperimental import space.kscience.dataforge.names.Name import space.kscience.dataforge.names.asName @@ -20,7 +19,7 @@ public fun MetaProvider.convertable( converter: MetaConverter, key: Name? = null, ): ReadOnlyProperty = ReadOnlyProperty { _, property -> - get(key ?: property.name.asName())?.let { converter.metaToObject(it) } + get(key ?: property.name.asName())?.let { converter.read(it) } } /** @@ -46,7 +45,7 @@ public fun Meta.listOfConvertable( key: Name? = null, ): ReadOnlyProperty> = ReadOnlyProperty{_, property -> val name = key ?: property.name.asName() - getIndexed(name).values.map { converter.metaToObject(it) } + getIndexed(name).values.map { converter.read(it) } } @DFExperimental diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MetaSpec.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MetaSpec.kt new file mode 100644 index 00000000..be21cef7 --- /dev/null +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MetaSpec.kt @@ -0,0 +1,18 @@ +package space.kscience.dataforge.meta + +import space.kscience.dataforge.meta.descriptors.Described + +public interface MetaSpec : Described { + + /** + * Read the source meta into an object and return null if Meta could not be interpreted as a target type + */ + public fun readOrNull(source: Meta): T? + + /** + * Read generic read-only meta with this [MetaSpec] producing instance of the desired type. + * Throws an error if conversion could not be done. + */ + public fun read(source: Meta): T = readOrNull(source) ?: error("Meta $source could not be interpreted by $this") +} + diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/transformations/MetaTransformation.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MetaTransformation.kt similarity index 98% rename from dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/transformations/MetaTransformation.kt rename to dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MetaTransformation.kt index d41365a6..dbf94f0f 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/transformations/MetaTransformation.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MetaTransformation.kt @@ -1,6 +1,5 @@ -package space.kscience.dataforge.meta.transformations +package space.kscience.dataforge.meta -import space.kscience.dataforge.meta.* import space.kscience.dataforge.misc.DFExperimental import space.kscience.dataforge.names.Name import kotlin.jvm.JvmInline diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MutableMetaDelegate.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MutableMetaDelegate.kt index d77348dd..f6e96109 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MutableMetaDelegate.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MutableMetaDelegate.kt @@ -1,7 +1,6 @@ package space.kscience.dataforge.meta import space.kscience.dataforge.meta.descriptors.MetaDescriptor -import space.kscience.dataforge.meta.transformations.MetaConverter import space.kscience.dataforge.misc.DFExperimental import space.kscience.dataforge.names.Name import space.kscience.dataforge.names.asName @@ -33,12 +32,12 @@ public fun MutableMetaProvider.convertable( object : ReadWriteProperty { override fun getValue(thisRef: Any?, property: KProperty<*>): T? { val name = key ?: property.name.asName() - return get(name)?.let { converter.metaToObject(it) } + return get(name)?.let { converter.read(it) } } override fun setValue(thisRef: Any?, property: KProperty<*>, value: T?) { val name = key ?: property.name.asName() - set(name, value?.let { converter.objectToMeta(it) }) + set(name, value?.let { converter.convert(it) }) } } @@ -66,12 +65,12 @@ public fun MutableMeta.listOfConvertable( ): ReadWriteProperty> = object : ReadWriteProperty> { override fun getValue(thisRef: Any?, property: KProperty<*>): List { val name = key ?: property.name.asName() - return getIndexed(name).values.map { converter.metaToObject(it) } + return getIndexed(name).values.map { converter.read(it) } } override fun setValue(thisRef: Any?, property: KProperty<*>, value: List) { val name = key ?: property.name.asName() - setIndexed(name, value.map { converter.objectToMeta(it) }) + setIndexed(name, value.map { converter.convert(it) }) } } diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/Scheme.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/Scheme.kt index ab504733..ff8d8095 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/Scheme.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/Scheme.kt @@ -7,9 +7,11 @@ import space.kscience.dataforge.meta.descriptors.validate import space.kscience.dataforge.misc.DFExperimental import space.kscience.dataforge.misc.ThreadSafe import space.kscience.dataforge.names.* +import kotlin.properties.ReadWriteProperty +import kotlin.reflect.KProperty /** - * A base for delegate-based or descriptor-based scheme. [Scheme] has an empty constructor to simplify usage from [Specification]. + * A base for delegate-based or descriptor-based scheme. [Scheme] has an empty constructor to simplify usage from [MetaSpec]. * Default item provider and [MetaDescriptor] are optional */ public open class Scheme : Described, MetaRepr, MutableMetaProvider, Configurable { @@ -165,23 +167,124 @@ public inline fun T.copy(spec: SchemeSpec, block: T.() -> Unit = */ public open class SchemeSpec( private val builder: () -> T, -) : Specification { +) : MetaSpec { override val descriptor: MetaDescriptor? get() = null - override fun read(source: Meta): T = builder().also { + override fun readOrNull(source: Meta): T = builder().also { it.initialize(MutableMeta(), source, descriptor) } - override fun write(target: MutableMeta): T = empty().also { + public fun write(target: MutableMeta): T = empty().also { it.initialize(target, Meta.EMPTY, descriptor) } - override fun empty(): T = builder().also { + /** + * Generate an empty object + */ + public fun empty(): T = builder().also { it.initialize(MutableMeta(), Meta.EMPTY, descriptor) } - @Suppress("OVERRIDE_BY_INLINE") - final override inline operator fun invoke(action: T.() -> Unit): T = empty().apply(action) + /** + * A convenience method to use specifications in builders + */ + public inline operator fun invoke(action: T.() -> Unit): T = empty().apply(action) -} \ No newline at end of file +} + + + +/** + * Update a [MutableMeta] using given specification + */ +public fun MutableMeta.updateWith( + spec: SchemeSpec, + action: T.() -> Unit, +): T = spec.write(this).apply(action) + + +/** + * Update configuration using given specification + */ +public fun Configurable.updateWith( + spec: SchemeSpec, + action: T.() -> Unit, +): T = spec.write(meta).apply(action) + + +/** + * A delegate that uses a [MetaSpec] to wrap a child of this provider + */ +public fun MutableMeta.scheme( + spec: SchemeSpec, + key: Name? = null, +): ReadWriteProperty = object : ReadWriteProperty { + override fun getValue(thisRef: Any?, property: KProperty<*>): T { + val name = key ?: property.name.asName() + return spec.write(getOrCreate(name)) + } + + override fun setValue(thisRef: Any?, property: KProperty<*>, value: T) { + val name = key ?: property.name.asName() + set(name, value.toMeta()) + } +} + +public fun Scheme.scheme( + spec: SchemeSpec, + key: Name? = null, +): ReadWriteProperty = meta.scheme(spec, key) + +/** + * A delegate that uses a [MetaSpec] to wrap a child of this provider. + * Returns null if meta with given name does not exist. + */ +public fun MutableMeta.schemeOrNull( + spec: SchemeSpec, + key: Name? = null, +): ReadWriteProperty = object : ReadWriteProperty { + override fun getValue(thisRef: Any?, property: KProperty<*>): T? { + val name = key ?: property.name.asName() + return if (get(name) == null) null else spec.write(getOrCreate(name)) + } + + override fun setValue(thisRef: Any?, property: KProperty<*>, value: T?) { + val name = key ?: property.name.asName() + if (value == null) remove(name) + else set(name, value.toMeta()) + } +} + +public fun Scheme.schemeOrNull( + spec: SchemeSpec, + key: Name? = null, +): ReadWriteProperty = meta.schemeOrNull(spec, key) + +/** + * A delegate that uses a [MetaSpec] to wrap a list of child providers. + * If children are mutable, the changes in list elements are reflected on them. + * The list is a snapshot of children state, so change in structure is not reflected on its composition. + */ +@DFExperimental +public fun MutableMeta.listOfScheme( + spec: SchemeSpec, + key: Name? = null, +): ReadWriteProperty> = object : ReadWriteProperty> { + override fun getValue(thisRef: Any?, property: KProperty<*>): List { + val name = key ?: property.name.asName() + return getIndexed(name).values.map { spec.write(it as MutableMeta) } + } + + override fun setValue(thisRef: Any?, property: KProperty<*>, value: List) { + val name = key ?: property.name.asName() + setIndexed(name, value.map { it.toMeta() }) + } +} + + +@DFExperimental +public fun Scheme.listOfScheme( + spec: SchemeSpec, + key: Name? = null, +): ReadWriteProperty> = meta.listOfScheme(spec, key) diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/Specification.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/Specification.kt deleted file mode 100644 index 7f3dd2a6..00000000 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/Specification.kt +++ /dev/null @@ -1,137 +0,0 @@ -package space.kscience.dataforge.meta - -import space.kscience.dataforge.meta.descriptors.Described -import space.kscience.dataforge.misc.DFExperimental -import space.kscience.dataforge.names.Name -import space.kscience.dataforge.names.asName -import kotlin.properties.ReadWriteProperty -import kotlin.reflect.KProperty - -public interface ReadOnlySpecification : Described { - - /** - * Read generic read-only meta with this [Specification] producing instance of desired type. - * The source is not mutated even if it is in theory mutable - */ - public fun read(source: Meta): T - - /** - * Generate an empty object - */ - public fun empty(): T - - /** - * A convenience method to use specifications in builders - */ - public operator fun invoke(action: T.() -> Unit): T = empty().apply(action) -} - - -/** - * Allows to apply custom configuration in a type safe way to simple untyped configuration. - * By convention [Scheme] companion should inherit this class - * - */ -public interface Specification : ReadOnlySpecification { - /** - * Wrap [MutableMeta], using it as inner storage (changes to [Specification] are reflected on [MutableMeta] - */ - public fun write(target: MutableMeta): T -} - -/** - * Update a [MutableMeta] using given specification - */ -public fun MutableMeta.updateWith( - spec: Specification, - action: T.() -> Unit, -): T = spec.write(this).apply(action) - - -/** - * Update configuration using given specification - */ -public fun Configurable.updateWith( - spec: Specification, - action: T.() -> Unit, -): T = spec.write(meta).apply(action) - -// -//public fun > MutableMeta.withSpec(spec: Specification): M? = -// spec.write(it) - -/** - * A delegate that uses a [Specification] to wrap a child of this provider - */ -public fun MutableMeta.spec( - spec: Specification, - key: Name? = null, -): ReadWriteProperty = object : ReadWriteProperty { - override fun getValue(thisRef: Any?, property: KProperty<*>): T { - val name = key ?: property.name.asName() - return spec.write(getOrCreate(name)) - } - - override fun setValue(thisRef: Any?, property: KProperty<*>, value: T) { - val name = key ?: property.name.asName() - set(name, value.toMeta()) - } -} - -public fun Scheme.spec( - spec: Specification, - key: Name? = null, -): ReadWriteProperty = meta.spec(spec, key) - -/** - * A delegate that uses a [Specification] to wrap a child of this provider. - * Returns null if meta with given name does not exist. - */ -public fun MutableMeta.specOrNull( - spec: Specification, - key: Name? = null, -): ReadWriteProperty = object : ReadWriteProperty { - override fun getValue(thisRef: Any?, property: KProperty<*>): T? { - val name = key ?: property.name.asName() - return if (get(name) == null) null else spec.write(getOrCreate(name)) - } - - override fun setValue(thisRef: Any?, property: KProperty<*>, value: T?) { - val name = key ?: property.name.asName() - if (value == null) remove(name) - else set(name, value.toMeta()) - } -} - -public fun Scheme.specOrNull( - spec: Specification, - key: Name? = null, -): ReadWriteProperty = meta.specOrNull(spec, key) - -/** - * A delegate that uses a [Specification] to wrap a list of child providers. - * If children are mutable, the changes in list elements are reflected on them. - * The list is a snapshot of children state, so change in structure is not reflected on its composition. - */ -@DFExperimental -public fun MutableMeta.listOfSpec( - spec: Specification, - key: Name? = null, -): ReadWriteProperty> = object : ReadWriteProperty> { - override fun getValue(thisRef: Any?, property: KProperty<*>): List { - val name = key ?: property.name.asName() - return getIndexed(name).values.map { spec.write(it as MutableMeta) } - } - - override fun setValue(thisRef: Any?, property: KProperty<*>, value: List) { - val name = key ?: property.name.asName() - setIndexed(name, value.map { it.toMeta() }) - } -} - - -@DFExperimental -public fun Scheme.listOfSpec( - spec: Specification, - key: Name? = null, -): ReadWriteProperty> = meta.listOfSpec(spec, key) diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/descriptors/MetaDescriptor.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/descriptors/MetaDescriptor.kt index 742b89ed..12bbd5d4 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/descriptors/MetaDescriptor.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/descriptors/MetaDescriptor.kt @@ -27,7 +27,7 @@ public enum class ValueRestriction { /** * The descriptor for a meta * @param description description text - * @param children child descriptors for this node + * @param nodes child descriptors for this node * @param multiple True if same name siblings with this name are allowed * @param valueRestriction The requirements for node content * @param valueTypes list of allowed types for [Meta.value], null if all values are allowed. @@ -39,7 +39,7 @@ public enum class ValueRestriction { @Serializable public data class MetaDescriptor( public val description: String? = null, - public val children: Map = emptyMap(), + public val nodes: Map = emptyMap(), public val multiple: Boolean = false, public val valueRestriction: ValueRestriction = ValueRestriction.NONE, public val valueTypes: List? = null, @@ -47,6 +47,9 @@ public data class MetaDescriptor( public val defaultValue: Value? = null, public val attributes: Meta = Meta.EMPTY, ) { + @Deprecated("Replace by nodes", ReplaceWith("nodes")) + public val children: Map get() = nodes + /** * A node constructed of default values for this descriptor and its children */ @@ -55,7 +58,7 @@ public data class MetaDescriptor( defaultValue?.let { defaultValue -> this.value = defaultValue } - children.forEach { (key, descriptor) -> + nodes.forEach { (key, descriptor) -> set(key, descriptor.defaultNode) } } @@ -67,13 +70,13 @@ public data class MetaDescriptor( } } -public val MetaDescriptor.required: Boolean get() = valueRestriction == ValueRestriction.REQUIRED || children.values.any { required } +public val MetaDescriptor.required: Boolean get() = valueRestriction == ValueRestriction.REQUIRED || nodes.values.any { required } public val MetaDescriptor.allowedValues: List? get() = attributes[MetaDescriptor.ALLOWED_VALUES_KEY]?.value?.list public operator fun MetaDescriptor.get(name: Name): MetaDescriptor? = when (name.length) { 0 -> this - 1 -> children[name.firstOrNull()!!.toString()] + 1 -> nodes[name.firstOrNull()!!.toString()] else -> get(name.firstOrNull()!!.asName())?.get(name.cutFirst()) } @@ -95,7 +98,7 @@ public fun MetaDescriptor.validate(item: Meta?): Boolean { if (item == null) return !required if (!validate(item.value)) return false - children.forEach { (key, childDescriptor) -> + nodes.forEach { (key, childDescriptor) -> if (!childDescriptor.validate(item[key])) return false } return true diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/descriptors/MetaDescriptorBuilder.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/descriptors/MetaDescriptorBuilder.kt index 751d2525..5d4d81ad 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/descriptors/MetaDescriptorBuilder.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/descriptors/MetaDescriptorBuilder.kt @@ -80,7 +80,7 @@ public class MetaDescriptorBuilder @PublishedApi internal constructor() { public fun from(descriptor: MetaDescriptor) { description = descriptor.description - children.putAll(descriptor.children.mapValues { it.value.toBuilder() }) + children.putAll(descriptor.nodes.mapValues { it.value.toBuilder() }) multiple = descriptor.multiple valueRestriction = descriptor.valueRestriction valueTypes = descriptor.valueTypes @@ -92,7 +92,7 @@ public class MetaDescriptorBuilder @PublishedApi internal constructor() { @PublishedApi internal fun build(): MetaDescriptor = MetaDescriptor( description = description, - children = children.mapValues { it.value.build() }, + nodes = children.mapValues { it.value.build() }, multiple = multiple, valueRestriction = valueRestriction, valueTypes = valueTypes, @@ -143,7 +143,7 @@ public fun MetaDescriptorBuilder.required() { private fun MetaDescriptor.toBuilder(): MetaDescriptorBuilder = MetaDescriptorBuilder().apply { description = this@toBuilder.description - children = this@toBuilder.children.mapValuesTo(LinkedHashMap()) { it.value.toBuilder() } + children = this@toBuilder.nodes.mapValuesTo(LinkedHashMap()) { it.value.toBuilder() } multiple = this@toBuilder.multiple valueRestriction = this@toBuilder.valueRestriction valueTypes = this@toBuilder.valueTypes diff --git a/dataforge-meta/src/commonTest/kotlin/space/kscience/dataforge/meta/MetaDelegateTest.kt b/dataforge-meta/src/commonTest/kotlin/space/kscience/dataforge/meta/MetaDelegateTest.kt index 7a2dbc22..4b99cc3b 100644 --- a/dataforge-meta/src/commonTest/kotlin/space/kscience/dataforge/meta/MetaDelegateTest.kt +++ b/dataforge-meta/src/commonTest/kotlin/space/kscience/dataforge/meta/MetaDelegateTest.kt @@ -20,7 +20,7 @@ class MetaDelegateTest { var myValue by string() var safeValue by double(2.2) var enumValue by enum(TestEnum.YES) - var inner by spec(InnerScheme) + var inner by scheme(InnerScheme) companion object : SchemeSpec(::TestScheme) } diff --git a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/Task.kt b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/Task.kt index 329d9c5a..19d16c68 100644 --- a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/Task.kt +++ b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/Task.kt @@ -6,7 +6,7 @@ import space.kscience.dataforge.data.DataTree import space.kscience.dataforge.data.GoalExecutionRestriction import space.kscience.dataforge.meta.Meta import space.kscience.dataforge.meta.MetaRepr -import space.kscience.dataforge.meta.Specification +import space.kscience.dataforge.meta.MetaSpec import space.kscience.dataforge.meta.descriptors.Described import space.kscience.dataforge.meta.descriptors.MetaDescriptor import space.kscience.dataforge.misc.DfType @@ -43,10 +43,10 @@ public interface Task : Described { } /** - * A [Task] with [Specification] for wrapping and unwrapping task configuration + * A [Task] with [MetaSpec] for wrapping and unwrapping task configuration */ public interface TaskWithSpec : Task { - public val spec: Specification + public val spec: MetaSpec override val descriptor: MetaDescriptor? get() = spec.descriptor public suspend fun execute(workspace: Workspace, taskName: Name, configuration: C): TaskResult @@ -55,11 +55,11 @@ public interface TaskWithSpec : Task { execute(workspace, taskName, spec.read(taskMeta)) } -public suspend fun TaskWithSpec.execute( - workspace: Workspace, - taskName: Name, - block: C.() -> Unit = {}, -): TaskResult = execute(workspace, taskName, spec(block)) +//public suspend fun TaskWithSpec.execute( +// workspace: Workspace, +// taskName: Name, +// block: C.() -> Unit = {}, +//): TaskResult = execute(workspace, taskName, spec(block)) public class TaskResultBuilder( public val workspace: Workspace, @@ -76,7 +76,6 @@ public class TaskResultBuilder( * @param descriptor of meta accepted by this task * @param builder for resulting data set */ -@Suppress("FunctionName") public fun Task( resultType: KType, descriptor: MetaDescriptor? = null, @@ -98,7 +97,6 @@ public fun Task( } } -@Suppress("FunctionName") public inline fun Task( descriptor: MetaDescriptor? = null, noinline builder: suspend TaskResultBuilder.() -> Unit, @@ -116,10 +114,10 @@ public inline fun Task( @Suppress("FunctionName") public fun Task( resultType: KType, - specification: Specification, + specification: MetaSpec, builder: suspend TaskResultBuilder.(C) -> Unit, ): TaskWithSpec = object : TaskWithSpec { - override val spec: Specification = specification + override val spec: MetaSpec = specification override suspend fun execute( workspace: Workspace, @@ -135,8 +133,7 @@ public fun Task( } } -@Suppress("FunctionName") public inline fun Task( - specification: Specification, + specification: MetaSpec, noinline builder: suspend TaskResultBuilder.(C) -> Unit, ): Task = Task(typeOf(), specification, builder) \ No newline at end of file diff --git a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/WorkspaceBuilder.kt b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/WorkspaceBuilder.kt index 1538460f..5489e200 100644 --- a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/WorkspaceBuilder.kt +++ b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/WorkspaceBuilder.kt @@ -7,8 +7,8 @@ import space.kscience.dataforge.context.Global import space.kscience.dataforge.data.* import space.kscience.dataforge.meta.Meta import space.kscience.dataforge.meta.MetaRepr +import space.kscience.dataforge.meta.MetaSpec import space.kscience.dataforge.meta.MutableMeta -import space.kscience.dataforge.meta.Specification import space.kscience.dataforge.meta.descriptors.MetaDescriptor import space.kscience.dataforge.meta.descriptors.MetaDescriptorBuilder import space.kscience.dataforge.misc.DFBuilder @@ -68,7 +68,7 @@ public inline fun TaskContainer.task( } public inline fun TaskContainer.task( - specification: Specification, + specification: MetaSpec, noinline builder: suspend TaskResultBuilder.(C) -> Unit, ): PropertyDelegateProvider>> = PropertyDelegateProvider { _, property -> val taskName = Name.parse(property.name) -- 2.34.1 From 991f77c45a82908de5923ce3665ca687945f01b1 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Sun, 31 Dec 2023 13:08:30 +0300 Subject: [PATCH 11/62] Merge MetaConverter and MetaSpec --- CHANGELOG.md | 1 + .../kscience/dataforge/properties/Property.kt | 2 -- .../kscience/dataforge/meta/MetaConverter.kt | 3 +-- .../kscience/dataforge/meta/MetaDelegate.kt | 20 +++++++------- .../space/kscience/dataforge/meta/MetaSpec.kt | 3 +++ .../kscience/dataforge/meta/ObservableMeta.kt | 26 +++---------------- .../space/kscience/dataforge/meta/Scheme.kt | 22 ++++++++++++++++ .../meta/descriptors/schemeDescriptor.kt | 3 +++ 8 files changed, 44 insertions(+), 36 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d6daac6c..7b9586a8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ ### Changed - Descriptor `children` renamed to `nodes` +- `MetaConverter` now inherits `MetaSpec` (former `Specifiction`). So `MetaConverter` could be used more universally. ### Deprecated - `node(key,converter)` in favor of `serializable` delegate diff --git a/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/properties/Property.kt b/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/properties/Property.kt index 7b0280eb..b11fb2e1 100644 --- a/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/properties/Property.kt +++ b/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/properties/Property.kt @@ -1,6 +1,5 @@ package space.kscience.dataforge.properties -import kotlinx.coroutines.ExperimentalCoroutinesApi import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.StateFlow import space.kscience.dataforge.misc.DFExperimental @@ -14,7 +13,6 @@ public interface Property { } @DFExperimental -@OptIn(ExperimentalCoroutinesApi::class) public fun Property.toFlow(): StateFlow = MutableStateFlow(value).also { stateFlow -> onChange { stateFlow.value = it diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MetaConverter.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MetaConverter.kt index 0da36d17..933cc13d 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MetaConverter.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MetaConverter.kt @@ -183,7 +183,6 @@ public interface MetaConverter: MetaSpec { } } -public fun MetaConverter.readNullable(item: Meta?): T? = item?.let { read(it) } public fun MetaConverter.convertNullable(obj: T?): Meta? = obj?.let { convert(it) } -public fun MetaConverter.readValue(value: Value): T? = read(Meta(value)) + diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MetaDelegate.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MetaDelegate.kt index 70ff460d..a24f7371 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MetaDelegate.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MetaDelegate.kt @@ -13,13 +13,13 @@ public fun MetaProvider.node(key: Name? = null): ReadOnlyProperty = } /** - * Use [converter] to read the Meta node + * Use [metaSpec] to read the Meta node */ -public fun MetaProvider.convertable( - converter: MetaConverter, +public fun MetaProvider.spec( + metaSpec: MetaSpec, key: Name? = null, ): ReadOnlyProperty = ReadOnlyProperty { _, property -> - get(key ?: property.name.asName())?.let { converter.read(it) } + get(key ?: property.name.asName())?.let { metaSpec.read(it) } } /** @@ -29,19 +29,19 @@ public fun MetaProvider.convertable( public inline fun MetaProvider.serializable( descriptor: MetaDescriptor? = null, key: Name? = null, -): ReadOnlyProperty = convertable(MetaConverter.serializable(descriptor), key) +): ReadOnlyProperty = spec(MetaConverter.serializable(descriptor), key) @Deprecated("Use convertable", ReplaceWith("convertable(converter, key)")) public fun MetaProvider.node( key: Name? = null, - converter: MetaConverter, -): ReadOnlyProperty = convertable(converter, key) + converter: MetaSpec, +): ReadOnlyProperty = spec(converter, key) /** * Use [converter] to convert a list of same name siblings meta to object */ -public fun Meta.listOfConvertable( - converter: MetaConverter, +public fun Meta.listOfSpec( + converter: MetaSpec, key: Name? = null, ): ReadOnlyProperty> = ReadOnlyProperty{_, property -> val name = key ?: property.name.asName() @@ -52,7 +52,7 @@ public fun Meta.listOfConvertable( public inline fun Meta.listOfSerializable( descriptor: MetaDescriptor? = null, key: Name? = null, -): ReadOnlyProperty> = listOfConvertable(MetaConverter.serializable(descriptor), key) +): ReadOnlyProperty> = listOfSpec(MetaConverter.serializable(descriptor), key) /** * A property delegate that uses custom key diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MetaSpec.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MetaSpec.kt index be21cef7..9918d504 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MetaSpec.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MetaSpec.kt @@ -16,3 +16,6 @@ public interface MetaSpec : Described { public fun read(source: Meta): T = readOrNull(source) ?: error("Meta $source could not be interpreted by $this") } + +public fun MetaSpec.readNullable(item: Meta?): T? = item?.let { read(it) } +public fun MetaSpec.readValue(value: Value): T? = read(Meta(value)) diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/ObservableMeta.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/ObservableMeta.kt index 5a2b05f7..4a3db511 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/ObservableMeta.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/ObservableMeta.kt @@ -1,8 +1,10 @@ package space.kscience.dataforge.meta import space.kscience.dataforge.misc.ThreadSafe -import space.kscience.dataforge.names.* -import kotlin.reflect.KProperty1 +import space.kscience.dataforge.names.Name +import space.kscience.dataforge.names.cutFirst +import space.kscience.dataforge.names.firstOrNull +import space.kscience.dataforge.names.isEmpty internal data class MetaListener( @@ -67,24 +69,4 @@ internal abstract class AbstractObservableMeta : ObservableMeta { override fun toString(): String = Meta.toString(this) override fun equals(other: Any?): Boolean = Meta.equals(this, other as? Meta) override fun hashCode(): Int = Meta.hashCode(this) -} - -/** - * Use the value of the property in a [callBack]. - * The callback is called once immediately after subscription to pass the initial value. - * - * Optional [owner] property is used for - */ -public fun S.useProperty( - property: KProperty1, - owner: Any? = null, - callBack: S.(T) -> Unit, -) { - //Pass initial value. - callBack(property.get(this)) - meta.onChange(owner) { name -> - if (name.startsWith(property.name.asName())) { - callBack(property.get(this@useProperty)) - } - } } \ No newline at end of file diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/Scheme.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/Scheme.kt index ff8d8095..1d654ac2 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/Scheme.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/Scheme.kt @@ -9,6 +9,7 @@ import space.kscience.dataforge.misc.ThreadSafe import space.kscience.dataforge.names.* import kotlin.properties.ReadWriteProperty import kotlin.reflect.KProperty +import kotlin.reflect.KProperty1 /** * A base for delegate-based or descriptor-based scheme. [Scheme] has an empty constructor to simplify usage from [MetaSpec]. @@ -288,3 +289,24 @@ public fun Scheme.listOfScheme( spec: SchemeSpec, key: Name? = null, ): ReadWriteProperty> = meta.listOfScheme(spec, key) + + +/** + * Use the value of the property in a [callBack]. + * The callback is called once immediately after subscription to pass the initial value. + * + * Optional [owner] property is used for + */ +public fun S.useProperty( + property: KProperty1, + owner: Any? = null, + callBack: S.(T) -> Unit, +) { + //Pass initial value. + callBack(property.get(this)) + meta.onChange(owner) { name -> + if (name.startsWith(property.name.asName())) { + callBack(property.get(this@useProperty)) + } + } +} \ No newline at end of file diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/descriptors/schemeDescriptor.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/descriptors/schemeDescriptor.kt index 803659a6..a2a77182 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/descriptors/schemeDescriptor.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/descriptors/schemeDescriptor.kt @@ -3,9 +3,11 @@ package space.kscience.dataforge.meta.descriptors import space.kscience.dataforge.meta.Scheme import space.kscience.dataforge.meta.SchemeSpec import space.kscience.dataforge.meta.ValueType +import space.kscience.dataforge.misc.DFExperimental import kotlin.reflect.KProperty1 import kotlin.reflect.typeOf +@DFExperimental public inline fun MetaDescriptorBuilder.value( property: KProperty1, noinline block: MetaDescriptorBuilder.() -> Unit = {}, @@ -37,6 +39,7 @@ public inline fun MetaDescriptorBuilder.value( else -> node(property.name, block) } +@DFExperimental public inline fun MetaDescriptorBuilder.scheme( property: KProperty1, spec: SchemeSpec, -- 2.34.1 From 25281d0f6d615ff3193b1c78316e99acdea1e607 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Sun, 31 Dec 2023 17:55:25 +0300 Subject: [PATCH 12/62] Replace properties with flows --- .../dataforge/properties/MetaProperty.kt | 31 ----------- .../kscience/dataforge/properties/Property.kt | 45 ---------------- .../dataforge/properties/metaAsFlow.kt | 51 +++++++++++++++++++ .../dataforge/properties/schemeProperty.kt | 31 ----------- .../properties/MetaPropertiesTest.kt | 28 ---------- .../kscience/dataforge/properties/bindings.kt | 32 ------------ .../kscience/dataforge/meta/ObservableMeta.kt | 7 ++- 7 files changed, 56 insertions(+), 169 deletions(-) delete mode 100644 dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/properties/MetaProperty.kt delete mode 100644 dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/properties/Property.kt create mode 100644 dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/properties/metaAsFlow.kt delete mode 100644 dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/properties/schemeProperty.kt delete mode 100644 dataforge-context/src/commonTest/kotlin/space/kscience/dataforge/properties/MetaPropertiesTest.kt delete mode 100644 dataforge-context/src/jsMain/kotlin/space/kscience/dataforge/properties/bindings.kt diff --git a/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/properties/MetaProperty.kt b/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/properties/MetaProperty.kt deleted file mode 100644 index 85435602..00000000 --- a/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/properties/MetaProperty.kt +++ /dev/null @@ -1,31 +0,0 @@ -package space.kscience.dataforge.properties - - -import space.kscience.dataforge.meta.* -import space.kscience.dataforge.misc.DFExperimental -import space.kscience.dataforge.names.Name -import space.kscience.dataforge.names.startsWith - -@DFExperimental -public class MetaProperty( - public val meta: ObservableMutableMeta, - public val name: Name, - public val converter: MetaConverter, -) : Property { - - override var value: T? - get() = converter.readNullable(meta[name]) - set(value) { - meta[name] = converter.convertNullable(value) ?: Meta.EMPTY - } - - override fun onChange(owner: Any?, callback: (T?) -> Unit) { - meta.onChange(owner) { name -> - if (name.startsWith(this@MetaProperty.name)) callback(converter.readNullable(this[name])) - } - } - - override fun removeChangeListener(owner: Any?) { - meta.removeListener(owner) - } -} \ No newline at end of file diff --git a/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/properties/Property.kt b/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/properties/Property.kt deleted file mode 100644 index b11fb2e1..00000000 --- a/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/properties/Property.kt +++ /dev/null @@ -1,45 +0,0 @@ -package space.kscience.dataforge.properties - -import kotlinx.coroutines.flow.MutableStateFlow -import kotlinx.coroutines.flow.StateFlow -import space.kscience.dataforge.misc.DFExperimental - -@DFExperimental -public interface Property { - public var value: T - - public fun onChange(owner: Any? = null, callback: (T) -> Unit) - public fun removeChangeListener(owner: Any? = null) -} - -@DFExperimental -public fun Property.toFlow(): StateFlow = MutableStateFlow(value).also { stateFlow -> - onChange { - stateFlow.value = it - } -} - -/** - * Reflect all changes in the [source] property onto this property. Does not reflect changes back. - * - * @return a mirroring job - */ -@DFExperimental -public fun Property.mirror(source: Property) { - source.onChange(this) { - this.value = it - } -} - -/** - * Bi-directional connection between properties - */ -@DFExperimental -public fun Property.bind(other: Property) { - onChange(other) { - other.value = it - } - other.onChange { - this.value = it - } -} \ No newline at end of file diff --git a/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/properties/metaAsFlow.kt b/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/properties/metaAsFlow.kt new file mode 100644 index 00000000..90fafc5e --- /dev/null +++ b/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/properties/metaAsFlow.kt @@ -0,0 +1,51 @@ +package space.kscience.dataforge.properties + + +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Job +import kotlinx.coroutines.channels.awaitClose +import kotlinx.coroutines.flow.* +import kotlinx.coroutines.launch +import space.kscience.dataforge.meta.* +import space.kscience.dataforge.misc.DFExperimental + +@DFExperimental +public fun ObservableMeta.asFlow(converter: MetaSpec): Flow = callbackFlow { + onChange(this){ + trySend(converter.read(this)) + } + + awaitClose{ + removeListener(this) + } +} + +@DFExperimental +public fun MutableMeta.listenTo( + scope: CoroutineScope, + converter: MetaConverter, + flow: Flow, +): Job = flow.onEach { + update(converter.convert(it)) +}.launchIn(scope) + +@DFExperimental +public fun ObservableMutableMeta.bind( + scope: CoroutineScope, + converter: MetaConverter, + flow: MutableSharedFlow, +): Job = scope.launch{ + listenTo(this, converter,flow) + onChange(flow){ + launch { + flow.emit(converter.read(this@onChange)) + } + } + flow.onCompletion { + removeListener(flow) + } +}.also { + it.invokeOnCompletion { + removeListener(flow) + } +} diff --git a/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/properties/schemeProperty.kt b/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/properties/schemeProperty.kt deleted file mode 100644 index 3b4d948d..00000000 --- a/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/properties/schemeProperty.kt +++ /dev/null @@ -1,31 +0,0 @@ -package space.kscience.dataforge.properties - - -import space.kscience.dataforge.meta.Scheme -import space.kscience.dataforge.misc.DFExperimental -import space.kscience.dataforge.names.parseAsName -import space.kscience.dataforge.names.startsWith -import kotlin.reflect.KMutableProperty1 - -@DFExperimental -public fun S.property(property: KMutableProperty1): Property = - object : Property { - override var value: T? - get() = property.get(this@property) - set(value) { - property.set(this@property, value) - } - - override fun onChange(owner: Any?, callback: (T?) -> Unit) { - this@property.meta.onChange(this) { name -> - if (name.startsWith(property.name.parseAsName(true))) { - callback(property.get(this@property)) - } - } - } - - override fun removeChangeListener(owner: Any?) { - this@property.meta.removeListener(this@property) - } - - } \ No newline at end of file diff --git a/dataforge-context/src/commonTest/kotlin/space/kscience/dataforge/properties/MetaPropertiesTest.kt b/dataforge-context/src/commonTest/kotlin/space/kscience/dataforge/properties/MetaPropertiesTest.kt deleted file mode 100644 index 00b71673..00000000 --- a/dataforge-context/src/commonTest/kotlin/space/kscience/dataforge/properties/MetaPropertiesTest.kt +++ /dev/null @@ -1,28 +0,0 @@ -package space.kscience.dataforge.properties - -import space.kscience.dataforge.meta.Scheme -import space.kscience.dataforge.meta.SchemeSpec -import space.kscience.dataforge.meta.int -import space.kscience.dataforge.misc.DFExperimental -import kotlin.test.Test -import kotlin.test.assertEquals - -internal class TestScheme : Scheme() { - var a by int() - var b by int() - companion object : SchemeSpec(::TestScheme) -} - -@DFExperimental -class MetaPropertiesTest { - @Test - fun testBinding() { - val scheme = TestScheme.empty() - val a = scheme.property(TestScheme::a) - val b = scheme.property(TestScheme::b) - a.bind(b) - scheme.a = 2 - assertEquals(2, scheme.b) - assertEquals(2, b.value) - } -} \ No newline at end of file diff --git a/dataforge-context/src/jsMain/kotlin/space/kscience/dataforge/properties/bindings.kt b/dataforge-context/src/jsMain/kotlin/space/kscience/dataforge/properties/bindings.kt deleted file mode 100644 index b4451c97..00000000 --- a/dataforge-context/src/jsMain/kotlin/space/kscience/dataforge/properties/bindings.kt +++ /dev/null @@ -1,32 +0,0 @@ -package space.kscience.dataforge.properties - -import org.w3c.dom.HTMLInputElement -import space.kscience.dataforge.misc.DFExperimental - -@DFExperimental -public fun HTMLInputElement.bindValue(property: Property) { - if (this.onchange != null) error("Input element already bound") - this.onchange = { - property.value = this.value - Unit - } - property.onChange(this) { - if (value != it) { - value = it - } - } -} - -@DFExperimental -public fun HTMLInputElement.bindChecked(property: Property) { - if (this.onchange != null) error("Input element already bound") - this.onchange = { - property.value = this.checked - Unit - } - property.onChange(this) { - if (checked != it) { - checked = it - } - } -} \ No newline at end of file diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/ObservableMeta.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/ObservableMeta.kt index 4a3db511..b481962e 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/ObservableMeta.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/ObservableMeta.kt @@ -17,12 +17,15 @@ internal data class MetaListener( */ public interface ObservableMeta : Meta { /** - * Add change listener to this meta. Owner is declared to be able to remove listeners later. Listener without owner could not be removed + * Add change listener to this meta. The Owner is declared to be able to remove listeners later. + * Listeners without an owner could be only removed all together. + * + * `this` object in the listener represents the current state of this meta. The name points to a changed node */ public fun onChange(owner: Any?, callback: Meta.(name: Name) -> Unit) /** - * Remove all listeners belonging to given owner + * Remove all listeners belonging to the given [owner]. Passing null removes all listeners. */ public fun removeListener(owner: Any?) -- 2.34.1 From 6ba189fa3473ed06c3d35b7015e4009b1c9f0f92 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Sun, 31 Dec 2023 19:57:57 +0300 Subject: [PATCH 13/62] Remove second invalidate on meta node remove --- .../kotlin/space/kscience/dataforge/meta/MutableMeta.kt | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MutableMeta.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MutableMeta.kt index 24a09240..f1e66060 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MutableMeta.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MutableMeta.kt @@ -324,8 +324,6 @@ private class MutableMetaImpl( //remove child and invalidate if argument is null if (node == null) { children.remove(token)?.removeListener(this) - // old item is not null otherwise we can't be here - invalidate(name) } else { val newNode = wrapItem(node) newNode.adoptBy(this, token) @@ -335,7 +333,7 @@ private class MutableMetaImpl( else -> { val token = name.firstOrNull()!! - //get existing or create new node. + //get an existing node or create a new node. if (items[token] == null) { val newNode = MutableMetaImpl(null) newNode.adoptBy(this, token) -- 2.34.1 From 8f3c2f3950a55400aefa858315d53e788030f66a Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Thu, 4 Jan 2024 18:59:57 +0300 Subject: [PATCH 14/62] refactoring of data transformations --- CHANGELOG.md | 2 + .../kscience/dataforge/provider/dfType.kt | 12 +-- .../kscience/dataforge/actions/Action.kt | 22 +++--- .../space/kscience/dataforge/data/Data.kt | 2 - .../kscience/dataforge/data/MetaMaskData.kt | 23 ++++++ .../kscience/dataforge/data/dataTransform.kt | 78 ++++++++++++------- .../kscience/dataforge/meta/MutableMeta.kt | 15 ++-- .../kscience/dataforge/meta/SealedMeta.kt | 10 +-- .../space/kscience/dataforge/misc/DfType.kt | 3 - .../dataforge/workspace/taskBuilders.kt | 26 ++++++- .../workspace/CachingWorkspaceTest.kt | 4 +- .../workspace/FileWorkspaceCacheTest.kt | 2 +- .../workspace/SimpleWorkspaceTest.kt | 6 +- 13 files changed, 136 insertions(+), 69 deletions(-) create mode 100644 dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/MetaMaskData.kt diff --git a/CHANGELOG.md b/CHANGELOG.md index 7b9586a8..4469f802 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,10 +6,12 @@ - Wasm artifacts - Add automatic MetaConverter for serializeable objects - Add Meta and MutableMeta delegates for convertable and serializeable +- Meta mapping for data. ### Changed - Descriptor `children` renamed to `nodes` - `MetaConverter` now inherits `MetaSpec` (former `Specifiction`). So `MetaConverter` could be used more universally. +- Meta copy and modification now use lightweight non-observable meta builders. ### Deprecated - `node(key,converter)` in favor of `serializable` delegate diff --git a/dataforge-context/src/jvmMain/kotlin/space/kscience/dataforge/provider/dfType.kt b/dataforge-context/src/jvmMain/kotlin/space/kscience/dataforge/provider/dfType.kt index 04e681da..cccc3efb 100644 --- a/dataforge-context/src/jvmMain/kotlin/space/kscience/dataforge/provider/dfType.kt +++ b/dataforge-context/src/jvmMain/kotlin/space/kscience/dataforge/provider/dfType.kt @@ -12,7 +12,7 @@ import kotlin.reflect.full.findAnnotation @DFExperimental -public val KClass<*>.dfId: String +public val KClass<*>.dfType: String get() = findAnnotation()?.id ?: simpleName ?: "" /** @@ -20,13 +20,13 @@ public val KClass<*>.dfId: String */ @DFExperimental public inline fun Provider.provideByType(name: String): T? { - val target = T::class.dfId + val target = T::class.dfType return provide(target, name) } @DFExperimental public inline fun Provider.top(): Map { - val target = T::class.dfId + val target = T::class.dfType return top(target) } @@ -35,15 +35,15 @@ public inline fun Provider.top(): Map { */ @DFExperimental public inline fun Context.gather(inherit: Boolean = true): Map = - gather(T::class.dfId, inherit) + gather(T::class.dfType, inherit) @DFExperimental public inline fun PluginBuilder.provides(items: Map) { - provides(T::class.dfId, items) + provides(T::class.dfType, items) } @DFExperimental public inline fun PluginBuilder.provides(vararg items: Named) { - provides(T::class.dfId, *items) + provides(T::class.dfType, *items) } diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/Action.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/Action.kt index 4fed8e51..3feec9bc 100644 --- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/Action.kt +++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/Action.kt @@ -7,30 +7,28 @@ import space.kscience.dataforge.misc.DFExperimental /** * A simple data transformation on a data node. Actions should avoid doing actual dependency evaluation in [execute]. */ -public interface Action { +public fun interface Action { /** * Transform the data in the node, producing a new node. By default, it is assumed that all calculations are lazy * so not actual computation is started at this moment. */ - public fun execute(dataSet: DataSet, meta: Meta = Meta.EMPTY): DataSet + public fun execute(dataSet: DataSet, meta: Meta): DataSet public companion object } +/** + * A convenience method to transform data using given [action] + */ +public fun DataSet.transform(action: Action, meta: Meta = Meta.EMPTY): DataSet = + action.execute(this, meta) + /** * Action composition. The result is terminal if one of its parts is terminal */ -public infix fun Action.then(action: Action): Action { - // TODO introduce composite action and add optimize by adding action to the list - return object : Action { - - override fun execute( - dataSet: DataSet, - meta: Meta, - ): DataSet = action.execute(this@then.execute(dataSet, meta), meta) - } -} +public infix fun Action.then(action: Action): Action = + Action { dataSet, meta -> action.execute(this@then.execute(dataSet, meta), meta) } @DFExperimental public operator fun Action.invoke( diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/Data.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/Data.kt index 4d883795..756cd840 100644 --- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/Data.kt +++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/Data.kt @@ -87,7 +87,6 @@ public class StaticData( public inline fun Data(value: T, meta: Meta = Meta.EMPTY): StaticData = StaticData(typeOf(), value, meta) -@Suppress("FunctionName") @DFInternal public fun Data( type: KType, @@ -98,7 +97,6 @@ public fun Data( ): Data = LazyData(type, meta, context, dependencies, block) @OptIn(DFInternal::class) -@Suppress("FunctionName") public inline fun Data( meta: Meta = Meta.EMPTY, context: CoroutineContext = EmptyCoroutineContext, diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/MetaMaskData.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/MetaMaskData.kt new file mode 100644 index 00000000..acf2410d --- /dev/null +++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/MetaMaskData.kt @@ -0,0 +1,23 @@ +package space.kscience.dataforge.data + +import space.kscience.dataforge.meta.Meta +import space.kscience.dataforge.meta.MutableMeta +import space.kscience.dataforge.meta.copy + + +private class MetaMaskData(val origin: Data, override val meta: Meta) : Data by origin + +/** + * A data with overriden meta. It reflects original data computed state. + */ +public fun Data.withMeta(newMeta: Meta): Data = if (this is MetaMaskData) { + MetaMaskData(origin, newMeta) +} else { + MetaMaskData(this, newMeta) +} + +/** + * Create a new [Data] with the same computation, but different meta. The meta is created by applying [block] to + * the existing data meta. + */ +public inline fun Data.mapMeta(block: MutableMeta.() -> Unit): Data = withMeta(meta.copy(block)) \ No newline at end of file diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataTransform.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataTransform.kt index 76577346..c1ecdc09 100644 --- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataTransform.kt +++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataTransform.kt @@ -1,11 +1,9 @@ package space.kscience.dataforge.data -import space.kscience.dataforge.meta.Meta -import space.kscience.dataforge.meta.MutableMeta -import space.kscience.dataforge.meta.seal -import space.kscience.dataforge.meta.toMutableMeta +import space.kscience.dataforge.meta.* import space.kscience.dataforge.misc.DFInternal import space.kscience.dataforge.names.Name +import space.kscience.dataforge.names.NameToken import kotlin.coroutines.CoroutineContext import kotlin.coroutines.EmptyCoroutineContext import kotlin.reflect.KType @@ -28,8 +26,8 @@ public suspend fun NamedData.awaitWithMeta(): NamedValueWithMeta * @param block the transformation itself */ public inline fun Data.map( - coroutineContext: CoroutineContext = EmptyCoroutineContext, meta: Meta = this.meta, + coroutineContext: CoroutineContext = EmptyCoroutineContext, crossinline block: suspend (T) -> R, ): Data = Data(meta, coroutineContext, listOf(this)) { block(await()) @@ -40,8 +38,8 @@ public inline fun Data.map( */ public inline fun Data.combine( other: Data, - coroutineContext: CoroutineContext = EmptyCoroutineContext, meta: Meta = this.meta, + coroutineContext: CoroutineContext = EmptyCoroutineContext, crossinline block: suspend (left: T1, right: T2) -> R, ): Data = Data(meta, coroutineContext, listOf(this, other)) { block(await(), other.await()) @@ -50,12 +48,22 @@ public inline fun Data.combine( //data collection operations +@PublishedApi +internal fun Iterable>.joinMeta(): Meta = Meta { + var counter = 0 + forEach { data -> + val inputIndex = (data as? NamedData)?.name?.toString() ?: (counter++).toString() + val token = NameToken("data", inputIndex) + set(token, data.meta) + } +} + /** * Lazily reduce a collection of [Data] to a single data. */ public inline fun Collection>.reduceToData( + meta: Meta = joinMeta(), coroutineContext: CoroutineContext = EmptyCoroutineContext, - meta: Meta = Meta.EMPTY, crossinline block: suspend (List>) -> R, ): Data = Data( meta, @@ -65,11 +73,19 @@ public inline fun Collection>.reduceToData( block(map { it.awaitWithMeta() }) } +@PublishedApi +internal fun Map<*, Data<*>>.joinMeta(): Meta = Meta { + forEach { (key, data) -> + val token = NameToken("data", key.toString()) + set(token, data.meta) + } +} + @DFInternal public fun Map>.reduceToData( outputType: KType, + meta: Meta = joinMeta(), coroutineContext: CoroutineContext = EmptyCoroutineContext, - meta: Meta = Meta.EMPTY, block: suspend (Map>) -> R, ): Data = Data( outputType, @@ -87,8 +103,8 @@ public fun Map>.reduceToData( * @param R type of the result goal */ public inline fun Map>.reduceToData( + meta: Meta = joinMeta(), coroutineContext: CoroutineContext = EmptyCoroutineContext, - meta: Meta = Meta.EMPTY, crossinline block: suspend (Map>) -> R, ): Data = Data( meta, @@ -103,8 +119,8 @@ public inline fun Map>.reduceToData( @DFInternal public inline fun Iterable>.reduceToData( outputType: KType, + meta: Meta = joinMeta(), coroutineContext: CoroutineContext = EmptyCoroutineContext, - meta: Meta = Meta.EMPTY, crossinline transformation: suspend (Collection>) -> R, ): Data = Data( outputType, @@ -117,20 +133,20 @@ public inline fun Iterable>.reduceToData( @OptIn(DFInternal::class) public inline fun Iterable>.reduceToData( + meta: Meta = joinMeta(), coroutineContext: CoroutineContext = EmptyCoroutineContext, - meta: Meta = Meta.EMPTY, crossinline transformation: suspend (Collection>) -> R, -): Data = reduceToData(typeOf(), coroutineContext, meta) { +): Data = reduceToData(typeOf(), meta, coroutineContext) { transformation(it) } public inline fun Iterable>.foldToData( initial: R, + meta: Meta = joinMeta(), coroutineContext: CoroutineContext = EmptyCoroutineContext, - meta: Meta = Meta.EMPTY, crossinline block: suspend (result: R, data: ValueWithMeta) -> R, ): Data = reduceToData( - coroutineContext, meta + meta, coroutineContext ) { it.fold(initial) { acc, t -> block(acc, t) } } @@ -141,8 +157,8 @@ public inline fun Iterable>.foldToData( @DFInternal public inline fun Iterable>.reduceNamedToData( outputType: KType, + meta: Meta = joinMeta(), coroutineContext: CoroutineContext = EmptyCoroutineContext, - meta: Meta = Meta.EMPTY, crossinline transformation: suspend (Collection>) -> R, ): Data = Data( outputType, @@ -155,10 +171,10 @@ public inline fun Iterable>.reduceNamedToData( @OptIn(DFInternal::class) public inline fun Iterable>.reduceNamedToData( + meta: Meta = joinMeta(), coroutineContext: CoroutineContext = EmptyCoroutineContext, - meta: Meta = Meta.EMPTY, crossinline transformation: suspend (Collection>) -> R, -): Data = reduceNamedToData(typeOf(), coroutineContext, meta) { +): Data = reduceNamedToData(typeOf(), meta, coroutineContext) { transformation(it) } @@ -167,11 +183,11 @@ public inline fun Iterable>.reduceNamedT */ public inline fun Iterable>.foldNamedToData( initial: R, + meta: Meta = joinMeta(), coroutineContext: CoroutineContext = EmptyCoroutineContext, - meta: Meta = Meta.EMPTY, crossinline block: suspend (result: R, data: NamedValueWithMeta) -> R, ): Data = reduceNamedToData( - coroutineContext, meta + meta, coroutineContext ) { it.fold(initial) { acc, t -> block(acc, t) } } @@ -181,8 +197,8 @@ public inline fun Iterable>.foldNamedToD @DFInternal public suspend fun DataSet.map( outputType: KType, - coroutineContext: CoroutineContext = EmptyCoroutineContext, metaTransform: MutableMeta.() -> Unit = {}, + coroutineContext: CoroutineContext = EmptyCoroutineContext, block: suspend (NamedValueWithMeta) -> R, ): DataTree = DataTree(outputType) { forEach { @@ -196,10 +212,10 @@ public suspend fun DataSet.map( @OptIn(DFInternal::class) public suspend inline fun DataSet.map( - coroutineContext: CoroutineContext = EmptyCoroutineContext, noinline metaTransform: MutableMeta.() -> Unit = {}, + coroutineContext: CoroutineContext = EmptyCoroutineContext, noinline block: suspend (NamedValueWithMeta) -> R, -): DataTree = map(typeOf(), coroutineContext, metaTransform, block) +): DataTree = map(typeOf(), metaTransform, coroutineContext, block) public inline fun DataSet.forEach(block: (NamedData) -> Unit) { for (d in this) { @@ -207,15 +223,25 @@ public inline fun DataSet.forEach(block: (NamedData) -> Unit) { } } +// DataSet reduction + +@PublishedApi +internal fun DataSet<*>.joinMeta(): Meta = Meta { + forEach { (key, data) -> + val token = NameToken("data", key.toString()) + set(token, data.meta) + } +} + public inline fun DataSet.reduceToData( + meta: Meta = joinMeta(), coroutineContext: CoroutineContext = EmptyCoroutineContext, - meta: Meta = Meta.EMPTY, crossinline transformation: suspend (Iterable>) -> R, -): Data = asIterable().reduceNamedToData(coroutineContext, meta, transformation) +): Data = asIterable().reduceNamedToData(meta, coroutineContext, transformation) public inline fun DataSet.foldToData( initial: R, + meta: Meta = joinMeta(), coroutineContext: CoroutineContext = EmptyCoroutineContext, - meta: Meta = Meta.EMPTY, crossinline block: suspend (result: R, data: NamedValueWithMeta) -> R, -): Data = asIterable().foldNamedToData(initial, coroutineContext, meta, block) \ No newline at end of file +): Data = asIterable().foldNamedToData(initial, meta, coroutineContext, block) \ No newline at end of file diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MutableMeta.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MutableMeta.kt index f1e66060..231f9e54 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MutableMeta.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MutableMeta.kt @@ -370,7 +370,8 @@ public fun MutableMeta.append(key: String, value: Value): Unit = append(Name.par /** * Create a mutable copy of this meta. The copy is created even if the Meta is already mutable */ -public fun Meta.toMutableMeta(): ObservableMutableMeta = MutableMetaImpl(value, items) +public fun Meta.toMutableMeta(): MutableMeta = + MutableMeta { update(this@toMutableMeta) } //MutableMetaImpl(value, items) public fun Meta.asMutableMeta(): MutableMeta = (this as? MutableMeta) ?: toMutableMeta() @@ -385,12 +386,14 @@ public inline fun ObservableMutableMeta(builder: MutableMeta.() -> Unit = {}): O /** - * Create a copy of this [Meta], optionally applying the given [block]. - * The listeners of the original Config are not retained. + * Create a read-only copy of this [Meta]. [modification] is an optional modification applied to [Meta] on copy. + * + * The copy does not reflect changes of the initial Meta. */ -public inline fun Meta.copy(block: MutableMeta.() -> Unit = {}): Meta = - toMutableMeta().apply(block) - +public inline fun Meta.copy(modification: MutableMeta.() -> Unit = {}): Meta = Meta { + update(this@copy) + modification() +} private class MutableMetaWithDefault( val source: MutableMeta, val default: MetaProvider, val rootName: Name, diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/SealedMeta.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/SealedMeta.kt index 217a6a04..e842b990 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/SealedMeta.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/SealedMeta.kt @@ -101,11 +101,6 @@ internal class MetaBuilder( override fun hashCode(): Int = Meta.hashCode(this) } -/** - * Create a read-only meta. - */ -public inline fun Meta(builder: MutableMeta.() -> Unit): Meta = - MetaBuilder().apply(builder).seal() /** * Create an immutable meta. @@ -113,6 +108,11 @@ public inline fun Meta(builder: MutableMeta.() -> Unit): Meta = public inline fun SealedMeta(builder: MutableMeta.() -> Unit): SealedMeta = MetaBuilder().apply(builder).seal() +/** + * Create a read-only meta. + */ +public inline fun Meta(builder: MutableMeta.() -> Unit): Meta = SealedMeta(builder) + /** * Create an empty meta mutable meta. */ diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/misc/DfType.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/misc/DfType.kt index 11f548ae..936e793a 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/misc/DfType.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/misc/DfType.kt @@ -6,6 +6,3 @@ package space.kscience.dataforge.misc @MustBeDocumented @Target(AnnotationTarget.CLASS) public annotation class DfType(val id: String) - -@Deprecated("Replace with DfType", replaceWith = ReplaceWith("DfType")) -public typealias DfId = DfType diff --git a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/taskBuilders.kt b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/taskBuilders.kt index bf3d5921..a7c1f0fb 100644 --- a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/taskBuilders.kt +++ b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/taskBuilders.kt @@ -1,5 +1,6 @@ package space.kscience.dataforge.workspace +import space.kscience.dataforge.actions.Action import space.kscience.dataforge.context.PluginFactory import space.kscience.dataforge.data.DataSet import space.kscience.dataforge.data.forEach @@ -68,7 +69,7 @@ public val TaskResultBuilder<*>.allData: DataSelector<*> } /** - * Perform a lazy mapping task using given [selector] and [action]. The meta of resulting + * Perform a lazy mapping task using given [selector] and one-to-one [action]. * TODO move selector to receiver with multi-receivers * * @param selector a workspace data selector. Could be either task selector or initial data selector. @@ -77,7 +78,7 @@ public val TaskResultBuilder<*>.allData: DataSelector<*> * @param action process individual data asynchronously. */ @DFExperimental -public suspend inline fun TaskResultBuilder.pipeFrom( +public suspend inline fun TaskResultBuilder.transformEach( selector: DataSelector, dependencyMeta: Meta = defaultDependencyMeta, dataMetaTransform: MutableMeta.(name: Name) -> Unit = {}, @@ -89,7 +90,7 @@ public suspend inline fun TaskResultBuilder.pipeFr dataMetaTransform(data.name) } - val res = data.map(workspace.context.coroutineContext, meta) { + val res = data.map(meta, workspace.context.coroutineContext) { action(it, data.name, meta) } @@ -97,4 +98,23 @@ public suspend inline fun TaskResultBuilder.pipeFr } } +/** + * Set given [dataSet] as a task result. + */ +public fun TaskResultBuilder.result(dataSet: DataSet) { + node(Name.EMPTY, dataSet) +} + +/** + * Use provided [action] to fill the result + */ +@DFExperimental +public suspend inline fun TaskResultBuilder.actionFrom( + selector: DataSelector, + action: Action, + dependencyMeta: Meta = defaultDependencyMeta, +) { + node(Name.EMPTY, action.execute(from(selector,dependencyMeta), dependencyMeta)) +} + diff --git a/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/CachingWorkspaceTest.kt b/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/CachingWorkspaceTest.kt index 4e1923bc..467b04b0 100644 --- a/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/CachingWorkspaceTest.kt +++ b/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/CachingWorkspaceTest.kt @@ -31,7 +31,7 @@ internal class CachingWorkspaceTest { inMemoryCache() val doFirst by task { - pipeFrom(allData) { _, name, _ -> + transformEach(allData) { _, name, _ -> firstCounter++ println("Done first on $name with flag=${taskMeta["flag"].boolean}") } @@ -39,7 +39,7 @@ internal class CachingWorkspaceTest { @Suppress("UNUSED_VARIABLE") val doSecond by task { - pipeFrom( + transformEach( doFirst, dependencyMeta = if(taskMeta["flag"].boolean == true) taskMeta else Meta.EMPTY ) { _, name, _ -> diff --git a/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/FileWorkspaceCacheTest.kt b/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/FileWorkspaceCacheTest.kt index 00ca67cb..03fda0eb 100644 --- a/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/FileWorkspaceCacheTest.kt +++ b/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/FileWorkspaceCacheTest.kt @@ -24,7 +24,7 @@ class FileWorkspaceCacheTest { @Suppress("UNUSED_VARIABLE") val echo by task { - pipeFrom(dataByType()) { arg, _, _ -> arg } + transformEach(dataByType()) { arg, _, _ -> arg } } } diff --git a/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/SimpleWorkspaceTest.kt b/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/SimpleWorkspaceTest.kt index 7bfe0927..b79ed2e7 100644 --- a/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/SimpleWorkspaceTest.kt +++ b/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/SimpleWorkspaceTest.kt @@ -37,7 +37,7 @@ internal object TestPlugin : WorkspacePlugin() { val test by task { // type is inferred - pipeFrom(dataByType()) { arg, _, _ -> + transformEach(dataByType()) { arg, _, _ -> logger.info { "Test: $arg" } arg } @@ -74,7 +74,7 @@ internal class SimpleWorkspaceTest { } val square by task { - pipeFrom(dataByType()) { arg, name, meta -> + transformEach(dataByType()) { arg, name, meta -> if (meta["testFlag"].boolean == true) { println("Side effect") } @@ -84,7 +84,7 @@ internal class SimpleWorkspaceTest { } val linear by task { - pipeFrom(dataByType()) { arg, name, _ -> + transformEach(dataByType()) { arg, name, _ -> workspace.logger.info { "Starting linear on $name" } arg * 2 + 1 } -- 2.34.1 From 5e3de7073747d35e0f44e28602be30ba6cdb8c36 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Fri, 5 Jan 2024 13:02:11 +0300 Subject: [PATCH 15/62] Add action delegate for task creation --- .../dataforge/data/DataTreeBuilder.kt | 12 ++--- .../kscience/dataforge/data/StaticDataTree.kt | 2 - .../kscience/dataforge/data/ActionsTest.kt | 2 +- .../dataforge/data/DataTreeBuilderTest.kt | 4 +- .../dataforge/workspace/WorkspaceBuilder.kt | 51 +++++++++++++------ 5 files changed, 41 insertions(+), 30 deletions(-) diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataTreeBuilder.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataTreeBuilder.kt index 303ba44e..f8a5dd4c 100644 --- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataTreeBuilder.kt +++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataTreeBuilder.kt @@ -10,7 +10,6 @@ import space.kscience.dataforge.misc.ThreadSafe import space.kscience.dataforge.names.* import kotlin.collections.set import kotlin.coroutines.CoroutineContext -import kotlin.coroutines.coroutineContext import kotlin.reflect.KType import kotlin.reflect.typeOf @@ -21,7 +20,7 @@ public interface DataSourceBuilder : DataSetBuilder, DataSource { /** * A mutable [DataTree] that propagates updates */ -public class DataTreeBuilder( +public class DataTreeBuilder internal constructor( override val dataType: KType, coroutineContext: CoroutineContext, ) : DataTree, DataSourceBuilder { @@ -100,19 +99,14 @@ public class DataTreeBuilder( public fun DataSource( type: KType, parent: CoroutineScope, - block: DataSourceBuilder.() -> Unit, + block: DataSourceBuilder.() -> Unit = {}, ): DataTreeBuilder = DataTreeBuilder(type, parent.coroutineContext).apply(block) @Suppress("OPT_IN_USAGE", "FunctionName") public inline fun DataSource( parent: CoroutineScope, - crossinline block: DataSourceBuilder.() -> Unit, -): DataTreeBuilder = DataSource(typeOf(), parent) { block() } - -@Suppress("FunctionName") -public suspend inline fun DataSource( crossinline block: DataSourceBuilder.() -> Unit = {}, -): DataTreeBuilder = DataTreeBuilder(typeOf(), coroutineContext).apply { block() } +): DataTreeBuilder = DataSource(typeOf(), parent) { block() } public inline fun DataSourceBuilder.emit( name: Name, diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/StaticDataTree.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/StaticDataTree.kt index 4f0f455e..186e5c2a 100644 --- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/StaticDataTree.kt +++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/StaticDataTree.kt @@ -65,13 +65,11 @@ internal class StaticDataTree( } } -@Suppress("FunctionName") public inline fun DataTree( dataType: KType, block: DataSetBuilder.() -> Unit, ): DataTree = StaticDataTree(dataType).apply { block() } -@Suppress("FunctionName") public inline fun DataTree( noinline block: DataSetBuilder.() -> Unit, ): DataTree = DataTree(typeOf(), block) diff --git a/dataforge-data/src/jvmTest/kotlin/space/kscience/dataforge/data/ActionsTest.kt b/dataforge-data/src/jvmTest/kotlin/space/kscience/dataforge/data/ActionsTest.kt index b24c4f27..06a7e755 100644 --- a/dataforge-data/src/jvmTest/kotlin/space/kscience/dataforge/data/ActionsTest.kt +++ b/dataforge-data/src/jvmTest/kotlin/space/kscience/dataforge/data/ActionsTest.kt @@ -29,7 +29,7 @@ internal class ActionsTest { @Test fun testDynamicMapAction() = runTest { - val data: DataSourceBuilder = DataSource() + val data: DataSourceBuilder = DataSource(this) val plusOne = Action.mapping { result { it + 1 } diff --git a/dataforge-data/src/jvmTest/kotlin/space/kscience/dataforge/data/DataTreeBuilderTest.kt b/dataforge-data/src/jvmTest/kotlin/space/kscience/dataforge/data/DataTreeBuilderTest.kt index b77f7ea2..a4d28eab 100644 --- a/dataforge-data/src/jvmTest/kotlin/space/kscience/dataforge/data/DataTreeBuilderTest.kt +++ b/dataforge-data/src/jvmTest/kotlin/space/kscience/dataforge/data/DataTreeBuilderTest.kt @@ -56,7 +56,7 @@ internal class DataTreeBuilderTest { try { lateinit var updateJob: Job supervisorScope { - val subNode = DataSource { + val subNode = DataSource(this) { updateJob = launch { repeat(10) { delay(10) @@ -70,7 +70,7 @@ internal class DataTreeBuilderTest { println(it) } } - val rootNode = DataSource { + val rootNode = DataSource(this) { setAndWatch("sub".asName(), subNode) } diff --git a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/WorkspaceBuilder.kt b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/WorkspaceBuilder.kt index 5489e200..39e35b39 100644 --- a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/WorkspaceBuilder.kt +++ b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/WorkspaceBuilder.kt @@ -1,18 +1,17 @@ package space.kscience.dataforge.workspace import kotlinx.coroutines.CoroutineScope +import space.kscience.dataforge.actions.Action import space.kscience.dataforge.context.Context import space.kscience.dataforge.context.ContextBuilder import space.kscience.dataforge.context.Global -import space.kscience.dataforge.data.* -import space.kscience.dataforge.meta.Meta -import space.kscience.dataforge.meta.MetaRepr -import space.kscience.dataforge.meta.MetaSpec -import space.kscience.dataforge.meta.MutableMeta +import space.kscience.dataforge.data.DataSet +import space.kscience.dataforge.data.DataSource +import space.kscience.dataforge.data.DataSourceBuilder +import space.kscience.dataforge.meta.* import space.kscience.dataforge.meta.descriptors.MetaDescriptor import space.kscience.dataforge.meta.descriptors.MetaDescriptorBuilder import space.kscience.dataforge.misc.DFBuilder -import space.kscience.dataforge.misc.DFExperimental import space.kscience.dataforge.names.Name import space.kscience.dataforge.names.asName import kotlin.collections.set @@ -45,6 +44,9 @@ public inline fun TaskContainer.registerTask( noinline builder: suspend TaskResultBuilder.() -> Unit, ): Unit = registerTask(Name.parse(name), Task(MetaDescriptor(descriptorBuilder), builder)) +/** + * Create a new t + */ public inline fun TaskContainer.buildTask( name: String, descriptorBuilder: MetaDescriptorBuilder.() -> Unit = {}, @@ -67,6 +69,9 @@ public inline fun TaskContainer.task( ReadOnlyProperty { _, _ -> TaskReference(taskName, task) } } +/** + * Create a task based on [MetaSpec] + */ public inline fun TaskContainer.task( specification: MetaSpec, noinline builder: suspend TaskResultBuilder.(C) -> Unit, @@ -77,15 +82,34 @@ public inline fun TaskContainer.task( ReadOnlyProperty { _, _ -> TaskReference(taskName, task) } } +/** + * A delegate to create a custom task + */ public inline fun TaskContainer.task( noinline descriptorBuilder: MetaDescriptorBuilder.() -> Unit = {}, noinline builder: suspend TaskResultBuilder.() -> Unit, ): PropertyDelegateProvider>> = task(MetaDescriptor(descriptorBuilder), builder) -public class WorkspaceBuilder(private val parentContext: Context = Global) : TaskContainer { +/** + * A delegate for creating a task based on [action] + */ +public inline fun TaskContainer.action( + selector: DataSelector, + action: Action, + noinline metaTransform: MutableMeta.()-> Unit = {}, + noinline descriptorBuilder: MetaDescriptorBuilder.() -> Unit = {}, +): PropertyDelegateProvider>> = + task(MetaDescriptor(descriptorBuilder)) { + result(action.execute(from(selector), taskMeta.copy(metaTransform))) + } + +public class WorkspaceBuilder( + private val parentContext: Context = Global, + private val coroutineScope: CoroutineScope = parentContext, +) : TaskContainer { private var context: Context? = null - private var data: DataSet<*>? = null + private val data = DataSource(coroutineScope) private val targets: HashMap = HashMap() private val tasks = HashMap>() private var cache: WorkspaceCache? = null @@ -100,13 +124,8 @@ public class WorkspaceBuilder(private val parentContext: Context = Global) : Tas /** * Define intrinsic data for the workspace */ - public fun data(builder: DataSetBuilder.() -> Unit) { - data = DataTree(builder) - } - - @DFExperimental - public fun data(scope: CoroutineScope, builder: DataSourceBuilder.() -> Unit) { - data = DataSource(scope, builder) + public fun data(builder: DataSourceBuilder.() -> Unit) { + data.apply(builder) } /** @@ -132,7 +151,7 @@ public class WorkspaceBuilder(private val parentContext: Context = Global) : Tas val postProcess: suspend (TaskResult<*>) -> TaskResult<*> = { result -> cache?.evaluate(result) ?: result } - return WorkspaceImpl(context ?: parentContext, data ?: DataSet.EMPTY, targets, tasks, postProcess) + return WorkspaceImpl(context ?: parentContext, data, targets, tasks, postProcess) } } -- 2.34.1 From 5fec0518d46f481747b748975c75f817eb5b6e64 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Fri, 19 Jan 2024 22:23:02 +0300 Subject: [PATCH 16/62] [WIP] change data structure --- dataforge-context/build.gradle.kts | 2 +- .../dataforge/actions/AbstractAction.kt | 43 ++- .../kscience/dataforge/actions/Action.kt | 25 +- .../kscience/dataforge/actions/MapAction.kt | 20 +- .../dataforge/actions/ReduceAction.kt | 14 +- .../kscience/dataforge/actions/SplitAction.kt | 11 +- .../space/kscience/dataforge/data/Data.kt | 20 +- .../kscience/dataforge/data/DataFilter.kt | 89 ++++++ .../space/kscience/dataforge/data/DataSet.kt | 124 -------- .../kscience/dataforge/data/DataSetBuilder.kt | 165 ----------- .../kscience/dataforge/data/DataSource.kt | 271 ++++++++++++++++++ .../space/kscience/dataforge/data/DataTree.kt | 119 -------- .../dataforge/data/DataTreeBuilder.kt | 121 -------- .../space/kscience/dataforge/data/Goal.kt | 8 +- .../kscience/dataforge/data/GroupRule.kt | 41 +-- .../kscience/dataforge/data/NamedData.kt | 6 +- .../kscience/dataforge/data/StaticDataTree.kt | 80 ------ .../kscience/dataforge/data/dataBuilders.kt | 121 ++++++++ .../kscience/dataforge/data/dataFilter.kt | 105 ------- .../kscience/dataforge/data/dataTransform.kt | 62 ++-- .../kscience/dataforge/data/dataFilterJvm.kt | 92 +++--- .../dataforge/data/dataSetBuilderInContext.kt | 30 +- .../kscience/dataforge/data/ActionsTest.kt | 3 +- ...kt => LegacyGenericDataTreeBuilderTest.kt} | 12 +- .../space/kscience/dataforge/io/IOFormat.kt | 17 +- .../space/kscience/dataforge/io/IOPlugin.kt | 5 +- .../kscience/dataforge/meta/MetaConverter.kt | 32 +-- .../space/kscience/dataforge/meta/Scheme.kt | 6 +- .../space/kscience/dataforge/names/Name.kt | 7 + .../dataforge/workspace/EnvelopeTask.kt | 46 --- .../kscience/dataforge/workspace/Task.kt | 19 +- .../kscience/dataforge/workspace/TaskData.kt | 50 ---- .../dataforge/workspace/TaskResult.kt | 55 +--- .../kscience/dataforge/workspace/Workspace.kt | 23 +- .../dataforge/workspace/WorkspaceBuilder.kt | 21 +- .../dataforge/workspace/WorkspaceCache.kt | 2 +- .../dataforge/workspace/WorkspaceImpl.kt | 6 +- .../dataforge/workspace/envelopeData.kt | 3 +- .../dataforge/workspace/taskBuilders.kt | 29 +- .../dataforge/workspace/FileWorkspaceCache.kt | 24 +- .../workspace/InMemoryWorkspaceCache.kt | 2 +- .../kscience/dataforge/workspace/fileData.kt | 135 ++++----- .../dataforge/workspace/workspaceJvm.kt | 13 +- .../kscience/dataforge/workspace/zipData.kt | 6 +- .../workspace/DataPropagationTest.kt | 4 +- .../dataforge/workspace/FileDataTest.kt | 2 +- .../workspace/SimpleWorkspaceTest.kt | 2 +- 47 files changed, 838 insertions(+), 1255 deletions(-) create mode 100644 dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataFilter.kt delete mode 100644 dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSet.kt delete mode 100644 dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSetBuilder.kt create mode 100644 dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSource.kt delete mode 100644 dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataTree.kt delete mode 100644 dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataTreeBuilder.kt delete mode 100644 dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/StaticDataTree.kt create mode 100644 dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataBuilders.kt delete mode 100644 dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataFilter.kt rename dataforge-data/src/jvmTest/kotlin/space/kscience/dataforge/data/{DataTreeBuilderTest.kt => LegacyGenericDataTreeBuilderTest.kt} (87%) delete mode 100644 dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/EnvelopeTask.kt delete mode 100644 dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/TaskData.kt diff --git a/dataforge-context/build.gradle.kts b/dataforge-context/build.gradle.kts index b59abed0..72a41693 100644 --- a/dataforge-context/build.gradle.kts +++ b/dataforge-context/build.gradle.kts @@ -12,7 +12,7 @@ kscience { useCoroutines() useSerialization() commonMain { - api(project(":dataforge-meta")) + api(projects.dataforgeMeta) api(spclibs.atomicfu) } jvmMain{ diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/AbstractAction.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/AbstractAction.kt index e7bbe6f6..33d4ca05 100644 --- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/AbstractAction.kt +++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/AbstractAction.kt @@ -1,9 +1,9 @@ package space.kscience.dataforge.actions +import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.launch import space.kscience.dataforge.data.* import space.kscience.dataforge.meta.Meta -import space.kscience.dataforge.misc.DFInternal import space.kscience.dataforge.names.Name import space.kscience.dataforge.names.startsWith import kotlin.reflect.KType @@ -19,47 +19,40 @@ internal fun MutableMap.removeWhatStartsWith(name: Name) { /** * An action that caches results on-demand and recalculates them on source push */ -public abstract class AbstractAction( +public abstract class AbstractAction( public val outputType: KType, ) : Action { /** * Generate initial content of the output */ - protected abstract fun DataSetBuilder.generate( - data: DataSet, + protected abstract fun DataSink.generate( + data: DataTree, meta: Meta, ) /** - * Update part of the data set when given [updateKey] is triggered by the source + * Update part of the data set using provided data */ - protected open fun DataSourceBuilder.update( - dataSet: DataSet, + protected open fun DataSink.update( + allData: DataTree, meta: Meta, - updateKey: Name, - ) { - // By default, recalculate the whole dataset - generate(dataSet, meta) + namedData: NamedData, + ){ + //by default regenerate the whole data set + generate(allData,meta) } - @OptIn(DFInternal::class) override fun execute( - dataSet: DataSet, + scope: CoroutineScope, + dataSet: DataTree, meta: Meta, - ): DataSet = if (dataSet is DataSource) { - DataSource(outputType, dataSet){ - generate(dataSet, meta) - - launch { - dataSet.updates.collect { name -> - update(dataSet, meta, name) - } + ): ObservableDataTree = MutableDataTree(outputType, scope).apply { + generate(dataSet, meta) + scope.launch { + dataSet.updates().collect { + update(dataSet, meta, it) } } - } else { - DataTree(outputType) { - generate(dataSet, meta) - } } } diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/Action.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/Action.kt index 3feec9bc..b9a9a0a8 100644 --- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/Action.kt +++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/Action.kt @@ -1,19 +1,22 @@ package space.kscience.dataforge.actions -import space.kscience.dataforge.data.DataSet +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.coroutineScope +import space.kscience.dataforge.data.DataTree +import space.kscience.dataforge.data.ObservableDataTree import space.kscience.dataforge.meta.Meta import space.kscience.dataforge.misc.DFExperimental /** * A simple data transformation on a data node. Actions should avoid doing actual dependency evaluation in [execute]. */ -public fun interface Action { +public fun interface Action { /** * Transform the data in the node, producing a new node. By default, it is assumed that all calculations are lazy * so not actual computation is started at this moment. */ - public fun execute(dataSet: DataSet, meta: Meta): DataSet + public fun execute(scope: CoroutineScope, dataSet: DataTree, meta: Meta): ObservableDataTree public companion object } @@ -21,18 +24,22 @@ public fun interface Action { /** * A convenience method to transform data using given [action] */ -public fun DataSet.transform(action: Action, meta: Meta = Meta.EMPTY): DataSet = - action.execute(this, meta) +public fun DataTree.transform( + action: Action, + scope: CoroutineScope, + meta: Meta = Meta.EMPTY, +): DataTree = action.execute(scope, this, meta) /** * Action composition. The result is terminal if one of its parts is terminal */ public infix fun Action.then(action: Action): Action = - Action { dataSet, meta -> action.execute(this@then.execute(dataSet, meta), meta) } + Action { scope, dataSet, meta -> action.execute(scope, this@then.execute(scope, dataSet, meta), meta) } @DFExperimental -public operator fun Action.invoke( - dataSet: DataSet, +public suspend operator fun Action.invoke( + dataSet: DataTree, meta: Meta = Meta.EMPTY, -): DataSet = execute(dataSet, meta) +): DataTree = coroutineScope { execute(this, dataSet, meta) } + diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/MapAction.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/MapAction.kt index 7b2c94f5..60f05910 100644 --- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/MapAction.kt +++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/MapAction.kt @@ -29,6 +29,7 @@ public class MapActionBuilder( public var name: Name, public var meta: MutableMeta, public val actionMeta: Meta, + public val dataType: KType, @PublishedApi internal var outputType: KType, ) { @@ -45,19 +46,16 @@ public class MapActionBuilder( /** * Calculate the result of goal */ - public inline fun result(noinline f: suspend ActionEnv.(T) -> R1) { - outputType = typeOf() - result = f; - } + public inline fun result(noinline f: suspend ActionEnv.(T) -> R1): Unit = result(typeOf(), f) } @PublishedApi -internal class MapAction( +internal class MapAction( outputType: KType, private val block: MapActionBuilder.() -> Unit, ) : AbstractAction(outputType) { - private fun DataSetBuilder.mapOne(name: Name, data: Data, meta: Meta) { + private fun DataSink.mapOne(name: Name, data: Data, meta: Meta) { // Creating a new environment for action using **old** name, old meta and task meta val env = ActionEnv(name, data.meta, meta) @@ -66,6 +64,7 @@ internal class MapAction( name, data.meta.toMutableMeta(), // using data meta meta, + data.type, outputType ).apply(block) @@ -80,16 +79,15 @@ internal class MapAction( builder.result(env, data.await()) } //setting the data node - data(newName, newData) + emit(newName, newData) } - override fun DataSetBuilder.generate(data: DataSet, meta: Meta) { + override fun DataSink.generate(data: DataTree, meta: Meta) { data.forEach { mapOne(it.name, it.data, meta) } } - override fun DataSourceBuilder.update(dataSet: DataSet, meta: Meta, updateKey: Name) { - remove(updateKey) - dataSet[updateKey]?.let { mapOne(updateKey, it, meta) } + override fun DataSink.update(allData: DataTree, meta: Meta, namedData: NamedData) { + mapOne(namedData.name, namedData.data, namedData.meta) } } diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/ReduceAction.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/ReduceAction.kt index a74cfad9..cc27b3d1 100644 --- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/ReduceAction.kt +++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/ReduceAction.kt @@ -14,7 +14,7 @@ import kotlin.reflect.typeOf public class JoinGroup( public var name: String, - internal val set: DataSet, + internal val set: DataTree, @PublishedApi internal var outputType: KType, ) { @@ -39,7 +39,7 @@ public class ReduceGroupBuilder( public val actionMeta: Meta, private val outputType: KType, ) { - private val groupRules: MutableList<(DataSet) -> List>> = ArrayList(); + private val groupRules: MutableList<(DataTree) -> List>> = ArrayList(); /** * introduce grouping by meta value @@ -54,12 +54,12 @@ public class ReduceGroupBuilder( public fun group( groupName: String, - predicate: (Name, Meta) -> Boolean, + predicate: DataFilter, action: JoinGroup.() -> Unit, ) { groupRules += { source -> listOf( - JoinGroup(groupName, source.filter(predicate), outputType).apply(action) + JoinGroup(groupName, source.filterData(predicate), outputType).apply(action) ) } } @@ -73,7 +73,7 @@ public class ReduceGroupBuilder( } } - internal fun buildGroups(input: DataSet): List> = + internal fun buildGroups(input: DataTree): List> = groupRules.flatMap { it.invoke(input) } } @@ -85,7 +85,7 @@ internal class ReduceAction( ) : AbstractAction(outputType) { //TODO optimize reduction. Currently, the whole action recalculates on push - override fun DataSetBuilder.generate(data: DataSet, meta: Meta) { + override fun DataSink.generate(data: DataTree, meta: Meta) { ReduceGroupBuilder(meta, outputType).apply(action).buildGroups(data).forEach { group -> val dataFlow: Map> = group.set.asSequence().fold(HashMap()) { acc, value -> acc.apply { @@ -103,7 +103,7 @@ internal class ReduceAction( meta = groupMeta ) { group.result.invoke(env, it) } - data(env.name, res) + emit(env.name, res) } } } diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/SplitAction.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/SplitAction.kt index 0ecde319..a2da44f8 100644 --- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/SplitAction.kt +++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/SplitAction.kt @@ -49,7 +49,7 @@ internal class SplitAction( private val action: SplitBuilder.() -> Unit, ) : AbstractAction(outputType) { - private fun DataSetBuilder.splitOne(name: Name, data: Data, meta: Meta) { + private fun DataSink.splitOne(name: Name, data: Data, meta: Meta) { val laminate = Laminate(data.meta, meta) val split = SplitBuilder(name, data.meta).apply(action) @@ -64,7 +64,7 @@ internal class SplitAction( ).apply(rule) //data.map(outputType, meta = env.meta) { env.result(it) }.named(fragmentName) - data( + emit( fragmentName, @Suppress("OPT_IN_USAGE") Data(outputType, meta = env.meta, dependencies = listOf(data)) { env.result(data.await()) @@ -73,13 +73,12 @@ internal class SplitAction( } } - override fun DataSetBuilder.generate(data: DataSet, meta: Meta) { + override fun DataSink.generate(data: DataTree, meta: Meta) { data.forEach { splitOne(it.name, it.data, meta) } } - override fun DataSourceBuilder.update(dataSet: DataSet, meta: Meta, updateKey: Name) { - remove(updateKey) - dataSet[updateKey]?.let { splitOne(updateKey, it, meta) } + override fun DataSink.update(allData: DataTree, meta: Meta, namedData: NamedData) { + splitOne(namedData.name, namedData.data, namedData.meta) } } diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/Data.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/Data.kt index 756cd840..3668af1e 100644 --- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/Data.kt +++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/Data.kt @@ -15,7 +15,7 @@ import kotlin.reflect.typeOf * A data element characterized by its meta */ @DfType(Data.TYPE) -public interface Data : Goal, MetaRepr { +public interface Data : Goal, MetaRepr { /** * Type marker for the data. The type is known before the calculation takes place so it could be checked. */ @@ -41,7 +41,7 @@ public interface Data : Goal, MetaRepr { */ internal val TYPE_OF_NOTHING: KType = typeOf() - public inline fun static( + public inline fun static( value: T, meta: Meta = Meta.EMPTY, ): Data = StaticData(typeOf(), value, meta) @@ -69,37 +69,37 @@ public interface Data : Goal, MetaRepr { * A lazily computed variant of [Data] based on [LazyGoal] * One must ensure that proper [type] is used so this method should not be used */ -private class LazyData( +private class LazyData( override val type: KType, override val meta: Meta = Meta.EMPTY, additionalContext: CoroutineContext = EmptyCoroutineContext, - dependencies: Collection> = emptyList(), + dependencies: Iterable> = emptyList(), block: suspend () -> T, ) : Data, LazyGoal(additionalContext, dependencies, block) -public class StaticData( +public class StaticData( override val type: KType, value: T, override val meta: Meta = Meta.EMPTY, ) : Data, StaticGoal(value) @Suppress("FunctionName") -public inline fun Data(value: T, meta: Meta = Meta.EMPTY): StaticData = +public inline fun Data(value: T, meta: Meta = Meta.EMPTY): StaticData = StaticData(typeOf(), value, meta) @DFInternal -public fun Data( +public fun Data( type: KType, meta: Meta = Meta.EMPTY, context: CoroutineContext = EmptyCoroutineContext, - dependencies: Collection> = emptyList(), + dependencies: Iterable> = emptyList(), block: suspend () -> T, ): Data = LazyData(type, meta, context, dependencies, block) @OptIn(DFInternal::class) -public inline fun Data( +public inline fun Data( meta: Meta = Meta.EMPTY, context: CoroutineContext = EmptyCoroutineContext, - dependencies: Collection> = emptyList(), + dependencies: Iterable> = emptyList(), noinline block: suspend () -> T, ): Data = Data(typeOf(), meta, context, dependencies, block) diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataFilter.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataFilter.kt new file mode 100644 index 00000000..f45570ad --- /dev/null +++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataFilter.kt @@ -0,0 +1,89 @@ +package space.kscience.dataforge.data + +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.filter +import space.kscience.dataforge.meta.Meta +import space.kscience.dataforge.names.Name +import kotlin.reflect.KType + +public fun interface DataFilter { + + public fun accepts(name: Name, meta: Meta, type: KType): Boolean + + public companion object { + public val EMPTY: DataFilter = DataFilter { _, _, _ -> true } + } +} + +public fun DataFilter.accepts(data: NamedData<*>): Boolean = accepts(data.name, data.meta, data.type) + +public fun Sequence>.filterData(predicate: DataFilter): Sequence> = filter { data -> + predicate.accepts(data) +} + +public fun Flow>.filterData(predicate: DataFilter): Flow> = filter { data -> + predicate.accepts(data) +} + +public fun DataSource.filterData( + predicate: DataFilter, +): DataSource = object : DataSource { + override val dataType: KType get() = this@filterData.dataType + + override fun read(name: Name): Data? = + this@filterData.read(name)?.takeIf { predicate.accepts(name, it.meta, it.type) } +} + +/** + * Stateless filtered [ObservableDataSource] + */ +public fun ObservableDataSource.filterData( + predicate: DataFilter, +): ObservableDataSource = object : ObservableDataSource { + override fun updates(): Flow> = this@filterData.updates().filter { predicate.accepts(it) } + + override val dataType: KType get() = this@filterData.dataType + + override fun read(name: Name): Data? = + this@filterData.read(name)?.takeIf { predicate.accepts(name, it.meta, it.type) } +} + +public fun GenericDataTree.filterData( + predicate: DataFilter, +): DataTree = asSequence().filterData(predicate).toTree(dataType) + +public fun GenericObservableDataTree.filterData( + scope: CoroutineScope, + predicate: DataFilter, +): ObservableDataTree = asSequence().filterData(predicate).toObservableTree(dataType, scope, updates().filterData(predicate)) + + +///** +// * Generate a wrapper data set with a given name prefix appended to all names +// */ +//public fun DataTree.withNamePrefix(prefix: Name): DataSet = if (prefix.isEmpty()) { +// this +//} else object : DataSource { +// +// override val dataType: KType get() = this@withNamePrefix.dataType +// +// override val coroutineContext: CoroutineContext +// get() = (this@withNamePrefix as? DataSource)?.coroutineContext ?: EmptyCoroutineContext +// +// override val meta: Meta get() = this@withNamePrefix.meta +// +// +// override fun iterator(): Iterator> = iterator { +// for (d in this@withNamePrefix) { +// yield(d.data.named(prefix + d.name)) +// } +// } +// +// override fun get(name: Name): Data? = +// name.removeFirstOrNull(name)?.let { this@withNamePrefix.get(it) } +// +// override val updates: Flow get() = this@withNamePrefix.updates.map { prefix + it } +//} +// + diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSet.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSet.kt deleted file mode 100644 index 44639653..00000000 --- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSet.kt +++ /dev/null @@ -1,124 +0,0 @@ -package space.kscience.dataforge.data - -import kotlinx.coroutines.* -import kotlinx.coroutines.flow.Flow -import kotlinx.coroutines.flow.emptyFlow -import kotlinx.coroutines.flow.mapNotNull -import space.kscience.dataforge.data.Data.Companion.TYPE_OF_NOTHING -import space.kscience.dataforge.meta.Meta -import space.kscience.dataforge.names.Name -import space.kscience.dataforge.names.asName -import space.kscience.dataforge.names.endsWith -import space.kscience.dataforge.names.parseAsName -import kotlin.reflect.KType - -public interface DataSet { - - /** - * The minimal common ancestor to all data in the node - */ - public val dataType: KType - - /** - * Meta-data associated with this node. If no meta is provided, returns [Meta.EMPTY]. - */ - public val meta: Meta - - /** - * Traverse this [DataSet] returning named data instances. The order is not guaranteed. - */ - public operator fun iterator(): Iterator> - - /** - * Get data with given name. - */ - public operator fun get(name: Name): Data? - - public companion object { - public val META_KEY: Name = "@meta".asName() - - /** - * An empty [DataSet] that suits all types - */ - public val EMPTY: DataSet = object : DataSet { - override val dataType: KType = TYPE_OF_NOTHING - override val meta: Meta get() = Meta.EMPTY - - override fun iterator(): Iterator> = emptySequence>().iterator() - - override fun get(name: Name): Data? = null - } - } -} - -public fun DataSet.asSequence(): Sequence> = object : Sequence> { - override fun iterator(): Iterator> = this@asSequence.iterator() -} - -/** - * Return a single [Data] in this [DataSet]. Throw error if it is not single. - */ -public fun DataSet.single(): NamedData = asSequence().single() - -public fun DataSet.asIterable(): Iterable> = object : Iterable> { - override fun iterator(): Iterator> = this@asIterable.iterator() -} - -public operator fun DataSet.get(name: String): Data? = get(name.parseAsName()) - -/** - * A [DataSet] with propagated updates. - */ -public interface DataSource : DataSet, CoroutineScope { - - /** - * A flow of updated item names. Updates are propagated in a form of [Flow] of names of updated nodes. - * Those can include new data items and replacement of existing ones. The replaced items could update existing data content - * and replace it completely, so they should be pulled again. - * - */ - public val updates: Flow - - /** - * Stop generating updates from this [DataSource] - */ - public fun close() { - coroutineContext[Job]?.cancel() - } -} - -public val DataSet.updates: Flow get() = if (this is DataSource) updates else emptyFlow() -// -///** -// * Flow all data nodes with names starting with [branchName] -// */ -//public fun DataSet.children(branchName: Name): Sequence> = -// this@children.asSequence().filter { -// it.name.startsWith(branchName) -// } - -/** - * Start computation for all goals in data node and return a job for the whole node - */ -public fun DataSet.startAll(coroutineScope: CoroutineScope): Job = coroutineScope.launch { - asIterable().map { - it.launch(this@launch) - }.joinAll() -} - -public suspend fun DataSet.computeAndJoinAll(): Unit = coroutineScope { startAll(this).join() } - -public fun DataSet<*>.toMeta(): Meta = Meta { - forEach { - if (it.name.endsWith(DataSet.META_KEY)) { - set(it.name, it.meta) - } else { - it.name put { - "type" put it.type.toString() - "meta" put it.meta - } - } - } -} - -public val DataSet.updatesWithData: Flow> get() = updates.mapNotNull { get(it)?.named(it) } \ No newline at end of file diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSetBuilder.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSetBuilder.kt deleted file mode 100644 index f9f14f37..00000000 --- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSetBuilder.kt +++ /dev/null @@ -1,165 +0,0 @@ -package space.kscience.dataforge.data - -import space.kscience.dataforge.meta.Meta -import space.kscience.dataforge.meta.MutableMeta -import space.kscience.dataforge.misc.DFExperimental -import space.kscience.dataforge.names.Name -import space.kscience.dataforge.names.isEmpty -import space.kscience.dataforge.names.plus -import kotlin.reflect.KType - -public interface DataSetBuilder { - public val dataType: KType - - /** - * Remove all data items starting with [name] - */ - public fun remove(name: Name) - - public fun data(name: Name, data: Data?) - - /** - * Set a current state of given [dataSet] into a branch [name]. Does not propagate updates - */ - public fun node(name: Name, dataSet: DataSet) { - //remove previous items - if (name != Name.EMPTY) { - remove(name) - } - - //Set new items - dataSet.forEach { - data(name + it.name, it.data) - } - } - - /** - * Set meta for the given node - */ - public fun meta(name: Name, meta: Meta) - -} - -/** - * Define meta in this [DataSet] - */ -public fun DataSetBuilder.meta(value: Meta): Unit = meta(Name.EMPTY, value) - -/** - * Define meta in this [DataSet] - */ -public fun DataSetBuilder.meta(mutableMeta: MutableMeta.() -> Unit): Unit = meta(Meta(mutableMeta)) - -@PublishedApi -internal class SubSetBuilder( - private val parent: DataSetBuilder, - private val branch: Name, -) : DataSetBuilder { - override val dataType: KType get() = parent.dataType - - override fun remove(name: Name) { - parent.remove(branch + name) - } - - override fun data(name: Name, data: Data?) { - parent.data(branch + name, data) - } - - override fun node(name: Name, dataSet: DataSet) { - parent.node(branch + name, dataSet) - } - - override fun meta(name: Name, meta: Meta) { - parent.meta(branch + name, meta) - } -} - -public inline fun DataSetBuilder.node( - name: Name, - crossinline block: DataSetBuilder.() -> Unit, -) { - if (name.isEmpty()) block() else SubSetBuilder(this, name).block() -} - - -public fun DataSetBuilder.data(name: String, value: Data) { - data(Name.parse(name), value) -} - -public fun DataSetBuilder.node(name: String, set: DataSet) { - node(Name.parse(name), set) -} - -public inline fun DataSetBuilder.node( - name: String, - crossinline block: DataSetBuilder.() -> Unit, -): Unit = node(Name.parse(name), block) - -public fun DataSetBuilder.set(value: NamedData) { - data(value.name, value.data) -} - -/** - * Produce lazy [Data] and emit it into the [DataSetBuilder] - */ -public inline fun DataSetBuilder.produce( - name: String, - meta: Meta = Meta.EMPTY, - noinline producer: suspend () -> T, -) { - val data = Data(meta, block = producer) - data(name, data) -} - -public inline fun DataSetBuilder.produce( - name: Name, - meta: Meta = Meta.EMPTY, - noinline producer: suspend () -> T, -) { - val data = Data(meta, block = producer) - data(name, data) -} - -/** - * Emit a static data with the fixed value - */ -public inline fun DataSetBuilder.static( - name: String, - data: T, - meta: Meta = Meta.EMPTY, -): Unit = data(name, Data.static(data, meta)) - -public inline fun DataSetBuilder.static( - name: Name, - data: T, - meta: Meta = Meta.EMPTY, -): Unit = data(name, Data.static(data, meta)) - -public inline fun DataSetBuilder.static( - name: String, - data: T, - mutableMeta: MutableMeta.() -> Unit, -): Unit = data(Name.parse(name), Data.static(data, Meta(mutableMeta))) - -/** - * Update data with given node data and meta with node meta. - */ -@DFExperimental -public fun DataSetBuilder.populateFrom(tree: DataSet): Unit { - tree.forEach { - //TODO check if the place is occupied - data(it.name, it.data) - } -} - -//public fun DataSetBuilder.populateFrom(flow: Flow>) { -// flow.collect { -// data(it.name, it.data) -// } -//} - -public fun DataSetBuilder.populateFrom(sequence: Sequence>) { - sequence.forEach { - data(it.name, it.data) - } -} diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSource.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSource.kt new file mode 100644 index 00000000..8c86d431 --- /dev/null +++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSource.kt @@ -0,0 +1,271 @@ +package space.kscience.dataforge.data + +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.flow.* +import kotlinx.coroutines.launch +import space.kscience.dataforge.meta.Meta +import space.kscience.dataforge.misc.DFInternal +import space.kscience.dataforge.names.* +import kotlin.reflect.KType +import kotlin.reflect.typeOf + +/** + * A generic data provider + */ +public interface DataSource { + + /** + * The minimal common ancestor to all data in the node + */ + public val dataType: KType + + /** + * Get data with given name. Or null if it is not present + */ + public fun read(name: Name): Data? +} + +/** + * A data provider with possible dynamic updates + */ +public interface ObservableDataSource : DataSource { + + /** + * Flow updates made to the data + */ + public fun updates(): Flow> +} + +/** + * A tree like structure for data holding + */ +public interface GenericDataTree> : DataSource { + public val self: TR + + public val data: Data? + public val items: Map + + + override fun read(name: Name): Data? = when (name.length) { + 0 -> data + else -> items[name.first()]?.read(name.cutFirst()) + } + + public companion object { + private object EmptyDataTree : GenericDataTree { + override val self: EmptyDataTree get() = this + override val data: Data? = null + override val items: Map = emptyMap() + override val dataType: KType = typeOf() + + override fun read(name: Name): Data? = null + + } + + public val EMPTY: GenericDataTree = EmptyDataTree + } +} + +public typealias DataTree = GenericDataTree + +/** + * An alias for easier access to tree values + */ +public operator fun DataTree.get(name: Name): Data? = read(name) + +public operator fun DataTree.get(name: String): Data? = read(name.parseAsName()) + +/** + * Return a sequence of all data items in this tree. + * This method does not take updates into account. + */ +public fun GenericDataTree>.asSequence( + namePrefix: Name = Name.EMPTY, +): Sequence> = sequence { + data?.let { yield(it.named(Name.EMPTY)) } + items.forEach { (token, tree) -> + yieldAll(tree.asSequence(namePrefix + token)) + } +} + +public val DataTree<*>.meta: Meta? get() = data?.meta + +/** + * Provide subtree if it exists + */ +public tailrec fun > GenericDataTree.branch(name: Name): TR? = + when (name.length) { + 0 -> self + 1 -> items[name.first()] + else -> items[name.first()]?.branch(name.cutFirst()) + } + +public fun GenericDataTree<*, *>.isEmpty(): Boolean = data == null && items.isEmpty() + +@PublishedApi +internal class FlatDataTree( + override val dataType: KType, + val dataSet: Map>, + val prefix: Name, +) : GenericDataTree> { + override val self: FlatDataTree get() = this + override val data: Data? get() = dataSet[prefix] + override val items: Map> + get() = dataSet.keys + .filter { it.startsWith(prefix) && it.length > prefix.length } + .map { it.tokens[prefix.length + 1] } + .associateWith { FlatDataTree(dataType, dataSet, prefix + it) } + + override fun read(name: Name): Data? = dataSet[prefix + name] +} + +/** + * Represent this flat data map as a [DataTree] without copying it + */ +public inline fun Map>.asTree(): DataTree = FlatDataTree(typeOf(), this, Name.EMPTY) + +internal fun Sequence>.toTree(type: KType): DataTree = + FlatDataTree(type, associate { it.name to it.data }, Name.EMPTY) + +/** + * Collect a sequence of [NamedData] to a [DataTree] + */ +public inline fun Sequence>.toTree(): DataTree = + FlatDataTree(typeOf(), associate { it.name to it.data }, Name.EMPTY) + +public interface GenericObservableDataTree> : GenericDataTree, + ObservableDataSource + +public typealias ObservableDataTree = GenericObservableDataTree + +public fun DataTree.updates(): Flow> = if (this is ObservableDataTree) updates() else emptyFlow() + +public fun interface DataSink { + public fun emit(name: Name, data: Data?) +} + +public class DataTreeBuilder(private val type: KType) : DataSink { + private val map = HashMap>() + override fun emit(name: Name, data: Data?) { + if (data == null) { + map.remove(name) + } else { + map[name] = data + } + } + + public fun build(): DataTree = FlatDataTree(type, map, Name.EMPTY) +} + +@DFInternal +public inline fun DataTree( + dataType: KType, + generator: DataSink.() -> Unit, +): DataTree = DataTreeBuilder(dataType).apply(generator).build() + +/** + * Create and a data tree. + */ +public inline fun DataTree( + generator: DataSink.() -> Unit, +): DataTree = DataTreeBuilder(typeOf()).apply(generator).build() + +/** + * A mutable version of [GenericDataTree] + */ +public interface MutableDataTree : GenericObservableDataTree>, DataSink { + public val scope: CoroutineScope + + override var data: Data? + + override val items: Map> + + public operator fun set(token: NameToken, data: Data?) + + override fun emit(name: Name, data: Data?): Unit = set(name, data) +} + +public tailrec operator fun MutableDataTree.set(name: Name, data: Data?): Unit { + when (name.length) { + 0 -> this.data = data + 1 -> set(name.first(), data) + else -> items[name.first()]?.set(name.cutFirst(), data) + } +} + +private class ObservableMutableDataTreeImpl( + override val dataType: KType, + override val scope: CoroutineScope, +) : MutableDataTree { + + private val updates = MutableSharedFlow>() + + private val children = HashMap>() + + override var data: Data? = null + set(value) { + field = value + if (value != null) { + scope.launch { + updates.emit(value.named(Name.EMPTY)) + } + } + } + + override val items: Map> get() = children + override val self: MutableDataTree get() = this + + override fun set(token: NameToken, data: Data?) { + children.getOrPut(token) { + ObservableMutableDataTreeImpl(dataType, scope).also { subTree -> + subTree.updates().onEach { + updates.emit(it.named(token + it.name)) + }.launchIn(scope) + } + }.data = data + } + + override fun updates(): Flow> = flow { + //emit this node updates + updates.collect { + emit(it) + } + } +} + +public fun MutableDataTree( + type: KType, + scope: CoroutineScope +): MutableDataTree = ObservableMutableDataTreeImpl(type, scope) + +/** + * Create and initialize a observable mutable data tree. + */ +public inline fun MutableDataTree( + scope: CoroutineScope, + generator: MutableDataTree.() -> Unit = {}, +): MutableDataTree = MutableDataTree(typeOf(), scope).apply { generator() } + +//@DFInternal +//public fun ObservableDataTree( +// type: KType, +// scope: CoroutineScope, +// generator: suspend MutableDataTree.() -> Unit = {}, +//): ObservableDataTree = MutableDataTree(type, scope.coroutineContext).apply(generator) + +public inline fun ObservableDataTree( + scope: CoroutineScope, + generator: MutableDataTree.() -> Unit = {}, +): ObservableDataTree = MutableDataTree(typeOf(), scope).apply(generator) + + +/** + * Collect a [Sequence] into an observable tree with additional [updates] + */ +public fun Sequence>.toObservableTree(dataType: KType, scope: CoroutineScope, updates: Flow>): ObservableDataTree = + MutableDataTree(dataType, scope).apply { + emitAll(this@toObservableTree) + updates.onEach { + emit(it.name, it.data) + }.launchIn(scope) + } diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataTree.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataTree.kt deleted file mode 100644 index b9273c07..00000000 --- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataTree.kt +++ /dev/null @@ -1,119 +0,0 @@ -package space.kscience.dataforge.data - -import space.kscience.dataforge.meta.Meta -import space.kscience.dataforge.misc.DFInternal -import space.kscience.dataforge.misc.DfType -import space.kscience.dataforge.names.* -import kotlin.collections.component1 -import kotlin.collections.component2 -import kotlin.reflect.KType -import kotlin.reflect.typeOf - -public sealed class DataTreeItem { - - public abstract val meta: Meta - - public class Node(public val tree: DataTree) : DataTreeItem() { - override val meta: Meta get() = tree.meta - } - - public class Leaf(public val data: Data) : DataTreeItem() { - override val meta: Meta get() = data.meta - } -} - -public val DataTreeItem.type: KType - get() = when (this) { - is DataTreeItem.Node -> tree.dataType - is DataTreeItem.Leaf -> data.type - } - -/** - * A tree-like [DataSet] grouped into the node. All data inside the node must inherit its type - */ -@DfType(DataTree.TYPE) -public interface DataTree : DataSet { - - /** - * Top-level children items of this [DataTree] - */ - public val items: Map> - - override val meta: Meta get() = items[META_ITEM_NAME_TOKEN]?.meta ?: Meta.EMPTY - - override fun iterator(): Iterator> = iterator { - items.forEach { (token, childItem: DataTreeItem) -> - if (!token.body.startsWith("@")) { - when (childItem) { - is DataTreeItem.Leaf -> yield(childItem.data.named(token.asName())) - is DataTreeItem.Node -> yieldAll(childItem.tree.asSequence().map { it.named(token + it.name) }) - } - } - } - } - - override fun get(name: Name): Data? = when (name.length) { - 0 -> null - 1 -> items[name.firstOrNull()!!].data - else -> items[name.firstOrNull()!!].tree?.get(name.cutFirst()) - } - - public companion object { - public const val TYPE: String = "dataTree" - - /** - * A name token used to designate tree node meta - */ - public val META_ITEM_NAME_TOKEN: NameToken = NameToken("@meta") - - @DFInternal - public fun emptyWithType(type: KType, meta: Meta = Meta.EMPTY): DataTree = object : DataTree { - override val items: Map> get() = emptyMap() - override val dataType: KType get() = type - override val meta: Meta get() = meta - } - - @OptIn(DFInternal::class) - public inline fun empty(meta: Meta = Meta.EMPTY): DataTree = - emptyWithType(typeOf(), meta) - } -} - -public fun DataTree.listChildren(prefix: Name): List = - getItem(prefix).tree?.items?.keys?.map { prefix + it } ?: emptyList() - -/** - * Get a [DataTreeItem] with given [name] or null if the item does not exist - */ -public tailrec fun DataTree.getItem(name: Name): DataTreeItem? = when (name.length) { - 0 -> DataTreeItem.Node(this) - 1 -> items[name.firstOrNull()] - else -> items[name.firstOrNull()!!].tree?.getItem(name.cutFirst()) -} - -public val DataTreeItem?.tree: DataTree? get() = (this as? DataTreeItem.Node)?.tree -public val DataTreeItem?.data: Data? get() = (this as? DataTreeItem.Leaf)?.data - -/** - * A [Sequence] of all children including nodes - */ -public fun DataTree.traverseItems(): Sequence>> = sequence { - items.forEach { (head, item) -> - yield(head.asName() to item) - if (item is DataTreeItem.Node) { - val subSequence = item.tree.traverseItems() - .map { (name, data) -> (head.asName() + name) to data } - yieldAll(subSequence) - } - } -} - -/** - * Get a branch of this [DataTree] with a given [branchName]. - * The difference from similar method for [DataSet] is that internal logic is more simple and the return value is a [DataTree] - */ -@OptIn(DFInternal::class) -public fun DataTree.branch(branchName: Name): DataTree = - getItem(branchName)?.tree ?: DataTree.emptyWithType(dataType) - -public fun DataTree.branch(branchName: String): DataTree = branch(branchName.parseAsName()) diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataTreeBuilder.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataTreeBuilder.kt deleted file mode 100644 index f8a5dd4c..00000000 --- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataTreeBuilder.kt +++ /dev/null @@ -1,121 +0,0 @@ -package space.kscience.dataforge.data - -import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.Job -import kotlinx.coroutines.flow.MutableSharedFlow -import kotlinx.coroutines.launch -import space.kscience.dataforge.meta.Meta -import space.kscience.dataforge.misc.DFInternal -import space.kscience.dataforge.misc.ThreadSafe -import space.kscience.dataforge.names.* -import kotlin.collections.set -import kotlin.coroutines.CoroutineContext -import kotlin.reflect.KType -import kotlin.reflect.typeOf - -public interface DataSourceBuilder : DataSetBuilder, DataSource { - override val updates: MutableSharedFlow -} - -/** - * A mutable [DataTree] that propagates updates - */ -public class DataTreeBuilder internal constructor( - override val dataType: KType, - coroutineContext: CoroutineContext, -) : DataTree, DataSourceBuilder { - - override val coroutineContext: CoroutineContext = - coroutineContext + Job(coroutineContext[Job]) + GoalExecutionRestriction() - - private val treeItems = HashMap>() - - override val items: Map> - get() = treeItems.filter { !it.key.body.startsWith("@") } - - override val updates: MutableSharedFlow = MutableSharedFlow() - - @ThreadSafe - private fun remove(token: NameToken) { - if (treeItems.remove(token) != null) { - launch { - updates.emit(token.asName()) - } - } - } - - override fun remove(name: Name) { - if (name.isEmpty()) error("Can't remove the root node") - (getItem(name.cutLast()).tree as? DataTreeBuilder)?.remove(name.lastOrNull()!!) - } - - @ThreadSafe - private fun set(token: NameToken, data: Data) { - treeItems[token] = DataTreeItem.Leaf(data) - } - - @ThreadSafe - private fun set(token: NameToken, node: DataTree) { - treeItems[token] = DataTreeItem.Node(node) - } - - private fun getOrCreateNode(token: NameToken): DataTreeBuilder = - (treeItems[token] as? DataTreeItem.Node)?.tree as? DataTreeBuilder - ?: DataTreeBuilder(dataType, coroutineContext).also { set(token, it) } - - private fun getOrCreateNode(name: Name): DataTreeBuilder = when (name.length) { - 0 -> this - 1 -> getOrCreateNode(name.firstOrNull()!!) - else -> getOrCreateNode(name.firstOrNull()!!).getOrCreateNode(name.cutFirst()) - } - - override fun data(name: Name, data: Data?) { - if (data == null) { - remove(name) - } else { - when (name.length) { - 0 -> error("Can't add data with empty name") - 1 -> set(name.firstOrNull()!!, data) - 2 -> getOrCreateNode(name.cutLast()).set(name.lastOrNull()!!, data) - } - } - launch { - updates.emit(name) - } - } - - override fun meta(name: Name, meta: Meta) { - val item = getItem(name) - if (item is DataTreeItem.Leaf) error("TODO: Can't change meta of existing leaf item.") - data(name + DataTree.META_ITEM_NAME_TOKEN, Data.empty(meta)) - } -} - -/** - * Create a dynamic [DataSource]. Initial data is placed synchronously. - */ -@DFInternal -@Suppress("FunctionName") -public fun DataSource( - type: KType, - parent: CoroutineScope, - block: DataSourceBuilder.() -> Unit = {}, -): DataTreeBuilder = DataTreeBuilder(type, parent.coroutineContext).apply(block) - -@Suppress("OPT_IN_USAGE", "FunctionName") -public inline fun DataSource( - parent: CoroutineScope, - crossinline block: DataSourceBuilder.() -> Unit = {}, -): DataTreeBuilder = DataSource(typeOf(), parent) { block() } - -public inline fun DataSourceBuilder.emit( - name: Name, - parent: CoroutineScope, - noinline block: DataSourceBuilder.() -> Unit, -): Unit = node(name, DataSource(parent, block)) - -public inline fun DataSourceBuilder.emit( - name: String, - parent: CoroutineScope, - noinline block: DataSourceBuilder.() -> Unit, -): Unit = node(Name.parse(name), DataSource(parent, block)) \ No newline at end of file diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/Goal.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/Goal.kt index 678711c1..95ddbbf7 100644 --- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/Goal.kt +++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/Goal.kt @@ -9,7 +9,7 @@ import kotlin.coroutines.EmptyCoroutineContext * Lazy computation result with its dependencies to allowing to stat computing dependencies ahead of time */ public interface Goal { - public val dependencies: Collection> + public val dependencies: Iterable> /** * Returns current running coroutine if the goal is started. Null if the computation is not started. @@ -54,7 +54,7 @@ public open class StaticGoal(public val value: T) : Goal { */ public open class LazyGoal( private val coroutineContext: CoroutineContext = EmptyCoroutineContext, - override val dependencies: Collection> = emptyList(), + override val dependencies: Iterable> = emptyList(), public val block: suspend () -> T, ) : Goal { @@ -82,8 +82,8 @@ public open class LazyGoal( } log?.emit { "Starting dependencies computation for ${this@LazyGoal}" } - val startedDependencies = this.dependencies.map { goal -> - goal.run { async(coroutineScope) } + val startedDependencies = dependencies.map { goal -> + goal.async(coroutineScope) } return deferred ?: coroutineScope.async( coroutineContext diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/GroupRule.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/GroupRule.kt index 189087a3..27d27507 100644 --- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/GroupRule.kt +++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/GroupRule.kt @@ -15,13 +15,11 @@ */ package space.kscience.dataforge.data -import kotlinx.coroutines.launch import space.kscience.dataforge.meta.get import space.kscience.dataforge.meta.string -import space.kscience.dataforge.misc.DFInternal public interface GroupRule { - public fun gather(set: DataSet): Map> + public fun gather(set: DataTree): Map> public companion object { /** @@ -32,46 +30,23 @@ public interface GroupRule { * @param defaultTagValue * @return */ - @OptIn(DFInternal::class) public fun byMetaValue( key: String, defaultTagValue: String, ): GroupRule = object : GroupRule { override fun gather( - set: DataSet, - ): Map> { - val map = HashMap>() + set: DataTree, + ): Map> { + val map = HashMap>() - if (set is DataSource) { - set.forEach { data -> - val tagValue: String = data.meta[key]?.string ?: defaultTagValue - (map.getOrPut(tagValue) { DataTreeBuilder(set.dataType, set.coroutineContext) } as DataTreeBuilder) - .data(data.name, data.data) - - set.launch { - set.updates.collect { name -> - val dataUpdate = set[name] - - val updateTagValue = dataUpdate?.meta?.get(key)?.string ?: defaultTagValue - map.getOrPut(updateTagValue) { - DataSource(set.dataType, this) { - data(name, dataUpdate) - } - } - } - } - } - } else { - set.forEach { data -> - val tagValue: String = data.meta[key]?.string ?: defaultTagValue - (map.getOrPut(tagValue) { StaticDataTree(set.dataType) } as StaticDataTree) - .data(data.name, data.data) - } + set.forEach { data -> + val tagValue: String = data.meta[key]?.string ?: defaultTagValue + map.getOrPut(tagValue) { DataTreeBuilder(set.dataType) }.emit(data.name,data.data) } - return map + return map.mapValues { it.value.build() } } } } diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/NamedData.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/NamedData.kt index 4c9d4bb3..26ad2dfb 100644 --- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/NamedData.kt +++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/NamedData.kt @@ -4,7 +4,7 @@ import space.kscience.dataforge.meta.isEmpty import space.kscience.dataforge.misc.Named import space.kscience.dataforge.names.Name -public interface NamedData : Named, Data { +public interface NamedData : Named, Data { override val name: Name public val data: Data } @@ -12,7 +12,7 @@ public interface NamedData : Named, Data { public operator fun NamedData<*>.component1(): Name = name public operator fun NamedData.component2(): Data = data -private class NamedDataImpl( +private class NamedDataImpl( override val name: Name, override val data: Data, ) : Data by data, NamedData { @@ -28,7 +28,7 @@ private class NamedDataImpl( } } -public fun Data.named(name: Name): NamedData = if (this is NamedData) { +public fun Data.named(name: Name): NamedData = if (this is NamedData) { NamedDataImpl(name, this.data) } else { NamedDataImpl(name, this) diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/StaticDataTree.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/StaticDataTree.kt deleted file mode 100644 index 186e5c2a..00000000 --- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/StaticDataTree.kt +++ /dev/null @@ -1,80 +0,0 @@ -package space.kscience.dataforge.data - -import space.kscience.dataforge.meta.Meta -import space.kscience.dataforge.misc.DFExperimental -import space.kscience.dataforge.names.* -import kotlin.reflect.KType -import kotlin.reflect.typeOf - -@PublishedApi -internal class StaticDataTree( - override val dataType: KType, -) : DataSetBuilder, DataTree { - - private val _items: MutableMap> = HashMap() - - override val items: Map> - get() = _items.filter { !it.key.body.startsWith("@") } - - override fun remove(name: Name) { - when (name.length) { - 0 -> error("Can't remove root tree node") - 1 -> _items.remove(name.firstOrNull()!!) - else -> (_items[name.firstOrNull()!!].tree as? StaticDataTree)?.remove(name.cutFirst()) - } - } - - private fun getOrCreateNode(name: Name): StaticDataTree = when (name.length) { - 0 -> this - 1 -> { - val itemName = name.firstOrNull()!! - (_items[itemName].tree as? StaticDataTree) ?: StaticDataTree(dataType).also { - _items[itemName] = DataTreeItem.Node(it) - } - } - else -> getOrCreateNode(name.cutLast()).getOrCreateNode(name.lastOrNull()!!.asName()) - } - - private fun set(name: Name, item: DataTreeItem?) { - if (name.isEmpty()) error("Can't set top level tree node") - if (item == null) { - remove(name) - } else { - getOrCreateNode(name.cutLast())._items[name.lastOrNull()!!] = item - } - } - - override fun data(name: Name, data: Data?) { - set(name, data?.let { DataTreeItem.Leaf(it) }) - } - - override fun node(name: Name, dataSet: DataSet) { - if (dataSet is StaticDataTree) { - set(name, DataTreeItem.Node(dataSet)) - } else { - dataSet.forEach { - data(name + it.name, it.data) - } - } - } - - override fun meta(name: Name, meta: Meta) { - val item = getItem(name) - if (item is DataTreeItem.Leaf) TODO("Can't change meta of existing leaf item.") - data(name + DataTree.META_ITEM_NAME_TOKEN, Data.empty(meta)) - } -} - -public inline fun DataTree( - dataType: KType, - block: DataSetBuilder.() -> Unit, -): DataTree = StaticDataTree(dataType).apply { block() } - -public inline fun DataTree( - noinline block: DataSetBuilder.() -> Unit, -): DataTree = DataTree(typeOf(), block) - -@OptIn(DFExperimental::class) -public fun DataSet.seal(): DataTree = DataTree(dataType) { - populateFrom(this@seal) -} \ No newline at end of file diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataBuilders.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataBuilders.kt new file mode 100644 index 00000000..7eecad8f --- /dev/null +++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataBuilders.kt @@ -0,0 +1,121 @@ +package space.kscience.dataforge.data + +import kotlinx.coroutines.Job +import kotlinx.coroutines.flow.launchIn +import kotlinx.coroutines.flow.onEach +import space.kscience.dataforge.meta.Meta +import space.kscience.dataforge.meta.MutableMeta +import space.kscience.dataforge.misc.DFExperimental +import space.kscience.dataforge.names.Name +import space.kscience.dataforge.names.isEmpty +import space.kscience.dataforge.names.plus + + +public fun DataSink.emit(value: NamedData) { + emit(value.name, value.data) +} + +public fun DataSink.emitAll(sequence: Sequence>) { + sequence.forEach { emit(it) } +} + +public fun DataSink.emitAll(dataTree: DataTree) { + emitAll(dataTree.asSequence()) +} + +public inline fun DataSink.emitAll( + prefix: Name, + block: DataSink.() -> Unit, +) { + if (prefix.isEmpty()) { + apply(block) + } else { + val proxyDataSink = DataSink { nameWithoutPrefix, data -> + this.emit(prefix + nameWithoutPrefix, data) + } + + proxyDataSink.apply(block) + } +} + + +public fun DataSink.emit(name: String, value: Data) { + emit(Name.parse(name), value) +} + +public fun DataSink.emitAll(name: Name, set: DataTree) { + emitAll(name) { emitAll(set.asSequence()) } +} + +public fun DataSink.emitAll(name: String, set: DataTree) { + emitAll(Name.parse(name)) { emitAll(set.asSequence()) } +} + +/** + * Produce lazy [Data] and emit it into the [MutableDataTree] + */ +public inline fun DataSink.produce( + name: String, + meta: Meta = Meta.EMPTY, + noinline producer: suspend () -> T, +) { + val data = Data(meta, block = producer) + emit(name, data) +} + +public inline fun DataSink.produce( + name: Name, + meta: Meta = Meta.EMPTY, + noinline producer: suspend () -> T, +) { + val data = Data(meta, block = producer) + emit(name, data) +} + +/** + * Emit static data with the fixed value + */ +public inline fun DataSink.static( + name: String, + data: T, + meta: Meta = Meta.EMPTY, +): Unit = emit(name, Data.static(data, meta)) + +public inline fun DataSink.static( + name: Name, + data: T, + meta: Meta = Meta.EMPTY, +): Unit = emit(name, Data.static(data, meta)) + +public inline fun DataSink.static( + name: String, + data: T, + mutableMeta: MutableMeta.() -> Unit, +): Unit = emit(Name.parse(name), Data.static(data, Meta(mutableMeta))) + + +public fun DataSink.populateFrom(sequence: Sequence>) { + sequence.forEach { + emit(it.name, it.data) + } +} + +public fun DataSink.populateFrom(tree: DataTree) { + populateFrom(tree.asSequence()) +} + + +/** + * Update data with given node data and meta with node meta. + */ +@DFExperimental +public fun MutableDataTree.populateFrom(flow: ObservableDataSource): Job = flow.updates().onEach { + //TODO check if the place is occupied + emit(it.name, it.data) +}.launchIn(scope) + +//public fun DataSetBuilder.populateFrom(flow: Flow>) { +// flow.collect { +// data(it.name, it.data) +// } +//} diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataFilter.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataFilter.kt deleted file mode 100644 index 3d2b6537..00000000 --- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataFilter.kt +++ /dev/null @@ -1,105 +0,0 @@ -package space.kscience.dataforge.data - -import kotlinx.coroutines.flow.Flow -import kotlinx.coroutines.flow.filter -import kotlinx.coroutines.flow.map -import kotlinx.coroutines.flow.mapNotNull -import space.kscience.dataforge.meta.Meta -import space.kscience.dataforge.misc.DFExperimental -import space.kscience.dataforge.names.* -import kotlin.coroutines.CoroutineContext -import kotlin.coroutines.EmptyCoroutineContext -import kotlin.reflect.KType - - -/** - * A stateless filtered [DataSet] - */ -public fun DataSet.filter( - predicate: (Name, Meta) -> Boolean, -): DataSource = object : DataSource { - - override val dataType: KType get() = this@filter.dataType - - override val coroutineContext: CoroutineContext - get() = (this@filter as? DataSource)?.coroutineContext ?: EmptyCoroutineContext - - - override val meta: Meta get() = this@filter.meta - - override fun iterator(): Iterator> = iterator { - for (d in this@filter) { - if (predicate(d.name, d.meta)) { - yield(d) - } - } - } - - override fun get(name: Name): Data? = this@filter.get(name)?.takeIf { - predicate(name, it.meta) - } - - override val updates: Flow = this@filter.updates.filter flowFilter@{ name -> - val theData = this@filter[name] ?: return@flowFilter false - predicate(name, theData.meta) - } -} - -/** - * Generate a wrapper data set with a given name prefix appended to all names - */ -public fun DataSet.withNamePrefix(prefix: Name): DataSet = if (prefix.isEmpty()) { - this -} else object : DataSource { - - override val dataType: KType get() = this@withNamePrefix.dataType - - override val coroutineContext: CoroutineContext - get() = (this@withNamePrefix as? DataSource)?.coroutineContext ?: EmptyCoroutineContext - - override val meta: Meta get() = this@withNamePrefix.meta - - - override fun iterator(): Iterator> = iterator { - for (d in this@withNamePrefix) { - yield(d.data.named(prefix + d.name)) - } - } - - override fun get(name: Name): Data? = - name.removeFirstOrNull(name)?.let { this@withNamePrefix.get(it) } - - override val updates: Flow get() = this@withNamePrefix.updates.map { prefix + it } -} - -/** - * Get a subset of data starting with a given [branchName] - */ -public fun DataSet.branch(branchName: Name): DataSet = if (branchName.isEmpty()) { - this -} else object : DataSource { - override val dataType: KType get() = this@branch.dataType - - override val coroutineContext: CoroutineContext - get() = (this@branch as? DataSource)?.coroutineContext ?: EmptyCoroutineContext - - override val meta: Meta get() = this@branch.meta - - override fun iterator(): Iterator> = iterator { - for (d in this@branch) { - d.name.removeFirstOrNull(branchName)?.let { name -> - yield(d.data.named(name)) - } - } - } - - override fun get(name: Name): Data? = this@branch.get(branchName + name) - - override val updates: Flow get() = this@branch.updates.mapNotNull { it.removeFirstOrNull(branchName) } -} - -public fun DataSet.branch(branchName: String): DataSet = this@branch.branch(branchName.parseAsName()) - -@DFExperimental -public suspend fun DataSet.rootData(): Data? = get(Name.EMPTY) - diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataTransform.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataTransform.kt index c1ecdc09..3afc716a 100644 --- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataTransform.kt +++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataTransform.kt @@ -11,11 +11,11 @@ import kotlin.reflect.typeOf public data class ValueWithMeta(val meta: Meta, val value: T) -public suspend fun Data.awaitWithMeta(): ValueWithMeta = ValueWithMeta(meta, await()) +public suspend fun Data.awaitWithMeta(): ValueWithMeta = ValueWithMeta(meta, await()) public data class NamedValueWithMeta(val name: Name, val meta: Meta, val value: T) -public suspend fun NamedData.awaitWithMeta(): NamedValueWithMeta = +public suspend fun NamedData.awaitWithMeta(): NamedValueWithMeta = NamedValueWithMeta(name, meta, await()) @@ -25,7 +25,7 @@ public suspend fun NamedData.awaitWithMeta(): NamedValueWithMeta * @param meta for the resulting data. By default equals input data. * @param block the transformation itself */ -public inline fun Data.map( +public inline fun Data.transform( meta: Meta = this.meta, coroutineContext: CoroutineContext = EmptyCoroutineContext, crossinline block: suspend (T) -> R, @@ -58,20 +58,7 @@ internal fun Iterable>.joinMeta(): Meta = Meta { } } -/** - * Lazily reduce a collection of [Data] to a single data. - */ -public inline fun Collection>.reduceToData( - meta: Meta = joinMeta(), - coroutineContext: CoroutineContext = EmptyCoroutineContext, - crossinline block: suspend (List>) -> R, -): Data = Data( - meta, - coroutineContext, - this -) { - block(map { it.awaitWithMeta() }) -} + @PublishedApi internal fun Map<*, Data<*>>.joinMeta(): Meta = Meta { @@ -195,53 +182,52 @@ public inline fun Iterable>.foldNamedToD //DataSet operations @DFInternal -public suspend fun DataSet.map( +public suspend fun DataTree.transform( outputType: KType, metaTransform: MutableMeta.() -> Unit = {}, coroutineContext: CoroutineContext = EmptyCoroutineContext, block: suspend (NamedValueWithMeta) -> R, -): DataTree = DataTree(outputType) { - forEach { - val newMeta = it.meta.toMutableMeta().apply(metaTransform).seal() - val d = Data(outputType, newMeta, coroutineContext, listOf(it)) { - block(it.awaitWithMeta()) +): DataTree = DataTree(outputType){ + //quasi-synchronous processing of elements in the tree + asSequence().forEach { namedData: NamedData -> + val newMeta = namedData.meta.toMutableMeta().apply(metaTransform).seal() + val d = Data(outputType, newMeta, coroutineContext, listOf(namedData)) { + block(namedData.awaitWithMeta()) } - data(it.name, d) + emit(namedData.name, d) } } @OptIn(DFInternal::class) -public suspend inline fun DataSet.map( +public suspend inline fun DataTree.transform( noinline metaTransform: MutableMeta.() -> Unit = {}, coroutineContext: CoroutineContext = EmptyCoroutineContext, noinline block: suspend (NamedValueWithMeta) -> R, -): DataTree = map(typeOf(), metaTransform, coroutineContext, block) +): DataTree = this@transform.transform(typeOf(), metaTransform, coroutineContext, block) -public inline fun DataSet.forEach(block: (NamedData) -> Unit) { - for (d in this) { - block(d) - } +public inline fun DataTree.forEach(block: (NamedData) -> Unit) { + asSequence().forEach(block) } // DataSet reduction @PublishedApi -internal fun DataSet<*>.joinMeta(): Meta = Meta { - forEach { (key, data) -> - val token = NameToken("data", key.toString()) - set(token, data.meta) +internal fun DataTree<*>.joinMeta(): Meta = Meta { + asSequence().forEach { + val token = NameToken("data", it.name.toString()) + set(token, it.meta) } } -public inline fun DataSet.reduceToData( +public inline fun DataTree.reduceToData( meta: Meta = joinMeta(), coroutineContext: CoroutineContext = EmptyCoroutineContext, crossinline transformation: suspend (Iterable>) -> R, -): Data = asIterable().reduceNamedToData(meta, coroutineContext, transformation) +): Data = asSequence().asIterable().reduceNamedToData(meta, coroutineContext, transformation) -public inline fun DataSet.foldToData( +public inline fun DataTree.foldToData( initial: R, meta: Meta = joinMeta(), coroutineContext: CoroutineContext = EmptyCoroutineContext, crossinline block: suspend (result: R, data: NamedValueWithMeta) -> R, -): Data = asIterable().foldNamedToData(initial, meta, coroutineContext, block) \ No newline at end of file +): Data = asSequence().asIterable().foldNamedToData(initial, meta, coroutineContext, block) \ No newline at end of file diff --git a/dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/dataFilterJvm.kt b/dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/dataFilterJvm.kt index 74d67d9d..aa5c500a 100644 --- a/dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/dataFilterJvm.kt +++ b/dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/dataFilterJvm.kt @@ -1,12 +1,10 @@ package space.kscience.dataforge.data +import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.filter -import space.kscience.dataforge.meta.Meta -import space.kscience.dataforge.misc.DFExperimental +import space.kscience.dataforge.misc.DFInternal import space.kscience.dataforge.names.Name -import kotlin.coroutines.CoroutineContext -import kotlin.coroutines.EmptyCoroutineContext import kotlin.reflect.KType import kotlin.reflect.full.isSubtypeOf import kotlin.reflect.typeOf @@ -25,61 +23,65 @@ private fun Data<*>.castOrNull(type: KType): Data? = } } +@Suppress("UNCHECKED_CAST") +@DFInternal +public fun Sequence>.filterByDataType(type: KType): Sequence> = + filter { it.type.isSubtypeOf(type) } as Sequence> + +@Suppress("UNCHECKED_CAST") +@DFInternal +public fun Flow>.filterByDataType(type: KType): Flow> = + filter { it.type.isSubtypeOf(type) } as Flow> + /** * Select all data matching given type and filters. Does not modify paths * - * @param predicate addition filtering condition based on item name and meta. By default, accepts all + * @param predicate additional filtering condition based on item name and meta. By default, accepts all */ -@OptIn(DFExperimental::class) -public fun DataSet<*>.filterByType( +@DFInternal +public fun DataTree<*>.filterByType( type: KType, - predicate: (name: Name, meta: Meta) -> Boolean = { _, _ -> true }, -): DataSource = object : DataSource { - override val dataType = type - - override val coroutineContext: CoroutineContext - get() = (this@filterByType as? DataSource)?.coroutineContext ?: EmptyCoroutineContext - - override val meta: Meta get() = this@filterByType.meta - - private fun checkDatum(name: Name, datum: Data<*>): Boolean = datum.type.isSubtypeOf(type) - && predicate(name, datum.meta) - - override fun iterator(): Iterator> = iterator { - for(d in this@filterByType){ - if(checkDatum(d.name,d.data)){ - @Suppress("UNCHECKED_CAST") - yield(d as NamedData) - } - } - } - - override fun get(name: Name): Data? = this@filterByType[name]?.let { datum -> - if (checkDatum(name, datum)) datum.castOrNull(type) else null - } - - override val updates: Flow = this@filterByType.updates.filter { name -> - get(name)?.let { datum -> - checkDatum(name, datum) - } ?: false - } -} + predicate: DataFilter = DataFilter.EMPTY, +): DataTree = asSequence().filterByDataType(type).filterData(predicate).toTree(type) /** * Select a single datum of the appropriate type */ -public inline fun DataSet<*>.filterByType( - noinline predicate: (name: Name, meta: Meta) -> Boolean = { _, _ -> true }, -): DataSet = filterByType(typeOf(), predicate) +@OptIn(DFInternal::class) +public inline fun DataTree<*>.filterByType( + predicate: DataFilter = DataFilter.EMPTY, +): DataTree = filterByType(typeOf(), predicate) /** * Select a single datum if it is present and of given [type] */ -public fun DataSet<*>.getByType(type: KType, name: Name): NamedData? = +public fun DataTree<*>.getByType(type: KType, name: Name): NamedData? = get(name)?.castOrNull(type)?.named(name) -public inline fun DataSet<*>.getByType(name: Name): NamedData? = +public inline fun DataTree<*>.getByType(name: Name): NamedData? = this@getByType.getByType(typeOf(), name) -public inline fun DataSet<*>.getByType(name: String): NamedData? = - this@getByType.getByType(typeOf(), Name.parse(name)) \ No newline at end of file +public inline fun DataTree<*>.getByType(name: String): NamedData? = + this@getByType.getByType(typeOf(), Name.parse(name)) + +/** + * Select all data matching given type and filters. Does not modify paths + * + * @param predicate additional filtering condition based on item name and meta. By default, accepts all + */ +@DFInternal +public fun ObservableDataTree<*>.filterByType( + type: KType, + scope: CoroutineScope, + predicate: DataFilter = DataFilter.EMPTY, +): ObservableDataTree = asSequence() + .filterByDataType(type) + .filterData(predicate) + .toObservableTree(type, scope, updates().filterByDataType(type).filterData(predicate)) + + +@OptIn(DFInternal::class) +public inline fun ObservableDataTree<*>.filterByType( + scope: CoroutineScope, + predicate: DataFilter = DataFilter.EMPTY, +): ObservableDataTree = filterByType(typeOf(),scope,predicate) \ No newline at end of file diff --git a/dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/dataSetBuilderInContext.kt b/dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/dataSetBuilderInContext.kt index cb222ea0..116ae52d 100644 --- a/dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/dataSetBuilderInContext.kt +++ b/dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/dataSetBuilderInContext.kt @@ -10,31 +10,35 @@ import space.kscience.dataforge.names.plus /** * Append data to node */ -context(DataSetBuilder) public infix fun String.put(data: Data): Unit = - data(Name.parse(this), data) +context(DataSink) +public infix fun String.put(data: Data): Unit = + emit(Name.parse(this), data) /** * Append node */ -context(DataSetBuilder) public infix fun String.put(dataSet: DataSet): Unit = - node(Name.parse(this), dataSet) +context(DataSink) +public infix fun String.put(dataSet: DataTree): Unit = + emitAll(this, dataSet) /** * Build and append node */ -context(DataSetBuilder) public infix fun String.put( - block: DataSetBuilder.() -> Unit, -): Unit = node(Name.parse(this), block) +context(DataSink) +public infix fun String.put( + block: DataSink.() -> Unit, +): Unit = emitAll(Name.parse(this), block) /** - * Copy given data set and mirror its changes to this [DataTreeBuilder] in [this@setAndObserve]. Returns an update [Job] + * Copy given data set and mirror its changes to this [LegacyDataTreeBuilder] in [this@setAndObserve]. Returns an update [Job] */ -context(DataSetBuilder) public fun CoroutineScope.setAndWatch( +context(DataSink) +public fun CoroutineScope.setAndWatch( name: Name, - dataSet: DataSet, + dataSet: DataTree, ): Job = launch { - node(name, dataSet) - dataSet.updates.collect { nameInBranch -> - data(name + nameInBranch, dataSet.get(nameInBranch)) + emitAll(name, dataSet) + dataSet.updates().collect { + emit(name + it.name, it.data) } } \ No newline at end of file diff --git a/dataforge-data/src/jvmTest/kotlin/space/kscience/dataforge/data/ActionsTest.kt b/dataforge-data/src/jvmTest/kotlin/space/kscience/dataforge/data/ActionsTest.kt index 06a7e755..17549b6b 100644 --- a/dataforge-data/src/jvmTest/kotlin/space/kscience/dataforge/data/ActionsTest.kt +++ b/dataforge-data/src/jvmTest/kotlin/space/kscience/dataforge/data/ActionsTest.kt @@ -29,7 +29,7 @@ internal class ActionsTest { @Test fun testDynamicMapAction() = runTest { - val data: DataSourceBuilder = DataSource(this) + val data: MutableDataTree = MutableDataTree(this) val plusOne = Action.mapping { result { it + 1 } @@ -44,7 +44,6 @@ internal class ActionsTest { delay(20) assertEquals(2, result["1"]?.await()) - data.close() } } \ No newline at end of file diff --git a/dataforge-data/src/jvmTest/kotlin/space/kscience/dataforge/data/DataTreeBuilderTest.kt b/dataforge-data/src/jvmTest/kotlin/space/kscience/dataforge/data/LegacyGenericDataTreeBuilderTest.kt similarity index 87% rename from dataforge-data/src/jvmTest/kotlin/space/kscience/dataforge/data/DataTreeBuilderTest.kt rename to dataforge-data/src/jvmTest/kotlin/space/kscience/dataforge/data/LegacyGenericDataTreeBuilderTest.kt index a4d28eab..a871b340 100644 --- a/dataforge-data/src/jvmTest/kotlin/space/kscience/dataforge/data/DataTreeBuilderTest.kt +++ b/dataforge-data/src/jvmTest/kotlin/space/kscience/dataforge/data/LegacyGenericDataTreeBuilderTest.kt @@ -7,7 +7,7 @@ import kotlin.test.Test import kotlin.test.assertEquals -internal class DataTreeBuilderTest { +internal class LegacyGenericDataTreeBuilderTest { @Test fun testTreeBuild() = runBlocking { val node = DataTree { @@ -29,7 +29,7 @@ internal class DataTreeBuilderTest { @OptIn(DFExperimental::class) @Test fun testDataUpdate() = runBlocking { - val updateData: DataTree = DataTree { + val updateData = DataTree { "update" put { "a" put Data.static("a") "b" put Data.static("b") @@ -56,7 +56,7 @@ internal class DataTreeBuilderTest { try { lateinit var updateJob: Job supervisorScope { - val subNode = DataSource(this) { + val subNode = ObservableDataTree(this) { updateJob = launch { repeat(10) { delay(10) @@ -66,16 +66,16 @@ internal class DataTreeBuilderTest { } } launch { - subNode.updatesWithData.collect { + subNode.updates().collect { println(it) } } - val rootNode = DataSource(this) { + val rootNode = ObservableDataTree(this) { setAndWatch("sub".asName(), subNode) } launch { - rootNode.updatesWithData.collect { + rootNode.updates().collect { println(it) } } diff --git a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/IOFormat.kt b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/IOFormat.kt index 390a8bf4..54e305c6 100644 --- a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/IOFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/IOFormat.kt @@ -17,11 +17,7 @@ import kotlin.reflect.typeOf /** * Reader of a custom object from input */ -public interface IOReader { - /** - * The type of object being read - */ - public val type: KType +public fun interface IOReader { public fun readFrom(source: Source): T @@ -32,7 +28,6 @@ public interface IOReader { * no-op reader for binaries. */ public val binary: IOReader = object : IOReader { - override val type: KType = typeOf() override fun readFrom(source: Source): Binary = source.readByteArray().asBinary() @@ -42,8 +37,6 @@ public interface IOReader { } public inline fun IOReader(crossinline read: Source.() -> T): IOReader = object : IOReader { - override val type: KType = typeOf() - override fun readFrom(source: Source): T = source.read() } @@ -61,19 +54,19 @@ public fun Source.readWith(format: IOReader): T = format.readFrom(t /** * Read given binary as an object using given format */ -public fun Binary.readWith(format: IOReader): T = read { +public fun Binary.readWith(format: IOReader): T = read { readWith(format) } /** * Write an object to the [Sink] with given [format] */ -public fun Sink.writeWith(format: IOWriter, obj: T): Unit = +public fun Sink.writeWith(format: IOWriter, obj: T): Unit = format.writeTo(this, obj) @DfType(IO_FORMAT_TYPE) -public interface IOFormatFactory : Factory>, Named { +public interface IOFormatFactory : Factory>, Named { /** * Explicit type for dynamic type checks */ @@ -86,7 +79,7 @@ public interface IOFormatFactory : Factory>, Named { } } -public fun Binary(obj: T, format: IOWriter): Binary = Binary { format.writeTo(this, obj) } +public fun Binary(obj: T, format: IOWriter): Binary = Binary { format.writeTo(this, obj) } public object FloatIOFormat : IOFormat, IOFormatFactory { override fun build(context: Context, meta: Meta): IOFormat = this diff --git a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/IOPlugin.kt b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/IOPlugin.kt index c3248021..0d79da4d 100644 --- a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/IOPlugin.kt +++ b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/IOPlugin.kt @@ -5,7 +5,6 @@ import space.kscience.dataforge.io.EnvelopeFormatFactory.Companion.ENVELOPE_FORM import space.kscience.dataforge.io.IOFormatFactory.Companion.IO_FORMAT_TYPE import space.kscience.dataforge.io.MetaFormatFactory.Companion.META_FORMAT_TYPE import space.kscience.dataforge.meta.Meta -import space.kscience.dataforge.meta.get import space.kscience.dataforge.meta.string import space.kscience.dataforge.misc.DFInternal import space.kscience.dataforge.names.Name @@ -21,11 +20,11 @@ public class IOPlugin(meta: Meta) : AbstractPlugin(meta) { @Suppress("UNCHECKED_CAST") @DFInternal - public fun resolveIOFormat(type: KType, meta: Meta): IOFormat? = + public fun resolveIOFormat(type: KType, meta: Meta): IOFormat? = ioFormatFactories.singleOrNull { it.type == type }?.build(context, meta) as? IOFormat @OptIn(DFInternal::class) - public inline fun resolveIOFormat(meta: Meta = Meta.EMPTY): IOFormat? = + public inline fun resolveIOFormat(meta: Meta = Meta.EMPTY): IOFormat? = resolveIOFormat(typeOf(), meta) diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MetaConverter.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MetaConverter.kt index 933cc13d..8959ae4a 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MetaConverter.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MetaConverter.kt @@ -6,8 +6,6 @@ import kotlinx.serialization.json.encodeToJsonElement import kotlinx.serialization.serializer import space.kscience.dataforge.meta.descriptors.MetaDescriptor import space.kscience.dataforge.misc.DFExperimental -import kotlin.reflect.KType -import kotlin.reflect.typeOf /** @@ -15,15 +13,10 @@ import kotlin.reflect.typeOf */ public interface MetaConverter: MetaSpec { - /** - * Runtime type of [T] - */ - public val type: KType - /** * A descriptor for resulting meta */ - override val descriptor: MetaDescriptor get() = MetaDescriptor.EMPTY + override val descriptor: MetaDescriptor? get() = null /** * Attempt conversion of [source] to an object or return null if conversion failed @@ -38,22 +31,16 @@ public interface MetaConverter: MetaSpec { public companion object { public val meta: MetaConverter = object : MetaConverter { - override val type: KType = typeOf() - override fun readOrNull(source: Meta): Meta = source override fun convert(obj: Meta): Meta = obj } public val value: MetaConverter = object : MetaConverter { - override val type: KType = typeOf() - override fun readOrNull(source: Meta): Value? = source.value override fun convert(obj: Value): Meta = Meta(obj) } public val string: MetaConverter = object : MetaConverter { - override val type: KType = typeOf() - override val descriptor: MetaDescriptor = MetaDescriptor { valueType(ValueType.STRING) } @@ -64,8 +51,6 @@ public interface MetaConverter: MetaSpec { } public val boolean: MetaConverter = object : MetaConverter { - override val type: KType = typeOf() - override val descriptor: MetaDescriptor = MetaDescriptor { valueType(ValueType.BOOLEAN) } @@ -75,8 +60,6 @@ public interface MetaConverter: MetaSpec { } public val number: MetaConverter = object : MetaConverter { - override val type: KType = typeOf() - override val descriptor: MetaDescriptor = MetaDescriptor { valueType(ValueType.NUMBER) } @@ -86,8 +69,6 @@ public interface MetaConverter: MetaSpec { } public val double: MetaConverter = object : MetaConverter { - override val type: KType = typeOf() - override val descriptor: MetaDescriptor = MetaDescriptor { valueType(ValueType.NUMBER) } @@ -97,8 +78,6 @@ public interface MetaConverter: MetaSpec { } public val float: MetaConverter = object : MetaConverter { - override val type: KType = typeOf() - override val descriptor: MetaDescriptor = MetaDescriptor { valueType(ValueType.NUMBER) } @@ -108,8 +87,6 @@ public interface MetaConverter: MetaSpec { } public val int: MetaConverter = object : MetaConverter { - override val type: KType = typeOf() - override val descriptor: MetaDescriptor = MetaDescriptor { valueType(ValueType.NUMBER) } @@ -119,8 +96,6 @@ public interface MetaConverter: MetaSpec { } public val long: MetaConverter = object : MetaConverter { - override val type: KType = typeOf() - override val descriptor: MetaDescriptor = MetaDescriptor { valueType(ValueType.NUMBER) } @@ -130,8 +105,6 @@ public interface MetaConverter: MetaSpec { } public inline fun > enum(): MetaConverter = object : MetaConverter { - override val type: KType = typeOf() - override val descriptor: MetaDescriptor = MetaDescriptor { valueType(ValueType.STRING) allowedValues(enumValues()) @@ -147,8 +120,6 @@ public interface MetaConverter: MetaSpec { writer: (T) -> Value = { Value.of(it) }, reader: (Value) -> T, ): MetaConverter> = object : MetaConverter> { - override val type: KType = typeOf>() - override val descriptor: MetaDescriptor = MetaDescriptor { valueType(ValueType.LIST) } @@ -165,7 +136,6 @@ public interface MetaConverter: MetaSpec { public inline fun serializable( descriptor: MetaDescriptor? = null, ): MetaConverter = object : MetaConverter { - override val type: KType = typeOf() private val serializer: KSerializer = serializer() override fun readOrNull(source: Meta): T? { diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/Scheme.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/Scheme.kt index 1d654ac2..2e9edc1d 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/Scheme.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/Scheme.kt @@ -166,9 +166,9 @@ public inline fun T.copy(spec: SchemeSpec, block: T.() -> Unit = /** * A specification for simplified generation of wrappers */ -public open class SchemeSpec( +public open class SchemeSpec( private val builder: () -> T, -) : MetaSpec { +) : MetaConverter { override val descriptor: MetaDescriptor? get() = null @@ -187,6 +187,8 @@ public open class SchemeSpec( it.initialize(MutableMeta(), Meta.EMPTY, descriptor) } + override fun convert(obj: T): Meta = obj.meta + /** * A convenience method to use specifications in builders */ diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/names/Name.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/names/Name.kt index 5e4b98a8..b46a3507 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/names/Name.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/names/Name.kt @@ -113,6 +113,13 @@ public class Name(public val tokens: List) { } } +/** + * Transform this [Name] to a string without escaping special characters in tokens. + * + * Parsing it back will produce a valid, but different name + */ +public fun Name.toStringUnescaped(): String = tokens.joinToString(separator = Name.NAME_SEPARATOR) { it.toStringUnescaped() } + public operator fun Name.get(i: Int): NameToken = tokens[i] /** diff --git a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/EnvelopeTask.kt b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/EnvelopeTask.kt deleted file mode 100644 index a1588a54..00000000 --- a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/EnvelopeTask.kt +++ /dev/null @@ -1,46 +0,0 @@ -package space.kscience.dataforge.workspace - -import space.kscience.dataforge.data.DataTree.Companion.META_ITEM_NAME_TOKEN -import space.kscience.dataforge.io.Envelope -import space.kscience.dataforge.io.IOReader -import space.kscience.dataforge.meta.Meta -import space.kscience.dataforge.meta.descriptors.MetaDescriptor -import space.kscience.dataforge.names.Name -import space.kscience.dataforge.names.asName -import kotlin.reflect.KType - -public abstract class EnvelopeTask( - override val descriptor: MetaDescriptor?, - private val reader: IOReader, -) : Task { - - public abstract suspend fun produceEnvelopes( - workspace: Workspace, - taskName: Name, - taskMeta: Meta, - ): Map - - override suspend fun execute(workspace: Workspace, taskName: Name, taskMeta: Meta): TaskResult = - Result(workspace, taskName, taskMeta, reader, produceEnvelopes(workspace, taskName, taskMeta)) - - private class Result( - override val workspace: Workspace, - override val taskName: Name, - override val taskMeta: Meta, - val reader: IOReader, - envelopes: Map, - ) : TaskResult { - - private val dataMap = envelopes.mapValues { - workspace.wrapData(it.value.toData(reader), it.key, taskName, taskMeta) - } - override val meta: Meta get() = dataMap[META_ITEM_NAME_TOKEN.asName()]?.meta ?: Meta.EMPTY - - override val dataType: KType get() = reader.type - - override fun iterator(): Iterator> = dataMap.values.iterator() - - override fun get(name: Name): TaskData? = dataMap[name] - } -} - diff --git a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/Task.kt b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/Task.kt index 19d16c68..af2520eb 100644 --- a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/Task.kt +++ b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/Task.kt @@ -1,9 +1,9 @@ package space.kscience.dataforge.workspace import kotlinx.coroutines.withContext -import space.kscience.dataforge.data.DataSetBuilder -import space.kscience.dataforge.data.DataTree +import space.kscience.dataforge.data.DataSink import space.kscience.dataforge.data.GoalExecutionRestriction +import space.kscience.dataforge.data.MutableDataTree import space.kscience.dataforge.meta.Meta import space.kscience.dataforge.meta.MetaRepr import space.kscience.dataforge.meta.MetaSpec @@ -20,7 +20,7 @@ import kotlin.reflect.typeOf * In general no computations should be made until the result is called. */ @DfType(TYPE) -public interface Task : Described { +public interface Task : Described { /** * A task identification string used to compare tasks and check task body for change @@ -45,7 +45,7 @@ public interface Task : Described { /** * A [Task] with [MetaSpec] for wrapping and unwrapping task configuration */ -public interface TaskWithSpec : Task { +public interface TaskWithSpec : Task { public val spec: MetaSpec override val descriptor: MetaDescriptor? get() = spec.descriptor @@ -61,12 +61,12 @@ public interface TaskWithSpec : Task { // block: C.() -> Unit = {}, //): TaskResult = execute(workspace, taskName, spec(block)) -public class TaskResultBuilder( +public class TaskResultBuilder( public val workspace: Workspace, public val taskName: Name, public val taskMeta: Meta, - private val dataDrop: DataSetBuilder, -) : DataSetBuilder by dataDrop + private val dataSink: DataSink, +) : DataSink by dataSink /** * Create a [Task] that composes a result using [builder]. Only data from the workspace could be used. @@ -90,7 +90,7 @@ public fun Task( taskMeta: Meta, ): TaskResult = withContext(GoalExecutionRestriction() + workspace.goalLogger) { //TODO use safe builder and check for external data on add and detects cycles - val dataset = DataTree(resultType) { + val dataset = MutableDataTree(resultType, this).apply { TaskResultBuilder(workspace, taskName, taskMeta, this).apply { builder() } } workspace.wrapResult(dataset, taskName, taskMeta) @@ -111,6 +111,7 @@ public inline fun Task( * @param specification a specification for task configuration * @param builder for resulting data set */ + @Suppress("FunctionName") public fun Task( resultType: KType, @@ -126,7 +127,7 @@ public fun Task( ): TaskResult = withContext(GoalExecutionRestriction() + workspace.goalLogger) { //TODO use safe builder and check for external data on add and detects cycles val taskMeta = configuration.toMeta() - val dataset = DataTree(resultType) { + val dataset = MutableDataTree(resultType, this).apply { TaskResultBuilder(workspace, taskName, taskMeta, this).apply { builder(configuration) } } workspace.wrapResult(dataset, taskName, taskMeta) diff --git a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/TaskData.kt b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/TaskData.kt deleted file mode 100644 index 080ffec3..00000000 --- a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/TaskData.kt +++ /dev/null @@ -1,50 +0,0 @@ -package space.kscience.dataforge.workspace - -import space.kscience.dataforge.data.Data -import space.kscience.dataforge.data.NamedData -import space.kscience.dataforge.meta.Meta -import space.kscience.dataforge.names.Name - -/** - * A [Workspace]-locked [NamedData], that serves as a computation model. - */ -public interface TaskData : NamedData { - /** - * The [Workspace] this data belongs to - */ - public val workspace: Workspace - - /** - * The name of the stage that produced this data. [Name.EMPTY] if the workspace intrinsic data is used. - */ - public val taskName: Name - - /** - * Stage configuration used to produce this data. - */ - public val taskMeta: Meta - - /** - * Dependencies that allow to compute transitive dependencies as well. - */ -// override val dependencies: Collection> -} - -private class TaskDataImpl( - override val workspace: Workspace, - override val data: Data, - override val name: Name, - override val taskName: Name, - override val taskMeta: Meta, -) : TaskData, Data by data { -// override val dependencies: Collection> = data.dependencies.map { -// it as? TaskData<*> ?: error("TaskData can't depend on external data") -// } -} - -/** - * Adopt data into this workspace - */ -public fun Workspace.wrapData(data: Data, name: Name, taskName: Name, taskMeta: Meta): TaskData = - TaskDataImpl(this, data, name, taskName, taskMeta) - diff --git a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/TaskResult.kt b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/TaskResult.kt index d8db6417..59cb7461 100644 --- a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/TaskResult.kt +++ b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/TaskResult.kt @@ -1,54 +1,27 @@ package space.kscience.dataforge.workspace -import space.kscience.dataforge.data.DataSet -import space.kscience.dataforge.data.forEach +import space.kscience.dataforge.data.ObservableDataTree import space.kscience.dataforge.meta.Meta import space.kscience.dataforge.names.Name +import kotlin.reflect.KType /** * A result of a [Task] + * @param workspace the [Workspace] that produced the result + * @param taskName the name of the task that produced the result + * @param taskMeta The configuration of the task that produced the result */ -public interface TaskResult : DataSet { - /** - * The [Workspace] this [DataSet] belongs to - */ - public val workspace: Workspace - - /** - * The [Name] of the stage that produced this [DataSet] - */ - public val taskName: Name - - /** - * The configuration of the stage that produced this [DataSet] - */ - public val taskMeta: Meta - - override fun iterator(): Iterator> - - override fun get(name: Name): TaskData? -} - -private class TaskResultImpl( - override val workspace: Workspace, - override val taskName: Name, - override val taskMeta: Meta, - val dataSet: DataSet, -) : TaskResult, DataSet by dataSet { - - override fun iterator(): Iterator> = iterator { - dataSet.forEach { - yield(workspace.wrapData(it, it.name, taskName, taskMeta)) - } - } - - override fun get(name: Name): TaskData? = dataSet[name]?.let { - workspace.wrapData(it, name, taskName, taskMeta) - } +public data class TaskResult( + public val data: ObservableDataTree, + public val workspace: Workspace, + public val taskName: Name, + public val taskMeta: Meta, +) { + val dataType: KType get() = data.dataType } /** * Wrap data into [TaskResult] */ -public fun Workspace.wrapResult(dataSet: DataSet, taskName: Name, taskMeta: Meta): TaskResult = - TaskResultImpl(this, taskName, taskMeta, dataSet) \ No newline at end of file +public fun Workspace.wrapResult(data: ObservableDataTree, taskName: Name, taskMeta: Meta): TaskResult = + TaskResult(data, this, taskName, taskMeta) \ No newline at end of file diff --git a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/Workspace.kt b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/Workspace.kt index 37b473db..bc1c19dc 100644 --- a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/Workspace.kt +++ b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/Workspace.kt @@ -1,29 +1,32 @@ package space.kscience.dataforge.workspace +import kotlinx.coroutines.CoroutineScope import space.kscience.dataforge.context.ContextAware -import space.kscience.dataforge.data.Data -import space.kscience.dataforge.data.DataSet -import space.kscience.dataforge.data.asSequence +import space.kscience.dataforge.data.* import space.kscience.dataforge.meta.Meta import space.kscience.dataforge.meta.MutableMeta import space.kscience.dataforge.misc.DfType import space.kscience.dataforge.names.Name import space.kscience.dataforge.provider.Provider +import kotlin.coroutines.CoroutineContext -public interface DataSelector{ - public suspend fun select(workspace: Workspace, meta: Meta): DataSet +public fun interface DataSelector { + public suspend fun select(workspace: Workspace, meta: Meta): DataTree } /** * An environment for pull-mode computation */ @DfType(Workspace.TYPE) -public interface Workspace : ContextAware, Provider { +public interface Workspace : ContextAware, Provider, CoroutineScope { + + override val coroutineContext: CoroutineContext get() = context.coroutineContext + /** * The whole data node for current workspace */ - public val data: TaskResult<*> + public val data: ObservableDataTree<*> /** * All targets associated with the workspace @@ -37,7 +40,7 @@ public interface Workspace : ContextAware, Provider { override fun content(target: String): Map { return when (target) { - "target", Meta.TYPE -> targets.mapKeys { Name.parse(it.key)} + "target", Meta.TYPE -> targets.mapKeys { Name.parse(it.key) } Task.TYPE -> tasks Data.TYPE -> data.asSequence().associateBy { it.name } else -> emptyMap() @@ -49,8 +52,8 @@ public interface Workspace : ContextAware, Provider { return task.execute(this, taskName, taskMeta) } - public suspend fun produceData(taskName: Name, taskMeta: Meta, name: Name): TaskData<*>? = - produce(taskName, taskMeta)[name] + public suspend fun produceData(taskName: Name, taskMeta: Meta, name: Name): Data<*>? = + produce(taskName, taskMeta).data[name] public companion object { public const val TYPE: String = "workspace" diff --git a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/WorkspaceBuilder.kt b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/WorkspaceBuilder.kt index 39e35b39..e0b7fad7 100644 --- a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/WorkspaceBuilder.kt +++ b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/WorkspaceBuilder.kt @@ -5,9 +5,9 @@ import space.kscience.dataforge.actions.Action import space.kscience.dataforge.context.Context import space.kscience.dataforge.context.ContextBuilder import space.kscience.dataforge.context.Global -import space.kscience.dataforge.data.DataSet -import space.kscience.dataforge.data.DataSource -import space.kscience.dataforge.data.DataSourceBuilder +import space.kscience.dataforge.data.DataSink +import space.kscience.dataforge.data.DataTree +import space.kscience.dataforge.data.MutableDataTree import space.kscience.dataforge.meta.* import space.kscience.dataforge.meta.descriptors.MetaDescriptor import space.kscience.dataforge.meta.descriptors.MetaDescriptorBuilder @@ -17,13 +17,14 @@ import space.kscience.dataforge.names.asName import kotlin.collections.set import kotlin.properties.PropertyDelegateProvider import kotlin.properties.ReadOnlyProperty +import kotlin.reflect.typeOf public data class TaskReference(public val taskName: Name, public val task: Task) : DataSelector { @Suppress("UNCHECKED_CAST") - override suspend fun select(workspace: Workspace, meta: Meta): DataSet { + override suspend fun select(workspace: Workspace, meta: Meta): DataTree { if (workspace.tasks[taskName] == task) { - return workspace.produce(taskName, meta) as TaskResult + return workspace.produce(taskName, meta).data as DataTree } else { error("Task $taskName does not belong to the workspace") } @@ -45,7 +46,7 @@ public inline fun TaskContainer.registerTask( ): Unit = registerTask(Name.parse(name), Task(MetaDescriptor(descriptorBuilder), builder)) /** - * Create a new t + * Create and register a new task */ public inline fun TaskContainer.buildTask( name: String, @@ -101,7 +102,7 @@ public inline fun TaskContainer.action( noinline descriptorBuilder: MetaDescriptorBuilder.() -> Unit = {}, ): PropertyDelegateProvider>> = task(MetaDescriptor(descriptorBuilder)) { - result(action.execute(from(selector), taskMeta.copy(metaTransform))) + result(action.execute(workspace.context, from(selector), taskMeta.copy(metaTransform))) } public class WorkspaceBuilder( @@ -109,7 +110,7 @@ public class WorkspaceBuilder( private val coroutineScope: CoroutineScope = parentContext, ) : TaskContainer { private var context: Context? = null - private val data = DataSource(coroutineScope) + private val data = MutableDataTree(typeOf(), coroutineScope) private val targets: HashMap = HashMap() private val tasks = HashMap>() private var cache: WorkspaceCache? = null @@ -124,7 +125,7 @@ public class WorkspaceBuilder( /** * Define intrinsic data for the workspace */ - public fun data(builder: DataSourceBuilder.() -> Unit) { + public fun data(builder: DataSink<*>.() -> Unit) { data.apply(builder) } @@ -149,7 +150,7 @@ public class WorkspaceBuilder( public fun build(): Workspace { val postProcess: suspend (TaskResult<*>) -> TaskResult<*> = { result -> - cache?.evaluate(result) ?: result + cache?.cache(result) ?: result } return WorkspaceImpl(context ?: parentContext, data, targets, tasks, postProcess) } diff --git a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/WorkspaceCache.kt b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/WorkspaceCache.kt index 62df6744..42cb7b4f 100644 --- a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/WorkspaceCache.kt +++ b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/WorkspaceCache.kt @@ -1,5 +1,5 @@ package space.kscience.dataforge.workspace public interface WorkspaceCache { - public suspend fun evaluate(result: TaskResult): TaskResult + public suspend fun cache(result: TaskResult): TaskResult } diff --git a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/WorkspaceImpl.kt b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/WorkspaceImpl.kt index dae9667a..21c5e8c2 100644 --- a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/WorkspaceImpl.kt +++ b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/WorkspaceImpl.kt @@ -2,21 +2,19 @@ package space.kscience.dataforge.workspace import space.kscience.dataforge.context.Context import space.kscience.dataforge.context.gather -import space.kscience.dataforge.data.DataSet +import space.kscience.dataforge.data.ObservableDataTree import space.kscience.dataforge.meta.Meta import space.kscience.dataforge.names.Name internal class WorkspaceImpl internal constructor( override val context: Context, - data: DataSet<*>, + override val data: ObservableDataTree<*>, override val targets: Map, tasks: Map>, private val postProcess: suspend (TaskResult<*>) -> TaskResult<*>, ) : Workspace { - override val data: TaskResult<*> = wrapResult(data, Name.EMPTY, Meta.EMPTY) - override val tasks: Map> by lazy { context.gather>(Task.TYPE) + tasks } override suspend fun produce(taskName: Name, taskMeta: Meta): TaskResult<*> { diff --git a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/envelopeData.kt b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/envelopeData.kt index 39bb0726..d54ff510 100644 --- a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/envelopeData.kt +++ b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/envelopeData.kt @@ -4,13 +4,14 @@ import space.kscience.dataforge.data.Data import space.kscience.dataforge.data.await import space.kscience.dataforge.io.* import space.kscience.dataforge.misc.DFInternal +import kotlin.reflect.typeOf /** * Convert an [Envelope] to a data via given format. The actual parsing is done lazily. */ @OptIn(DFInternal::class) -public fun Envelope.toData(format: IOReader): Data = Data(format.type, meta) { +public inline fun Envelope.toData(format: IOReader): Data = Data(typeOf(), meta) { data?.readWith(format) ?: error("Can't convert envelope without data to Data") } diff --git a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/taskBuilders.kt b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/taskBuilders.kt index a7c1f0fb..11594c2b 100644 --- a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/taskBuilders.kt +++ b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/taskBuilders.kt @@ -2,9 +2,10 @@ package space.kscience.dataforge.workspace import space.kscience.dataforge.actions.Action import space.kscience.dataforge.context.PluginFactory -import space.kscience.dataforge.data.DataSet +import space.kscience.dataforge.data.DataTree +import space.kscience.dataforge.data.emitAll import space.kscience.dataforge.data.forEach -import space.kscience.dataforge.data.map +import space.kscience.dataforge.data.transform import space.kscience.dataforge.meta.* import space.kscience.dataforge.misc.DFExperimental import space.kscience.dataforge.names.Name @@ -27,19 +28,19 @@ public val TaskResultBuilder<*>.defaultDependencyMeta: Meta public suspend fun TaskResultBuilder<*>.from( selector: DataSelector, dependencyMeta: Meta = defaultDependencyMeta, -): DataSet = selector.select(workspace, dependencyMeta) +): DataTree = selector.select(workspace, dependencyMeta) public suspend inline fun TaskResultBuilder<*>.from( plugin: P, dependencyMeta: Meta = defaultDependencyMeta, selectorBuilder: P.() -> TaskReference, -): DataSet { +): TaskResult { require(workspace.context.plugins.contains(plugin)) { "Plugin $plugin is not loaded into $workspace" } val taskReference: TaskReference = plugin.selectorBuilder() val res = workspace.produce(plugin.name + taskReference.taskName, dependencyMeta) //TODO add explicit check after https://youtrack.jetbrains.com/issue/KT-32956 @Suppress("UNCHECKED_CAST") - return res as TaskResult + return res as TaskResult } /** @@ -53,7 +54,7 @@ public suspend inline fun TaskRes pluginFactory: PluginFactory

, dependencyMeta: Meta = defaultDependencyMeta, selectorBuilder: P.() -> TaskReference, -): DataSet { +): TaskResult { val plugin = workspace.context.plugins[pluginFactory] ?: error("Plugin ${pluginFactory.tag} not loaded into workspace context") val taskReference: TaskReference = plugin.selectorBuilder() @@ -64,9 +65,7 @@ public suspend inline fun TaskRes } public val TaskResultBuilder<*>.allData: DataSelector<*> - get() = object : DataSelector { - override suspend fun select(workspace: Workspace, meta: Meta): DataSet = workspace.data - } + get() = DataSelector { workspace, _ -> workspace.data } /** * Perform a lazy mapping task using given [selector] and one-to-one [action]. @@ -90,19 +89,19 @@ public suspend inline fun TaskResultBuilder.transf dataMetaTransform(data.name) } - val res = data.map(meta, workspace.context.coroutineContext) { + val res = data.transform(meta, workspace.context.coroutineContext) { action(it, data.name, meta) } - data(data.name, res) + emit(data.name, res) } } /** * Set given [dataSet] as a task result. */ -public fun TaskResultBuilder.result(dataSet: DataSet) { - node(Name.EMPTY, dataSet) +public fun TaskResultBuilder.result(dataSet: DataTree) { + emitAll(dataSet) } /** @@ -111,10 +110,10 @@ public fun TaskResultBuilder.result(dataSet: DataSet) { @DFExperimental public suspend inline fun TaskResultBuilder.actionFrom( selector: DataSelector, - action: Action, + action: Action, dependencyMeta: Meta = defaultDependencyMeta, ) { - node(Name.EMPTY, action.execute(from(selector,dependencyMeta), dependencyMeta)) + emitAll(action.execute(workspace.context, from(selector, dependencyMeta), dependencyMeta)) } diff --git a/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/FileWorkspaceCache.kt b/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/FileWorkspaceCache.kt index 279e61a2..e6ddf026 100644 --- a/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/FileWorkspaceCache.kt +++ b/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/FileWorkspaceCache.kt @@ -1,5 +1,6 @@ package space.kscience.dataforge.workspace +import kotlinx.coroutines.flow.map import kotlinx.io.* import kotlinx.serialization.ExperimentalSerializationApi import kotlinx.serialization.KSerializer @@ -9,12 +10,10 @@ import kotlinx.serialization.serializer import space.kscience.dataforge.context.error import space.kscience.dataforge.context.logger import space.kscience.dataforge.context.request -import space.kscience.dataforge.data.Data -import space.kscience.dataforge.data.await +import space.kscience.dataforge.data.* import space.kscience.dataforge.io.* import space.kscience.dataforge.misc.DFExperimental import space.kscience.dataforge.misc.DFInternal -import space.kscience.dataforge.names.Name import space.kscience.dataforge.names.withIndex import java.nio.file.Path import kotlin.io.path.deleteIfExists @@ -22,7 +21,7 @@ import kotlin.io.path.div import kotlin.io.path.exists import kotlin.reflect.KType -public class JsonIOFormat(override val type: KType) : IOFormat { +public class JsonIOFormat(private val type: KType) : IOFormat { @Suppress("UNCHECKED_CAST") private val serializer: KSerializer = serializer(type) as KSerializer @@ -35,7 +34,7 @@ public class JsonIOFormat(override val type: KType) : IOFormat { } @OptIn(ExperimentalSerializationApi::class) -public class ProtobufIOFormat(override val type: KType) : IOFormat { +public class ProtobufIOFormat(private val type: KType) : IOFormat { @Suppress("UNCHECKED_CAST") private val serializer: KSerializer = serializer(type) as KSerializer @@ -53,14 +52,14 @@ public class FileWorkspaceCache(public val cacheDirectory: Path) : WorkspaceCach // private fun TaskData<*>.checkType(taskType: KType): TaskData = this as TaskData @OptIn(DFExperimental::class, DFInternal::class) - override suspend fun evaluate(result: TaskResult): TaskResult { + override suspend fun cache(result: TaskResult): TaskResult { val io = result.workspace.context.request(IOPlugin) val format: IOFormat = io.resolveIOFormat(result.dataType, result.taskMeta) ?: ProtobufIOFormat(result.dataType) ?: error("Can't resolve IOFormat for ${result.dataType}") - fun evaluateDatum(data: TaskData): TaskData { + fun cacheOne(data: NamedData): NamedData { val path = cacheDirectory / result.taskName.withIndex(result.taskMeta.hashCode().toString(16)).toString() / @@ -92,15 +91,14 @@ public class FileWorkspaceCache(public val cacheDirectory: Path) : WorkspaceCach } } - return data.workspace.wrapData(datum, data.name, data.taskName, data.taskMeta) + return datum.named(data.name) } - return object : TaskResult by result { - override fun iterator(): Iterator> = - result.iterator().asSequence().map { evaluateDatum(it) }.iterator() - override fun get(name: Name): TaskData? = result[name]?.let { evaluateDatum(it) } - } + val cachedTree = result.data.asSequence().map { cacheOne(it) } + .toObservableTree(result.dataType, result.workspace, result.data.updates().map { cacheOne(it) }) + + return result.workspace.wrapResult(cachedTree, result.taskName, result.taskMeta) } } diff --git a/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/InMemoryWorkspaceCache.kt b/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/InMemoryWorkspaceCache.kt index bb8ea9a4..a36e2b90 100644 --- a/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/InMemoryWorkspaceCache.kt +++ b/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/InMemoryWorkspaceCache.kt @@ -18,7 +18,7 @@ public class InMemoryWorkspaceCache : WorkspaceCache { if (type.isSubtypeOf(taskType)) this as TaskData else error("Cached data type mismatch: expected $taskType but got $type") - override suspend fun evaluate(result: TaskResult): TaskResult { + override suspend fun cache(result: TaskResult): TaskResult { for (d: TaskData in result) { cache.getOrPut(result.taskName to result.taskMeta) { HashMap() }.getOrPut(d.name) { d } } diff --git a/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/fileData.kt b/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/fileData.kt index ce1b5152..a0e6ab42 100644 --- a/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/fileData.kt +++ b/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/fileData.kt @@ -14,11 +14,8 @@ import space.kscience.dataforge.meta.copy import space.kscience.dataforge.meta.string import space.kscience.dataforge.misc.DFExperimental import space.kscience.dataforge.misc.DFInternal -import space.kscience.dataforge.names.Name -import space.kscience.dataforge.names.NameToken -import space.kscience.dataforge.names.asName -import space.kscience.dataforge.names.plus -import space.kscience.dataforge.workspace.FileData.Companion.DEFAULT_IGNORE_EXTENSIONS +import space.kscience.dataforge.names.* +import space.kscience.dataforge.workspace.FileData.Companion.defaultPathToName import java.nio.file.Files import java.nio.file.Path import java.nio.file.StandardWatchEventKinds @@ -35,6 +32,7 @@ import kotlin.reflect.typeOf public typealias FileFormatResolver = (path: Path, meta: Meta) -> IOReader? + /** * A data based on a filesystem [Path] */ @@ -54,13 +52,28 @@ public class FileData internal constructor(private val data: Data, public public val FILE_UPDATE_TIME_KEY: Name = FILE_KEY + "updated" public const val DF_FILE_EXTENSION: String = "df" public val DEFAULT_IGNORE_EXTENSIONS: Set = setOf(DF_FILE_EXTENSION) + + /** + * Transform file name into DataForg name. Ignores DataForge file extensions. + */ + public val defaultPathToName: (Path) -> Name = { path -> + Name( + path.map { segment -> + if (segment.isRegularFile() && segment.extension in DEFAULT_IGNORE_EXTENSIONS) { + NameToken(path.nameWithoutExtension) + } else { + NameToken(path.name) + } + } + ) + } } } /** - * Read data with supported envelope format and binary format. If envelope format is null, then read binary directly from file. - * The operation is blocking since it must read meta header. The reading of envelope body is lazy + * Read data with supported envelope format and binary format. If the envelope format is null, then read binary directly from file. + * The operation is blocking since it must read the meta header. The reading of envelope body is lazy */ @OptIn(DFInternal::class) @DFExperimental @@ -90,7 +103,7 @@ public fun IOPlugin.readDataFile( context(IOPlugin) @DFExperimental public fun DataSetBuilder.directory( path: Path, - ignoreExtensions: Set, + pathToName: (Path) -> Name = defaultPathToName, formatResolver: FileFormatResolver, ) { Files.list(path).forEach { childPath -> @@ -98,7 +111,7 @@ public fun DataSetBuilder.directory( if (fileName.startsWith(IOPlugin.META_FILE_NAME)) { meta(readMetaFile(childPath)) } else if (!fileName.startsWith("@")) { - file(childPath, ignoreExtensions, formatResolver) + file(childPath, pathToName, formatResolver) } } } @@ -111,9 +124,9 @@ public fun DataSetBuilder.directory( public fun IOPlugin.readDataDirectory( type: KType, path: Path, - ignoreExtensions: Set = DEFAULT_IGNORE_EXTENSIONS, + pathToName: (Path) -> Name = defaultPathToName, formatResolver: FileFormatResolver, -): DataTree { +): LegacyDataTree { //read zipped data node if (path.fileName != null && path.fileName.toString().endsWith(".zip")) { //Using explicit Zip file system to avoid bizarre compatibility bugs @@ -121,14 +134,14 @@ public fun IOPlugin.readDataDirectory( ?: error("Zip file system provider not found") val fs = fsProvider.newFileSystem(path, mapOf("create" to "true")) - return readDataDirectory(type, fs.rootDirectories.first(), ignoreExtensions, formatResolver) + return readDataDirectory(type, fs.rootDirectories.first(), pathToName, formatResolver) } if (!Files.isDirectory(path)) error("Provided path $path is not a directory") return DataTree(type) { meta { FileData.FILE_PATH_KEY put path.toString() } - directory(path, ignoreExtensions, formatResolver) + directory(path, pathToName, formatResolver) } } @@ -136,9 +149,9 @@ public fun IOPlugin.readDataDirectory( @DFExperimental public inline fun IOPlugin.readDataDirectory( path: Path, - ignoreExtensions: Set = DEFAULT_IGNORE_EXTENSIONS, + noinline pathToName: (Path) -> Name = defaultPathToName, noinline formatResolver: FileFormatResolver, -): DataTree = readDataDirectory(typeOf(), path, ignoreExtensions, formatResolver) +): LegacyDataTree = readDataDirectory(typeOf(), path, pathToName, formatResolver) /** * Read a raw binary data tree from the directory. All files are read as-is (save for meta files). @@ -146,8 +159,8 @@ public inline fun IOPlugin.readDataDirectory( @DFExperimental public fun IOPlugin.readRawDirectory( path: Path, - ignoreExtensions: Set = emptySet(), -): DataTree = readDataDirectory(path, ignoreExtensions) { _, _ -> IOReader.binary } + pathToName: (Path) -> Name = defaultPathToName, +): LegacyDataTree = readDataDirectory(path, pathToName) { _, _ -> IOReader.binary } private fun Path.toName() = Name(map { NameToken.parse(it.nameWithoutExtension) }) @@ -157,13 +170,13 @@ private fun Path.toName() = Name(map { NameToken.parse(it.nameWithoutExtension) public fun IOPlugin.monitorDataDirectory( type: KType, path: Path, - ignoreExtensions: Set = DEFAULT_IGNORE_EXTENSIONS, + pathToName: (Path) -> Name = defaultPathToName, formatResolver: FileFormatResolver, ): DataSource { if (path.fileName.toString().endsWith(".zip")) error("Monitoring not supported for ZipFS") if (!Files.isDirectory(path)) error("Provided path $path is not a directory") return DataSource(type, context) { - directory(path, ignoreExtensions, formatResolver) + directory(path, pathToName, formatResolver) launch(Dispatchers.IO) { val watchService = path.fileSystem.newWatchService() @@ -186,7 +199,7 @@ public fun IOPlugin.monitorDataDirectory( if (fileName.startsWith(IOPlugin.META_FILE_NAME)) { meta(readMetaFile(eventPath)) } else if (!fileName.startsWith("@")) { - file(eventPath, ignoreExtensions, formatResolver) + file(eventPath, pathToName, formatResolver) } } } @@ -205,9 +218,9 @@ public fun IOPlugin.monitorDataDirectory( @DFExperimental public inline fun IOPlugin.monitorDataDirectory( path: Path, - ignoreExtensions: Set = DEFAULT_IGNORE_EXTENSIONS, + noinline pathToName: (Path) -> Name = defaultPathToName, noinline formatResolver: FileFormatResolver, -): DataSource = monitorDataDirectory(typeOf(), path, ignoreExtensions, formatResolver) +): DataSource = monitorDataDirectory(typeOf(), path, pathToName, formatResolver) /** * Read and monitor raw binary data tree from the directory. All files are read as-is (save for meta files). @@ -215,18 +228,23 @@ public inline fun IOPlugin.monitorDataDirectory( @DFExperimental public fun IOPlugin.monitorRawDirectory( path: Path, - ignoreExtensions: Set = DEFAULT_IGNORE_EXTENSIONS, -): DataSource = monitorDataDirectory(path, ignoreExtensions) { _, _ -> IOReader.binary } + pathToName: (Path) -> Name = defaultPathToName, +): DataSource = monitorDataDirectory(path, pathToName) { _, _ -> IOReader.binary } /** - * Write data tree to existing directory or create a new one using default [java.nio.file.FileSystem] provider + * Write the data tree to existing directory or create a new one using default [java.nio.file.FileSystem] provider + * + * @param nameToPath a [Name] to [Path] converter used to create */ @DFExperimental public suspend fun IOPlugin.writeDataDirectory( path: Path, - tree: DataTree, + dataSet: DataSet, format: IOWriter, envelopeFormat: EnvelopeFormat? = null, + nameToPath: (name: Name, data: Data) -> Path = { name, _ -> + Path(name.tokens.joinToString("/") { token -> token.toStringUnescaped() }) + }, ) { withContext(Dispatchers.IO) { if (!Files.exists(path)) { @@ -234,67 +252,54 @@ public suspend fun IOPlugin.writeDataDirectory( } else if (!Files.isDirectory(path)) { error("Can't write a node into file") } - tree.items.forEach { (token, item) -> - val childPath = path.resolve(token.toString()) - when (item) { - is DataTreeItem.Node -> { - writeDataDirectory(childPath, item.tree, format, envelopeFormat) - } - - is DataTreeItem.Leaf -> { - val envelope = item.data.toEnvelope(format) - if (envelopeFormat != null) { - writeEnvelopeFile(childPath, envelope, envelopeFormat) - } else { - writeEnvelopeDirectory(childPath, envelope) - } - } + dataSet.forEach { (name, data) -> + val childPath = path.resolve(nameToPath(name, data)) + childPath.parent.createDirectories() + val envelope = data.toEnvelope(format) + if (envelopeFormat != null) { + writeEnvelopeFile(childPath, envelope, envelopeFormat) + } else { + writeEnvelopeDirectory(childPath, envelope) } } - val treeMeta = tree.meta - writeMetaFile(path, treeMeta) + val directoryMeta = dataSet.meta + writeMetaFile(path, directoryMeta) } } /** - * Reads the specified resources and returns a [DataTree] containing the data. + * Reads the specified resources and returns a [LegacyDataTree] containing the data. * * @param resources The names of the resources to read. * @param classLoader The class loader to use for loading the resources. By default, it uses the current thread's context class loader. * @return A DataTree containing the data read from the resources. */ @DFExperimental -private fun IOPlugin.readResources( +public fun IOPlugin.readResources( vararg resources: String, + pathToName: (Path) -> Name = defaultPathToName, classLoader: ClassLoader = Thread.currentThread().contextClassLoader, -): DataTree { -// require(resource.isNotBlank()) {"Can't mount root resource tree as data root"} - return DataTree { - resources.forEach { resource -> - val path = classLoader.getResource(resource)?.toURI()?.toPath() ?: error( - "Resource with name $resource is not resolved" - ) - node(resource, readRawDirectory(path)) - } +): LegacyDataTree = GenericDataTree { + resources.forEach { resource -> + val path = classLoader.getResource(resource)?.toURI()?.toPath() ?: error( + "Resource with name $resource is not resolved" + ) + node(resource, readRawDirectory(path, pathToName)) } } /** * Add file/directory-based data tree item - * - * @param ignoreExtensions a list of file extensions for which extension should be cut from the resulting item name */ context(IOPlugin) @OptIn(DFInternal::class) @DFExperimental public fun DataSetBuilder.file( path: Path, - ignoreExtensions: Set = DEFAULT_IGNORE_EXTENSIONS, + pathToName: (Path) -> Name = defaultPathToName, formatResolver: FileFormatResolver, ) { - fun defaultPath() = if (path.extension in ignoreExtensions) path.nameWithoutExtension else path.name - try { //If path is a single file or a special directory, read it as single datum if (!Files.isDirectory(path) || Files.list(path).allMatch { it.fileName.toString().startsWith("@") }) { @@ -303,16 +308,16 @@ public fun DataSetBuilder.file( logger.warn { "File format is not resolved for $path. Skipping." } return } - val name: String = data.meta[Envelope.ENVELOPE_NAME_KEY].string ?: defaultPath() - data(name.asName(), data) + val name: Name = data.meta[Envelope.ENVELOPE_NAME_KEY].string?.parseAsName() ?: pathToName(path.last()) + data(name, data) } else { //otherwise, read as directory - val data: DataTree = readDataDirectory(dataType, path, ignoreExtensions, formatResolver) - val name = data.meta[Envelope.ENVELOPE_NAME_KEY].string ?: defaultPath() - node(name.asName(), data) + val data: LegacyDataTree = readDataDirectory(dataType, path, pathToName, formatResolver) + val name = data.meta[Envelope.ENVELOPE_NAME_KEY].string?.parseAsName() ?: pathToName(path.last()) + node(name, data) } } catch (ex: Exception) { - logger.error { "Failed to read file or directory at $path: ${ex.message}" } + logger.error(ex) { "Failed to read file or directory at $path: ${ex.message}" } } } diff --git a/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/workspaceJvm.kt b/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/workspaceJvm.kt index ea6ffb85..67570a04 100644 --- a/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/workspaceJvm.kt +++ b/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/workspaceJvm.kt @@ -1,6 +1,6 @@ package space.kscience.dataforge.workspace -import space.kscience.dataforge.data.DataSet +import space.kscience.dataforge.data.DataTree import space.kscience.dataforge.data.filterByType import space.kscience.dataforge.meta.Meta import space.kscience.dataforge.misc.DFExperimental @@ -16,14 +16,13 @@ import space.kscience.dataforge.names.matches */ @OptIn(DFExperimental::class) public inline fun TaskResultBuilder<*>.dataByType(namePattern: Name? = null): DataSelector = - object : DataSelector { - override suspend fun select(workspace: Workspace, meta: Meta): DataSet = - workspace.data.filterByType { name, _ -> - namePattern == null || name.matches(namePattern) - } + DataSelector { workspace, meta -> + workspace.data.filterByType { name, _ -> + namePattern == null || name.matches(namePattern) + } } public suspend inline fun TaskResultBuilder<*>.fromTask( task: Name, taskMeta: Meta = Meta.EMPTY, -): DataSet = workspace.produce(task, taskMeta).filterByType() \ No newline at end of file +): DataTree = workspace.produce(task, taskMeta).data.filterByType() \ No newline at end of file diff --git a/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/zipData.kt b/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/zipData.kt index 466552c2..02d319de 100644 --- a/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/zipData.kt +++ b/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/zipData.kt @@ -2,8 +2,8 @@ package space.kscience.dataforge.workspace import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.withContext -import space.kscience.dataforge.data.DataTree import space.kscience.dataforge.data.DataTreeItem +import space.kscience.dataforge.data.LegacyDataTree import space.kscience.dataforge.io.* import space.kscience.dataforge.misc.DFExperimental import java.nio.file.Files @@ -47,10 +47,10 @@ private suspend fun ZipOutputStream.writeNode( } /** - * Write this [DataTree] as a zip archive + * Write this [LegacyDataTree] as a zip archive */ @DFExperimental -public suspend fun DataTree.writeZip( +public suspend fun LegacyDataTree.writeZip( path: Path, format: IOFormat, envelopeFormat: EnvelopeFormat = TaggedEnvelopeFormat, diff --git a/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/DataPropagationTest.kt b/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/DataPropagationTest.kt index 18086902..dbc9cb5e 100644 --- a/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/DataPropagationTest.kt +++ b/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/DataPropagationTest.kt @@ -20,13 +20,13 @@ class DataPropagationTestPlugin : WorkspacePlugin() { val result: Data = selectedData.foldToData(0) { result, data -> result + data.value } - data("result", result) + emit("result", result) } val singleData by task { workspace.data.filterByType()["myData[12]"]?.let { - data("result", it) + emit("result", it) } } diff --git a/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/FileDataTest.kt b/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/FileDataTest.kt index 451c76f4..20d03cd3 100644 --- a/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/FileDataTest.kt +++ b/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/FileDataTest.kt @@ -26,7 +26,7 @@ import kotlin.test.assertEquals class FileDataTest { - val dataNode = DataTree { + val dataNode = GenericDataTree { node("dir") { static("a", "Some string") { "content" put "Some string" diff --git a/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/SimpleWorkspaceTest.kt b/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/SimpleWorkspaceTest.kt index b79ed2e7..a41a73bc 100644 --- a/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/SimpleWorkspaceTest.kt +++ b/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/SimpleWorkspaceTest.kt @@ -140,7 +140,7 @@ internal class SimpleWorkspaceTest { val meta = data.meta.toMutableMeta().apply { "newValue" put 22 } - data(data.name + "new", data.map { (data.meta["value"].int ?: 0) + it }) + data(data.name + "new", data.transform { (data.meta["value"].int ?: 0) + it }) } } -- 2.34.1 From 90999f424f2254f3ccc0b8b20f1f01983f627e98 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Mon, 29 Jan 2024 22:10:06 +0300 Subject: [PATCH 17/62] [WIP] change data structure --- .../kscience/dataforge/actions/Action.kt | 8 +- .../kscience/dataforge/actions/MapAction.kt | 2 +- .../dataforge/actions/ReduceAction.kt | 2 +- .../kscience/dataforge/actions/SplitAction.kt | 2 +- .../space/kscience/dataforge/data/Data.kt | 2 +- .../kscience/dataforge/data/DataSource.kt | 36 ++- .../kscience/dataforge/data/GroupRule.kt | 2 +- .../kscience/dataforge/data/NamedData.kt | 2 +- .../kscience/dataforge/data/dataBuilders.kt | 48 +-- .../kscience/dataforge/data/dataTransform.kt | 38 +-- .../kscience/dataforge/data/dataFilterJvm.kt | 4 +- .../dataforge/data/dataSetBuilderInContext.kt | 10 +- ...eBuilderTest.kt => DataTreeBuilderTest.kt} | 2 +- .../kscience/dataforge/io/EnvelopeFormat.kt | 5 +- .../space/kscience/dataforge/io/IOFormat.kt | 4 +- .../space/kscience/dataforge/io/MetaFormat.kt | 2 - .../dataforge/workspace/TaskResult.kt | 24 +- .../kscience/dataforge/workspace/Workspace.kt | 2 +- .../dataforge/workspace/WorkspaceBuilder.kt | 6 +- .../dataforge/workspace/taskBuilders.kt | 20 +- .../dataforge/workspace/FileWorkspaceCache.kt | 4 +- .../workspace/InMemoryWorkspaceCache.kt | 38 +-- .../kscience/dataforge/workspace/fileData.kt | 293 ++++++------------ .../dataforge/workspace/workspaceJvm.kt | 6 +- .../kscience/dataforge/workspace/zipData.kt | 52 ++-- .../workspace/CachingWorkspaceTest.kt | 10 +- .../workspace/DataPropagationTest.kt | 8 +- .../dataforge/workspace/FileDataTest.kt | 39 ++- .../workspace/FileWorkspaceCacheTest.kt | 4 +- .../workspace/SimpleWorkspaceTest.kt | 39 ++- 30 files changed, 316 insertions(+), 398 deletions(-) rename dataforge-data/src/jvmTest/kotlin/space/kscience/dataforge/data/{LegacyGenericDataTreeBuilderTest.kt => DataTreeBuilderTest.kt} (98%) diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/Action.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/Action.kt index b9a9a0a8..ac903aee 100644 --- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/Action.kt +++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/Action.kt @@ -10,7 +10,7 @@ import space.kscience.dataforge.misc.DFExperimental /** * A simple data transformation on a data node. Actions should avoid doing actual dependency evaluation in [execute]. */ -public fun interface Action { +public fun interface Action { /** * Transform the data in the node, producing a new node. By default, it is assumed that all calculations are lazy @@ -24,7 +24,7 @@ public fun interface Action { /** * A convenience method to transform data using given [action] */ -public fun DataTree.transform( +public fun DataTree.transform( action: Action, scope: CoroutineScope, meta: Meta = Meta.EMPTY, @@ -33,11 +33,11 @@ public fun DataTree.transform( /** * Action composition. The result is terminal if one of its parts is terminal */ -public infix fun Action.then(action: Action): Action = +public infix fun Action.then(action: Action): Action = Action { scope, dataSet, meta -> action.execute(scope, this@then.execute(scope, dataSet, meta), meta) } @DFExperimental -public suspend operator fun Action.invoke( +public suspend operator fun Action.invoke( dataSet: DataTree, meta: Meta = Meta.EMPTY, ): DataTree = coroutineScope { execute(this, dataSet, meta) } diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/MapAction.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/MapAction.kt index 60f05910..3cf7c788 100644 --- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/MapAction.kt +++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/MapAction.kt @@ -79,7 +79,7 @@ internal class MapAction( builder.result(env, data.await()) } //setting the data node - emit(newName, newData) + data(newName, newData) } override fun DataSink.generate(data: DataTree, meta: Meta) { diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/ReduceAction.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/ReduceAction.kt index cc27b3d1..5b862046 100644 --- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/ReduceAction.kt +++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/ReduceAction.kt @@ -103,7 +103,7 @@ internal class ReduceAction( meta = groupMeta ) { group.result.invoke(env, it) } - emit(env.name, res) + data(env.name, res) } } } diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/SplitAction.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/SplitAction.kt index a2da44f8..7926ce0b 100644 --- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/SplitAction.kt +++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/SplitAction.kt @@ -64,7 +64,7 @@ internal class SplitAction( ).apply(rule) //data.map(outputType, meta = env.meta) { env.result(it) }.named(fragmentName) - emit( + data( fragmentName, @Suppress("OPT_IN_USAGE") Data(outputType, meta = env.meta, dependencies = listOf(data)) { env.result(data.await()) diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/Data.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/Data.kt index 3668af1e..a0bb58ea 100644 --- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/Data.kt +++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/Data.kt @@ -15,7 +15,7 @@ import kotlin.reflect.typeOf * A data element characterized by its meta */ @DfType(Data.TYPE) -public interface Data : Goal, MetaRepr { +public interface Data : Goal, MetaRepr { /** * Type marker for the data. The type is known before the calculation takes place so it could be checked. */ diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSource.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSource.kt index 8c86d431..d6011839 100644 --- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSource.kt +++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSource.kt @@ -12,7 +12,7 @@ import kotlin.reflect.typeOf /** * A generic data provider */ -public interface DataSource { +public interface DataSource { /** * The minimal common ancestor to all data in the node @@ -28,7 +28,7 @@ public interface DataSource { /** * A data provider with possible dynamic updates */ -public interface ObservableDataSource : DataSource { +public interface ObservableDataSource : DataSource { /** * Flow updates made to the data @@ -39,7 +39,7 @@ public interface ObservableDataSource : DataSource { /** * A tree like structure for data holding */ -public interface GenericDataTree> : DataSource { +public interface GenericDataTree> : DataSource { public val self: TR public val data: Data? @@ -66,7 +66,12 @@ public interface GenericDataTree> : DataSourc } } -public typealias DataTree = GenericDataTree +public typealias DataTree = GenericDataTree> + +/** + * Return a single data in this tree. Throw error if it is not single. + */ +public fun DataTree.single(): NamedData = asSequence().single() /** * An alias for easier access to tree values @@ -79,7 +84,7 @@ public operator fun DataTree.get(name: String): Data? = read(name.pars * Return a sequence of all data items in this tree. * This method does not take updates into account. */ -public fun GenericDataTree>.asSequence( +public fun DataTree.asSequence( namePrefix: Name = Name.EMPTY, ): Sequence> = sequence { data?.let { yield(it.named(Name.EMPTY)) } @@ -100,6 +105,9 @@ public tailrec fun > GenericDataTree.branc else -> items[name.first()]?.branch(name.cutFirst()) } +public fun > GenericDataTree.branch(name: String): TR? = + branch(name.parseAsName()) + public fun GenericDataTree<*, *>.isEmpty(): Boolean = data == null && items.isEmpty() @PublishedApi @@ -113,7 +121,7 @@ internal class FlatDataTree( override val items: Map> get() = dataSet.keys .filter { it.startsWith(prefix) && it.length > prefix.length } - .map { it.tokens[prefix.length + 1] } + .map { it.tokens[prefix.length] } .associateWith { FlatDataTree(dataType, dataSet, prefix + it) } override fun read(name: Name): Data? = dataSet[prefix + name] @@ -133,20 +141,20 @@ internal fun Sequence>.toTree(type: KType): DataTree = public inline fun Sequence>.toTree(): DataTree = FlatDataTree(typeOf(), associate { it.name to it.data }, Name.EMPTY) -public interface GenericObservableDataTree> : GenericDataTree, +public interface GenericObservableDataTree> : GenericDataTree, ObservableDataSource -public typealias ObservableDataTree = GenericObservableDataTree +public typealias ObservableDataTree = GenericObservableDataTree> -public fun DataTree.updates(): Flow> = if (this is ObservableDataTree) updates() else emptyFlow() +public fun DataTree.updates(): Flow> = if (this is GenericObservableDataTree) updates() else emptyFlow() -public fun interface DataSink { - public fun emit(name: Name, data: Data?) +public fun interface DataSink { + public fun data(name: Name, data: Data?) } public class DataTreeBuilder(private val type: KType) : DataSink { private val map = HashMap>() - override fun emit(name: Name, data: Data?) { + override fun data(name: Name, data: Data?) { if (data == null) { map.remove(name) } else { @@ -182,7 +190,7 @@ public interface MutableDataTree : GenericObservableDataTree?) - override fun emit(name: Name, data: Data?): Unit = set(name, data) + override fun data(name: Name, data: Data?): Unit = set(name, data) } public tailrec operator fun MutableDataTree.set(name: Name, data: Data?): Unit { @@ -266,6 +274,6 @@ public fun Sequence>.toObservableTree(dataType: KType, scope: C MutableDataTree(dataType, scope).apply { emitAll(this@toObservableTree) updates.onEach { - emit(it.name, it.data) + data(it.name, it.data) }.launchIn(scope) } diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/GroupRule.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/GroupRule.kt index 27d27507..1a1c66b2 100644 --- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/GroupRule.kt +++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/GroupRule.kt @@ -42,7 +42,7 @@ public interface GroupRule { set.forEach { data -> val tagValue: String = data.meta[key]?.string ?: defaultTagValue - map.getOrPut(tagValue) { DataTreeBuilder(set.dataType) }.emit(data.name,data.data) + map.getOrPut(tagValue) { DataTreeBuilder(set.dataType) }.data(data.name,data.data) } diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/NamedData.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/NamedData.kt index 26ad2dfb..63e36a3f 100644 --- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/NamedData.kt +++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/NamedData.kt @@ -4,7 +4,7 @@ import space.kscience.dataforge.meta.isEmpty import space.kscience.dataforge.misc.Named import space.kscience.dataforge.names.Name -public interface NamedData : Named, Data { +public interface NamedData : Named, Data { override val name: Name public val data: Data } diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataBuilders.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataBuilders.kt index 7eecad8f..72b1bb33 100644 --- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataBuilders.kt +++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataBuilders.kt @@ -7,23 +7,24 @@ import space.kscience.dataforge.meta.Meta import space.kscience.dataforge.meta.MutableMeta import space.kscience.dataforge.misc.DFExperimental import space.kscience.dataforge.names.Name +import space.kscience.dataforge.names.asName import space.kscience.dataforge.names.isEmpty import space.kscience.dataforge.names.plus -public fun DataSink.emit(value: NamedData) { - emit(value.name, value.data) +public fun DataSink.data(value: NamedData) { + data(value.name, value.data) } public fun DataSink.emitAll(sequence: Sequence>) { - sequence.forEach { emit(it) } + sequence.forEach { data(it) } } -public fun DataSink.emitAll(dataTree: DataTree) { +public fun DataSink.branch(dataTree: DataTree) { emitAll(dataTree.asSequence()) } -public inline fun DataSink.emitAll( +public inline fun DataSink.branch( prefix: Name, block: DataSink.() -> Unit, ) { @@ -31,45 +32,50 @@ public inline fun DataSink.emitAll( apply(block) } else { val proxyDataSink = DataSink { nameWithoutPrefix, data -> - this.emit(prefix + nameWithoutPrefix, data) + this.data(prefix + nameWithoutPrefix, data) } proxyDataSink.apply(block) } } +public inline fun DataSink.branch( + prefix: String, + block: DataSink.() -> Unit, +): Unit = branch(prefix.asName(), block) -public fun DataSink.emit(name: String, value: Data) { - emit(Name.parse(name), value) + +public fun DataSink.data(name: String, value: Data) { + data(Name.parse(name), value) } -public fun DataSink.emitAll(name: Name, set: DataTree) { - emitAll(name) { emitAll(set.asSequence()) } +public fun DataSink.branch(name: Name, set: DataTree) { + branch(name) { emitAll(set.asSequence()) } } -public fun DataSink.emitAll(name: String, set: DataTree) { - emitAll(Name.parse(name)) { emitAll(set.asSequence()) } +public fun DataSink.branch(name: String, set: DataTree) { + branch(Name.parse(name)) { emitAll(set.asSequence()) } } /** * Produce lazy [Data] and emit it into the [MutableDataTree] */ -public inline fun DataSink.produce( +public inline fun DataSink.data( name: String, meta: Meta = Meta.EMPTY, noinline producer: suspend () -> T, ) { val data = Data(meta, block = producer) - emit(name, data) + data(name, data) } -public inline fun DataSink.produce( +public inline fun DataSink.data( name: Name, meta: Meta = Meta.EMPTY, noinline producer: suspend () -> T, ) { val data = Data(meta, block = producer) - emit(name, data) + data(name, data) } /** @@ -79,24 +85,24 @@ public inline fun DataSink.static( name: String, data: T, meta: Meta = Meta.EMPTY, -): Unit = emit(name, Data.static(data, meta)) +): Unit = data(name, Data.static(data, meta)) public inline fun DataSink.static( name: Name, data: T, meta: Meta = Meta.EMPTY, -): Unit = emit(name, Data.static(data, meta)) +): Unit = data(name, Data.static(data, meta)) public inline fun DataSink.static( name: String, data: T, mutableMeta: MutableMeta.() -> Unit, -): Unit = emit(Name.parse(name), Data.static(data, Meta(mutableMeta))) +): Unit = data(Name.parse(name), Data.static(data, Meta(mutableMeta))) public fun DataSink.populateFrom(sequence: Sequence>) { sequence.forEach { - emit(it.name, it.data) + data(it.name, it.data) } } @@ -111,7 +117,7 @@ public fun DataSink.populateFrom(tree: DataTree) { @DFExperimental public fun MutableDataTree.populateFrom(flow: ObservableDataSource): Job = flow.updates().onEach { //TODO check if the place is occupied - emit(it.name, it.data) + data(it.name, it.data) }.launchIn(scope) //public fun DataSetBuilder.populateFrom(flow: Flow>) { diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataTransform.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataTransform.kt index 3afc716a..f33ec1f0 100644 --- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataTransform.kt +++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataTransform.kt @@ -9,14 +9,14 @@ import kotlin.coroutines.EmptyCoroutineContext import kotlin.reflect.KType import kotlin.reflect.typeOf -public data class ValueWithMeta(val meta: Meta, val value: T) +public data class ValueWithMeta(val value: T, val meta: Meta) -public suspend fun Data.awaitWithMeta(): ValueWithMeta = ValueWithMeta(meta, await()) +public suspend fun Data.awaitWithMeta(): ValueWithMeta = ValueWithMeta(await(), meta) -public data class NamedValueWithMeta(val name: Name, val meta: Meta, val value: T) +public data class NamedValueWithMeta(val name: Name, val value: T, val meta: Meta) public suspend fun NamedData.awaitWithMeta(): NamedValueWithMeta = - NamedValueWithMeta(name, meta, await()) + NamedValueWithMeta(name, await(), meta) /** @@ -25,7 +25,7 @@ public suspend fun NamedData.awaitWithMeta(): NamedValueWithMeta = * @param meta for the resulting data. By default equals input data. * @param block the transformation itself */ -public inline fun Data.transform( +public inline fun Data.transform( meta: Meta = this.meta, coroutineContext: CoroutineContext = EmptyCoroutineContext, crossinline block: suspend (T) -> R, @@ -36,7 +36,7 @@ public inline fun Data.transform( /** * Combine this data with the other data using [block]. See [Data::map] for other details */ -public inline fun Data.combine( +public inline fun Data.combine( other: Data, meta: Meta = this.meta, coroutineContext: CoroutineContext = EmptyCoroutineContext, @@ -69,7 +69,7 @@ internal fun Map<*, Data<*>>.joinMeta(): Meta = Meta { } @DFInternal -public fun Map>.reduceToData( +public fun Map>.reduceToData( outputType: KType, meta: Meta = joinMeta(), coroutineContext: CoroutineContext = EmptyCoroutineContext, @@ -89,7 +89,7 @@ public fun Map>.reduceToData( * @param T type of the input goal * @param R type of the result goal */ -public inline fun Map>.reduceToData( +public inline fun Map>.reduceToData( meta: Meta = joinMeta(), coroutineContext: CoroutineContext = EmptyCoroutineContext, crossinline block: suspend (Map>) -> R, @@ -104,7 +104,7 @@ public inline fun Map>.reduceToData( //Iterable operations @DFInternal -public inline fun Iterable>.reduceToData( +public inline fun Iterable>.reduceToData( outputType: KType, meta: Meta = joinMeta(), coroutineContext: CoroutineContext = EmptyCoroutineContext, @@ -119,7 +119,7 @@ public inline fun Iterable>.reduceToData( } @OptIn(DFInternal::class) -public inline fun Iterable>.reduceToData( +public inline fun Iterable>.reduceToData( meta: Meta = joinMeta(), coroutineContext: CoroutineContext = EmptyCoroutineContext, crossinline transformation: suspend (Collection>) -> R, @@ -127,7 +127,7 @@ public inline fun Iterable>.reduceToData( transformation(it) } -public inline fun Iterable>.foldToData( +public inline fun Iterable>.foldToData( initial: R, meta: Meta = joinMeta(), coroutineContext: CoroutineContext = EmptyCoroutineContext, @@ -142,7 +142,7 @@ public inline fun Iterable>.foldToData( * Transform an [Iterable] of [NamedData] to a single [Data]. */ @DFInternal -public inline fun Iterable>.reduceNamedToData( +public inline fun Iterable>.reduceNamedToData( outputType: KType, meta: Meta = joinMeta(), coroutineContext: CoroutineContext = EmptyCoroutineContext, @@ -157,7 +157,7 @@ public inline fun Iterable>.reduceNamedToData( } @OptIn(DFInternal::class) -public inline fun Iterable>.reduceNamedToData( +public inline fun Iterable>.reduceNamedToData( meta: Meta = joinMeta(), coroutineContext: CoroutineContext = EmptyCoroutineContext, crossinline transformation: suspend (Collection>) -> R, @@ -168,7 +168,7 @@ public inline fun Iterable>.reduceNamedT /** * Fold a [Iterable] of named data into a single [Data] */ -public inline fun Iterable>.foldNamedToData( +public inline fun Iterable>.foldNamedToData( initial: R, meta: Meta = joinMeta(), coroutineContext: CoroutineContext = EmptyCoroutineContext, @@ -194,18 +194,18 @@ public suspend fun DataTree.transform( val d = Data(outputType, newMeta, coroutineContext, listOf(namedData)) { block(namedData.awaitWithMeta()) } - emit(namedData.name, d) + data(namedData.name, d) } } @OptIn(DFInternal::class) -public suspend inline fun DataTree.transform( +public suspend inline fun DataTree.transform( noinline metaTransform: MutableMeta.() -> Unit = {}, coroutineContext: CoroutineContext = EmptyCoroutineContext, noinline block: suspend (NamedValueWithMeta) -> R, ): DataTree = this@transform.transform(typeOf(), metaTransform, coroutineContext, block) -public inline fun DataTree.forEach(block: (NamedData) -> Unit) { +public inline fun DataTree.forEach(block: (NamedData) -> Unit) { asSequence().forEach(block) } @@ -219,13 +219,13 @@ internal fun DataTree<*>.joinMeta(): Meta = Meta { } } -public inline fun DataTree.reduceToData( +public inline fun DataTree.reduceToData( meta: Meta = joinMeta(), coroutineContext: CoroutineContext = EmptyCoroutineContext, crossinline transformation: suspend (Iterable>) -> R, ): Data = asSequence().asIterable().reduceNamedToData(meta, coroutineContext, transformation) -public inline fun DataTree.foldToData( +public inline fun DataTree.foldToData( initial: R, meta: Meta = joinMeta(), coroutineContext: CoroutineContext = EmptyCoroutineContext, diff --git a/dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/dataFilterJvm.kt b/dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/dataFilterJvm.kt index aa5c500a..48155595 100644 --- a/dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/dataFilterJvm.kt +++ b/dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/dataFilterJvm.kt @@ -14,7 +14,7 @@ import kotlin.reflect.typeOf * Cast the node to given type if the cast is possible or return null */ @Suppress("UNCHECKED_CAST") -private fun Data<*>.castOrNull(type: KType): Data? = +private fun Data<*>.castOrNull(type: KType): Data? = if (!this.type.isSubtypeOf(type)) { null } else { @@ -55,7 +55,7 @@ public inline fun DataTree<*>.filterByType( /** * Select a single datum if it is present and of given [type] */ -public fun DataTree<*>.getByType(type: KType, name: Name): NamedData? = +public fun DataTree<*>.getByType(type: KType, name: Name): NamedData? = get(name)?.castOrNull(type)?.named(name) public inline fun DataTree<*>.getByType(name: Name): NamedData? = diff --git a/dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/dataSetBuilderInContext.kt b/dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/dataSetBuilderInContext.kt index 116ae52d..5f8791d0 100644 --- a/dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/dataSetBuilderInContext.kt +++ b/dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/dataSetBuilderInContext.kt @@ -12,14 +12,14 @@ import space.kscience.dataforge.names.plus */ context(DataSink) public infix fun String.put(data: Data): Unit = - emit(Name.parse(this), data) + data(Name.parse(this), data) /** * Append node */ context(DataSink) public infix fun String.put(dataSet: DataTree): Unit = - emitAll(this, dataSet) + branch(this, dataSet) /** * Build and append node @@ -27,7 +27,7 @@ public infix fun String.put(dataSet: DataTree): Unit = context(DataSink) public infix fun String.put( block: DataSink.() -> Unit, -): Unit = emitAll(Name.parse(this), block) +): Unit = branch(Name.parse(this), block) /** * Copy given data set and mirror its changes to this [LegacyDataTreeBuilder] in [this@setAndObserve]. Returns an update [Job] @@ -37,8 +37,8 @@ public fun CoroutineScope.setAndWatch( name: Name, dataSet: DataTree, ): Job = launch { - emitAll(name, dataSet) + branch(name, dataSet) dataSet.updates().collect { - emit(name + it.name, it.data) + data(name + it.name, it.data) } } \ No newline at end of file diff --git a/dataforge-data/src/jvmTest/kotlin/space/kscience/dataforge/data/LegacyGenericDataTreeBuilderTest.kt b/dataforge-data/src/jvmTest/kotlin/space/kscience/dataforge/data/DataTreeBuilderTest.kt similarity index 98% rename from dataforge-data/src/jvmTest/kotlin/space/kscience/dataforge/data/LegacyGenericDataTreeBuilderTest.kt rename to dataforge-data/src/jvmTest/kotlin/space/kscience/dataforge/data/DataTreeBuilderTest.kt index a871b340..c5c450f7 100644 --- a/dataforge-data/src/jvmTest/kotlin/space/kscience/dataforge/data/LegacyGenericDataTreeBuilderTest.kt +++ b/dataforge-data/src/jvmTest/kotlin/space/kscience/dataforge/data/DataTreeBuilderTest.kt @@ -7,7 +7,7 @@ import kotlin.test.Test import kotlin.test.assertEquals -internal class LegacyGenericDataTreeBuilderTest { +internal class DataTreeBuilderTest { @Test fun testTreeBuild() = runBlocking { val node = DataTree { diff --git a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/EnvelopeFormat.kt b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/EnvelopeFormat.kt index 0e998760..27a192d7 100644 --- a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/EnvelopeFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/EnvelopeFormat.kt @@ -10,10 +10,7 @@ import space.kscience.dataforge.names.asName import kotlin.reflect.KType import kotlin.reflect.typeOf -public interface EnvelopeFormat : IOFormat { - - override val type: KType get() = typeOf() -} +public interface EnvelopeFormat : IOFormat public fun EnvelopeFormat.read(input: Source): Envelope = readFrom(input) diff --git a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/IOFormat.kt b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/IOFormat.kt index 54e305c6..39fa2be1 100644 --- a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/IOFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/IOFormat.kt @@ -49,13 +49,13 @@ public fun interface IOWriter { */ public interface IOFormat : IOReader, IOWriter -public fun Source.readWith(format: IOReader): T = format.readFrom(this) +public fun Source.readWith(format: IOReader): T = format.readFrom(this) /** * Read given binary as an object using given format */ public fun Binary.readWith(format: IOReader): T = read { - readWith(format) + this.readWith(format) } /** diff --git a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/MetaFormat.kt b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/MetaFormat.kt index f864dd2f..b36ddc2a 100644 --- a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/MetaFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/MetaFormat.kt @@ -21,8 +21,6 @@ import kotlin.reflect.typeOf */ public interface MetaFormat : IOFormat { - override val type: KType get() = typeOf() - override fun writeTo(sink: Sink, obj: Meta) { writeMeta(sink, obj, null) } diff --git a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/TaskResult.kt b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/TaskResult.kt index 59cb7461..94da4383 100644 --- a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/TaskResult.kt +++ b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/TaskResult.kt @@ -1,9 +1,13 @@ package space.kscience.dataforge.workspace +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Job +import kotlinx.coroutines.launch import space.kscience.dataforge.data.ObservableDataTree +import space.kscience.dataforge.data.asSequence +import space.kscience.dataforge.data.launch import space.kscience.dataforge.meta.Meta import space.kscience.dataforge.names.Name -import kotlin.reflect.KType /** * A result of a [Task] @@ -12,16 +16,24 @@ import kotlin.reflect.KType * @param taskMeta The configuration of the task that produced the result */ public data class TaskResult( - public val data: ObservableDataTree, + public val content: ObservableDataTree, public val workspace: Workspace, public val taskName: Name, public val taskMeta: Meta, -) { - val dataType: KType get() = data.dataType -} +): ObservableDataTree by content /** * Wrap data into [TaskResult] */ public fun Workspace.wrapResult(data: ObservableDataTree, taskName: Name, taskMeta: Meta): TaskResult = - TaskResult(data, this, taskName, taskMeta) \ No newline at end of file + TaskResult(data, this, taskName, taskMeta) + +/** + * Start computation for all data elements of this node. + * The resulting [Job] is completed only when all of them are completed. + */ +public fun TaskResult<*>.compute(scope: CoroutineScope): Job = scope.launch { + asSequence().forEach { + it.data.launch(scope) + } +} \ No newline at end of file diff --git a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/Workspace.kt b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/Workspace.kt index bc1c19dc..f3ea322c 100644 --- a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/Workspace.kt +++ b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/Workspace.kt @@ -53,7 +53,7 @@ public interface Workspace : ContextAware, Provider, CoroutineScope { } public suspend fun produceData(taskName: Name, taskMeta: Meta, name: Name): Data<*>? = - produce(taskName, taskMeta).data[name] + produce(taskName, taskMeta)[name] public companion object { public const val TYPE: String = "workspace" diff --git a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/WorkspaceBuilder.kt b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/WorkspaceBuilder.kt index e0b7fad7..6ccdcb1c 100644 --- a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/WorkspaceBuilder.kt +++ b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/WorkspaceBuilder.kt @@ -19,12 +19,12 @@ import kotlin.properties.PropertyDelegateProvider import kotlin.properties.ReadOnlyProperty import kotlin.reflect.typeOf -public data class TaskReference(public val taskName: Name, public val task: Task) : DataSelector { +public data class TaskReference(public val taskName: Name, public val task: Task) : DataSelector { @Suppress("UNCHECKED_CAST") override suspend fun select(workspace: Workspace, meta: Meta): DataTree { if (workspace.tasks[taskName] == task) { - return workspace.produce(taskName, meta).data as DataTree + return workspace.produce(taskName, meta) as DataTree } else { error("Task $taskName does not belong to the workspace") } @@ -125,7 +125,7 @@ public class WorkspaceBuilder( /** * Define intrinsic data for the workspace */ - public fun data(builder: DataSink<*>.() -> Unit) { + public fun data(builder: DataSink.() -> Unit) { data.apply(builder) } diff --git a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/taskBuilders.kt b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/taskBuilders.kt index 11594c2b..95e7d481 100644 --- a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/taskBuilders.kt +++ b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/taskBuilders.kt @@ -3,7 +3,7 @@ package space.kscience.dataforge.workspace import space.kscience.dataforge.actions.Action import space.kscience.dataforge.context.PluginFactory import space.kscience.dataforge.data.DataTree -import space.kscience.dataforge.data.emitAll +import space.kscience.dataforge.data.branch import space.kscience.dataforge.data.forEach import space.kscience.dataforge.data.transform import space.kscience.dataforge.meta.* @@ -25,12 +25,12 @@ public val TaskResultBuilder<*>.defaultDependencyMeta: Meta * @param selector a workspace data selector. Could be either task selector or initial data selector. * @param dependencyMeta meta used for selector. The same meta is used for caching. By default, uses [defaultDependencyMeta]. */ -public suspend fun TaskResultBuilder<*>.from( +public suspend fun TaskResultBuilder<*>.from( selector: DataSelector, dependencyMeta: Meta = defaultDependencyMeta, ): DataTree = selector.select(workspace, dependencyMeta) -public suspend inline fun TaskResultBuilder<*>.from( +public suspend inline fun TaskResultBuilder<*>.from( plugin: P, dependencyMeta: Meta = defaultDependencyMeta, selectorBuilder: P.() -> TaskReference, @@ -50,7 +50,7 @@ public suspend inline fun TaskResultBuild * @param dependencyMeta meta used for selector. The same meta is used for caching. By default, uses [defaultDependencyMeta]. * @param selectorBuilder a builder of task from the plugin. */ -public suspend inline fun TaskResultBuilder<*>.from( +public suspend inline fun TaskResultBuilder<*>.from( pluginFactory: PluginFactory

, dependencyMeta: Meta = defaultDependencyMeta, selectorBuilder: P.() -> TaskReference, @@ -77,7 +77,7 @@ public val TaskResultBuilder<*>.allData: DataSelector<*> * @param action process individual data asynchronously. */ @DFExperimental -public suspend inline fun TaskResultBuilder.transformEach( +public suspend inline fun TaskResultBuilder.transformEach( selector: DataSelector, dependencyMeta: Meta = defaultDependencyMeta, dataMetaTransform: MutableMeta.(name: Name) -> Unit = {}, @@ -93,27 +93,27 @@ public suspend inline fun TaskResultBuilder.transf action(it, data.name, meta) } - emit(data.name, res) + data(data.name, res) } } /** * Set given [dataSet] as a task result. */ -public fun TaskResultBuilder.result(dataSet: DataTree) { - emitAll(dataSet) +public fun TaskResultBuilder.result(dataSet: DataTree) { + branch(dataSet) } /** * Use provided [action] to fill the result */ @DFExperimental -public suspend inline fun TaskResultBuilder.actionFrom( +public suspend inline fun TaskResultBuilder.actionFrom( selector: DataSelector, action: Action, dependencyMeta: Meta = defaultDependencyMeta, ) { - emitAll(action.execute(workspace.context, from(selector, dependencyMeta), dependencyMeta)) + branch(action.execute(workspace.context, from(selector, dependencyMeta), dependencyMeta)) } diff --git a/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/FileWorkspaceCache.kt b/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/FileWorkspaceCache.kt index e6ddf026..a43657f8 100644 --- a/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/FileWorkspaceCache.kt +++ b/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/FileWorkspaceCache.kt @@ -95,8 +95,8 @@ public class FileWorkspaceCache(public val cacheDirectory: Path) : WorkspaceCach } - val cachedTree = result.data.asSequence().map { cacheOne(it) } - .toObservableTree(result.dataType, result.workspace, result.data.updates().map { cacheOne(it) }) + val cachedTree = result.asSequence().map { cacheOne(it) } + .toObservableTree(result.dataType, result.workspace, result.updates().map { cacheOne(it) }) return result.workspace.wrapResult(cachedTree, result.taskName, result.taskMeta) } diff --git a/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/InMemoryWorkspaceCache.kt b/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/InMemoryWorkspaceCache.kt index a36e2b90..a3792231 100644 --- a/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/InMemoryWorkspaceCache.kt +++ b/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/InMemoryWorkspaceCache.kt @@ -1,39 +1,39 @@ package space.kscience.dataforge.workspace +import kotlinx.coroutines.flow.map +import space.kscience.dataforge.data.* import space.kscience.dataforge.meta.Meta import space.kscience.dataforge.names.Name import kotlin.reflect.KType import kotlin.reflect.full.isSubtypeOf -private typealias TaskResultId = Pair +private data class TaskResultId(val name: Name, val meta: Meta) public class InMemoryWorkspaceCache : WorkspaceCache { - // never do that at home! - private val cache = HashMap>>() + private val cache = HashMap>>() @Suppress("UNCHECKED_CAST") - private fun TaskData<*>.checkType(taskType: KType): TaskData = - if (type.isSubtypeOf(taskType)) this as TaskData + private fun Data<*>.checkType(taskType: KType): Data = + if (type.isSubtypeOf(taskType)) this as Data else error("Cached data type mismatch: expected $taskType but got $type") - override suspend fun cache(result: TaskResult): TaskResult { - for (d: TaskData in result) { - cache.getOrPut(result.taskName to result.taskMeta) { HashMap() }.getOrPut(d.name) { d } - } - - return object : TaskResult by result { - override fun iterator(): Iterator> = (cache[result.taskName to result.taskMeta] - ?.values?.map { it.checkType(result.dataType) } - ?: emptyList()).iterator() - - override fun get(name: Name): TaskData? { - val cached: TaskData<*> = cache[result.taskName to result.taskMeta]?.get(name) ?: return null - //TODO check types - return cached.checkType(result.dataType) + override suspend fun cache(result: TaskResult): TaskResult { + fun cacheOne(data: NamedData): NamedData { + val cachedData = cache.getOrPut(TaskResultId(result.taskName, result.taskMeta)){ + HashMap() + }.getOrPut(data.name){ + data.data } + return cachedData.checkType(result.dataType).named(data.name) } + + + val cachedTree = result.asSequence().map { cacheOne(it) } + .toObservableTree(result.dataType, result.workspace, result.updates().map { cacheOne(it) }) + + return result.workspace.wrapResult(cachedTree, result.taskName, result.taskMeta) } } diff --git a/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/fileData.kt b/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/fileData.kt index a0e6ab42..4eba363b 100644 --- a/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/fileData.kt +++ b/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/fileData.kt @@ -1,30 +1,24 @@ package space.kscience.dataforge.workspace -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.isActive -import kotlinx.coroutines.launch -import kotlinx.coroutines.withContext -import space.kscience.dataforge.context.error -import space.kscience.dataforge.context.logger -import space.kscience.dataforge.context.warn +import kotlinx.coroutines.* import space.kscience.dataforge.data.* import space.kscience.dataforge.io.* import space.kscience.dataforge.meta.Meta import space.kscience.dataforge.meta.copy -import space.kscience.dataforge.meta.string import space.kscience.dataforge.misc.DFExperimental import space.kscience.dataforge.misc.DFInternal -import space.kscience.dataforge.names.* -import space.kscience.dataforge.workspace.FileData.Companion.defaultPathToName +import space.kscience.dataforge.names.Name +import space.kscience.dataforge.names.NameToken +import space.kscience.dataforge.names.asName +import space.kscience.dataforge.names.plus +import space.kscience.dataforge.workspace.FileData.defaultPathToName import java.nio.file.Files import java.nio.file.Path import java.nio.file.StandardWatchEventKinds import java.nio.file.WatchEvent import java.nio.file.attribute.BasicFileAttributes import java.nio.file.spi.FileSystemProvider -import java.time.Instant import kotlin.io.path.* -import kotlin.reflect.KType import kotlin.reflect.typeOf @@ -33,40 +27,28 @@ import kotlin.reflect.typeOf public typealias FileFormatResolver = (path: Path, meta: Meta) -> IOReader? -/** - * A data based on a filesystem [Path] - */ -public class FileData internal constructor(private val data: Data, public val path: Path) : Data by data { +public object FileData { + public val FILE_KEY: Name = "file".asName() + public val FILE_PATH_KEY: Name = FILE_KEY + "path" + public val FILE_EXTENSION_KEY: Name = FILE_KEY + "extension" + public val FILE_CREATE_TIME_KEY: Name = FILE_KEY + "created" + public val FILE_UPDATE_TIME_KEY: Name = FILE_KEY + "updated" + public const val DF_FILE_EXTENSION: String = "df" + public val DEFAULT_IGNORE_EXTENSIONS: Set = setOf(DF_FILE_EXTENSION) - // public val path: String? get() = meta[META_FILE_PATH_KEY].string -// public val extension: String? get() = meta[META_FILE_EXTENSION_KEY].string -// - public val createdTime: Instant? get() = meta[FILE_CREATE_TIME_KEY].string?.let { Instant.parse(it) } - public val updatedTime: Instant? get() = meta[FILE_UPDATE_TIME_KEY].string?.let { Instant.parse(it) } - - public companion object { - public val FILE_KEY: Name = "file".asName() - public val FILE_PATH_KEY: Name = FILE_KEY + "path" - public val FILE_EXTENSION_KEY: Name = FILE_KEY + "extension" - public val FILE_CREATE_TIME_KEY: Name = FILE_KEY + "created" - public val FILE_UPDATE_TIME_KEY: Name = FILE_KEY + "updated" - public const val DF_FILE_EXTENSION: String = "df" - public val DEFAULT_IGNORE_EXTENSIONS: Set = setOf(DF_FILE_EXTENSION) - - /** - * Transform file name into DataForg name. Ignores DataForge file extensions. - */ - public val defaultPathToName: (Path) -> Name = { path -> - Name( - path.map { segment -> - if (segment.isRegularFile() && segment.extension in DEFAULT_IGNORE_EXTENSIONS) { - NameToken(path.nameWithoutExtension) - } else { - NameToken(path.name) - } + /** + * Transform file name into DataForg name. Ignores DataForge file extensions. + */ + public val defaultPathToName: (Path) -> Name = { path -> + Name( + path.map { segment -> + if (segment.isRegularFile() && segment.extension in DEFAULT_IGNORE_EXTENSIONS) { + NameToken(path.nameWithoutExtension) + } else { + NameToken(path.name) } - ) - } + } + ) } } @@ -75,14 +57,11 @@ public class FileData internal constructor(private val data: Data, public * Read data with supported envelope format and binary format. If the envelope format is null, then read binary directly from file. * The operation is blocking since it must read the meta header. The reading of envelope body is lazy */ -@OptIn(DFInternal::class) -@DFExperimental -public fun IOPlugin.readDataFile( +@OptIn(DFExperimental::class) +public fun IOPlugin.readFileData( path: Path, - formatResolver: FileFormatResolver, -): FileData? { +): Data { val envelope = readEnvelopeFile(path, true) - val format = formatResolver(path, envelope.meta) ?: return null val updatedMeta = envelope.meta.copy { FileData.FILE_PATH_KEY put path.toString() FileData.FILE_EXTENSION_KEY put path.extension @@ -91,146 +70,104 @@ public fun IOPlugin.readDataFile( FileData.FILE_UPDATE_TIME_KEY put attributes.lastModifiedTime().toInstant().toString() FileData.FILE_CREATE_TIME_KEY put attributes.creationTime().toInstant().toString() } - return FileData( - Data(format.type, updatedMeta) { - (envelope.data ?: Binary.EMPTY).readWith(format) - }, - path + return StaticData( + typeOf(), + envelope.data ?: Binary.EMPTY, + updatedMeta ) } +public fun DataSink.file(io: IOPlugin, path: Path, name: Name) { + if (!path.isRegularFile()) error("Only regular files could be handled by this function") + data(name, io.readFileData(path)) +} -context(IOPlugin) @DFExperimental -public fun DataSetBuilder.directory( +public fun DataSink.directory( + io: IOPlugin, path: Path, pathToName: (Path) -> Name = defaultPathToName, - formatResolver: FileFormatResolver, ) { + if (!path.isDirectory()) error("Only directories could be handled by this function") + val metaFile = path.resolve(IOPlugin.META_FILE_NAME) + val dataFile = path.resolve(IOPlugin.DATA_FILE_NAME) + //process root data + if (metaFile.exists() || dataFile.exists()) { + data( + Name.EMPTY, + StaticData( + typeOf(), + dataFile.takeIf { it.exists() }?.asBinary() ?: Binary.EMPTY, + io.readMetaFileOrNull(metaFile) ?: Meta.EMPTY + ) + ) + } Files.list(path).forEach { childPath -> val fileName = childPath.fileName.toString() - if (fileName.startsWith(IOPlugin.META_FILE_NAME)) { - meta(readMetaFile(childPath)) - } else if (!fileName.startsWith("@")) { - file(childPath, pathToName, formatResolver) + if (!fileName.startsWith("@")) { + files(io, childPath, pathToName) } } } -/** - * Read the directory as a data node. If [path] is a zip archive, read it as directory - */ -@DFExperimental -@DFInternal -public fun IOPlugin.readDataDirectory( - type: KType, - path: Path, - pathToName: (Path) -> Name = defaultPathToName, - formatResolver: FileFormatResolver, -): LegacyDataTree { - //read zipped data node - if (path.fileName != null && path.fileName.toString().endsWith(".zip")) { +public fun DataSink.files(io: IOPlugin, path: Path, pathToName: (Path) -> Name = defaultPathToName) { + if (path.isRegularFile() && path.extension == "zip") { //Using explicit Zip file system to avoid bizarre compatibility bugs val fsProvider = FileSystemProvider.installedProviders().find { it.scheme == "jar" } ?: error("Zip file system provider not found") val fs = fsProvider.newFileSystem(path, mapOf("create" to "true")) - return readDataDirectory(type, fs.rootDirectories.first(), pathToName, formatResolver) + return files(io, fs.rootDirectories.first(), pathToName) } - if (!Files.isDirectory(path)) error("Provided path $path is not a directory") - return DataTree(type) { - meta { - FileData.FILE_PATH_KEY put path.toString() - } - directory(path, pathToName, formatResolver) + if (path.isRegularFile()) { + file(io, path, pathToName(path)) + } else { + directory(io, path, pathToName) } } -@OptIn(DFInternal::class) -@DFExperimental -public inline fun IOPlugin.readDataDirectory( - path: Path, - noinline pathToName: (Path) -> Name = defaultPathToName, - noinline formatResolver: FileFormatResolver, -): LegacyDataTree = readDataDirectory(typeOf(), path, pathToName, formatResolver) - -/** - * Read a raw binary data tree from the directory. All files are read as-is (save for meta files). - */ -@DFExperimental -public fun IOPlugin.readRawDirectory( - path: Path, - pathToName: (Path) -> Name = defaultPathToName, -): LegacyDataTree = readDataDirectory(path, pathToName) { _, _ -> IOReader.binary } - private fun Path.toName() = Name(map { NameToken.parse(it.nameWithoutExtension) }) @DFInternal @DFExperimental -public fun IOPlugin.monitorDataDirectory( - type: KType, +public fun DataSink.monitorFiles( + io: IOPlugin, path: Path, pathToName: (Path) -> Name = defaultPathToName, - formatResolver: FileFormatResolver, -): DataSource { - if (path.fileName.toString().endsWith(".zip")) error("Monitoring not supported for ZipFS") - if (!Files.isDirectory(path)) error("Provided path $path is not a directory") - return DataSource(type, context) { - directory(path, pathToName, formatResolver) - launch(Dispatchers.IO) { - val watchService = path.fileSystem.newWatchService() + scope: CoroutineScope = io.context, +): Job { + files(io, path, pathToName) + return scope.launch(Dispatchers.IO) { + val watchService = path.fileSystem.newWatchService() - path.register( - watchService, - StandardWatchEventKinds.ENTRY_DELETE, - StandardWatchEventKinds.ENTRY_MODIFY, - StandardWatchEventKinds.ENTRY_CREATE - ) + path.register( + watchService, + StandardWatchEventKinds.ENTRY_DELETE, + StandardWatchEventKinds.ENTRY_MODIFY, + StandardWatchEventKinds.ENTRY_CREATE + ) - do { - val key = watchService.take() - if (key != null) { - for (event: WatchEvent<*> in key.pollEvents()) { - val eventPath = event.context() as Path - if (event.kind() == StandardWatchEventKinds.ENTRY_DELETE) { - remove(eventPath.toName()) - } else { - val fileName = eventPath.fileName.toString() - if (fileName.startsWith(IOPlugin.META_FILE_NAME)) { - meta(readMetaFile(eventPath)) - } else if (!fileName.startsWith("@")) { - file(eventPath, pathToName, formatResolver) - } + do { + val key = watchService.take() + if (key != null) { + for (event: WatchEvent<*> in key.pollEvents()) { + val eventPath = event.context() as Path + if (event.kind() == StandardWatchEventKinds.ENTRY_DELETE) { + data(eventPath.toName(), null) + } else { + val fileName = eventPath.fileName.toString() + if (!fileName.startsWith("@")) { + files(io, eventPath, pathToName) } } - key.reset() } - } while (isActive && key != null) - } + key.reset() + } + } while (isActive && key != null) } + } - -/** - * Start monitoring given directory ([path]) as a [DataSource]. - */ -@OptIn(DFInternal::class) -@DFExperimental -public inline fun IOPlugin.monitorDataDirectory( - path: Path, - noinline pathToName: (Path) -> Name = defaultPathToName, - noinline formatResolver: FileFormatResolver, -): DataSource = monitorDataDirectory(typeOf(), path, pathToName, formatResolver) - -/** - * Read and monitor raw binary data tree from the directory. All files are read as-is (save for meta files). - */ -@DFExperimental -public fun IOPlugin.monitorRawDirectory( - path: Path, - pathToName: (Path) -> Name = defaultPathToName, -): DataSource = monitorDataDirectory(path, pathToName) { _, _ -> IOReader.binary } - /** * Write the data tree to existing directory or create a new one using default [java.nio.file.FileSystem] provider * @@ -239,7 +176,7 @@ public fun IOPlugin.monitorRawDirectory( @DFExperimental public suspend fun IOPlugin.writeDataDirectory( path: Path, - dataSet: DataSet, + dataSet: DataTree, format: IOWriter, envelopeFormat: EnvelopeFormat? = null, nameToPath: (name: Name, data: Data) -> Path = { name, _ -> @@ -262,62 +199,28 @@ public suspend fun IOPlugin.writeDataDirectory( writeEnvelopeDirectory(childPath, envelope) } } - val directoryMeta = dataSet.meta - writeMetaFile(path, directoryMeta) + dataSet.meta?.let { writeMetaFile(path, it) } + } } /** - * Reads the specified resources and returns a [LegacyDataTree] containing the data. - * * @param resources The names of the resources to read. * @param classLoader The class loader to use for loading the resources. By default, it uses the current thread's context class loader. - * @return A DataTree containing the data read from the resources. */ @DFExperimental -public fun IOPlugin.readResources( +public fun DataSink.resources( + io: IOPlugin, vararg resources: String, pathToName: (Path) -> Name = defaultPathToName, classLoader: ClassLoader = Thread.currentThread().contextClassLoader, -): LegacyDataTree = GenericDataTree { +) { resources.forEach { resource -> val path = classLoader.getResource(resource)?.toURI()?.toPath() ?: error( "Resource with name $resource is not resolved" ) - node(resource, readRawDirectory(path, pathToName)) - } -} - -/** - * Add file/directory-based data tree item - */ -context(IOPlugin) -@OptIn(DFInternal::class) -@DFExperimental -public fun DataSetBuilder.file( - path: Path, - pathToName: (Path) -> Name = defaultPathToName, - formatResolver: FileFormatResolver, -) { - - try { - //If path is a single file or a special directory, read it as single datum - if (!Files.isDirectory(path) || Files.list(path).allMatch { it.fileName.toString().startsWith("@") }) { - val data = readDataFile(path, formatResolver) - if (data == null) { - logger.warn { "File format is not resolved for $path. Skipping." } - return - } - val name: Name = data.meta[Envelope.ENVELOPE_NAME_KEY].string?.parseAsName() ?: pathToName(path.last()) - data(name, data) - } else { - //otherwise, read as directory - val data: LegacyDataTree = readDataDirectory(dataType, path, pathToName, formatResolver) - val name = data.meta[Envelope.ENVELOPE_NAME_KEY].string?.parseAsName() ?: pathToName(path.last()) - node(name, data) + branch(resource.asName()) { + files(io, path, pathToName) } - } catch (ex: Exception) { - logger.error(ex) { "Failed to read file or directory at $path: ${ex.message}" } } } - diff --git a/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/workspaceJvm.kt b/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/workspaceJvm.kt index 67570a04..61caf7e0 100644 --- a/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/workspaceJvm.kt +++ b/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/workspaceJvm.kt @@ -16,8 +16,8 @@ import space.kscience.dataforge.names.matches */ @OptIn(DFExperimental::class) public inline fun TaskResultBuilder<*>.dataByType(namePattern: Name? = null): DataSelector = - DataSelector { workspace, meta -> - workspace.data.filterByType { name, _ -> + DataSelector { workspace, _ -> + workspace.data.filterByType { name, _, _ -> namePattern == null || name.matches(namePattern) } } @@ -25,4 +25,4 @@ public inline fun TaskResultBuilder<*>.dataByType(namePattern: public suspend inline fun TaskResultBuilder<*>.fromTask( task: Name, taskMeta: Meta = Meta.EMPTY, -): DataTree = workspace.produce(task, taskMeta).data.filterByType() \ No newline at end of file +): DataTree = workspace.produce(task, taskMeta).filterByType() \ No newline at end of file diff --git a/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/zipData.kt b/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/zipData.kt index 02d319de..83b617cb 100644 --- a/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/zipData.kt +++ b/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/zipData.kt @@ -2,8 +2,7 @@ package space.kscience.dataforge.workspace import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.withContext -import space.kscience.dataforge.data.DataTreeItem -import space.kscience.dataforge.data.LegacyDataTree +import space.kscience.dataforge.data.DataTree import space.kscience.dataforge.io.* import space.kscience.dataforge.misc.DFExperimental import java.nio.file.Files @@ -15,42 +14,39 @@ import java.util.zip.ZipOutputStream private suspend fun ZipOutputStream.writeNode( name: String, - treeItem: DataTreeItem, + tree: DataTree, dataFormat: IOFormat, envelopeFormat: EnvelopeFormat = TaggedEnvelopeFormat, ): Unit = withContext(Dispatchers.IO) { - when (treeItem) { - is DataTreeItem.Leaf -> { - //TODO add directory-based envelope writer - val envelope = treeItem.data.toEnvelope(dataFormat) - val entry = ZipEntry(name) - putNextEntry(entry) - - //TODO remove additional copy - val bytes = ByteArray { - writeWith(envelopeFormat, envelope) - } - write(bytes) + //TODO add directory-based envelope writer + tree.data?.let { + val envelope = it.toEnvelope(dataFormat) + val entry = ZipEntry(name) + putNextEntry(entry) + //TODO remove additional copy + val bytes = ByteArray { + writeWith(envelopeFormat, envelope) } - - is DataTreeItem.Node -> { - val entry = ZipEntry("$name/") - putNextEntry(entry) - closeEntry() - treeItem.tree.items.forEach { (token, item) -> - val childName = "$name/$token" - writeNode(childName, item, dataFormat, envelopeFormat) - } - } + write(bytes) } + + + val entry = ZipEntry("$name/") + putNextEntry(entry) + closeEntry() + tree.items.forEach { (token, item) -> + val childName = "$name/$token" + writeNode(childName, item, dataFormat, envelopeFormat) + } + } /** - * Write this [LegacyDataTree] as a zip archive + * Write this [DataTree] as a zip archive */ @DFExperimental -public suspend fun LegacyDataTree.writeZip( +public suspend fun DataTree.writeZip( path: Path, format: IOFormat, envelopeFormat: EnvelopeFormat = TaggedEnvelopeFormat, @@ -68,6 +64,6 @@ public suspend fun LegacyDataTree.writeZip( ) val zos = ZipOutputStream(fos) zos.use { - it.writeNode("", DataTreeItem.Node(this@writeZip), format, envelopeFormat) + it.writeNode("", this@writeZip, format, envelopeFormat) } } \ No newline at end of file diff --git a/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/CachingWorkspaceTest.kt b/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/CachingWorkspaceTest.kt index 467b04b0..78ce853e 100644 --- a/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/CachingWorkspaceTest.kt +++ b/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/CachingWorkspaceTest.kt @@ -4,7 +4,6 @@ import kotlinx.coroutines.ExperimentalCoroutinesApi import kotlinx.coroutines.coroutineScope import kotlinx.coroutines.test.runTest import org.junit.jupiter.api.Test -import space.kscience.dataforge.data.startAll import space.kscience.dataforge.data.static import space.kscience.dataforge.meta.Meta import space.kscience.dataforge.meta.boolean @@ -37,7 +36,6 @@ internal class CachingWorkspaceTest { } } - @Suppress("UNUSED_VARIABLE") val doSecond by task { transformEach( doFirst, @@ -54,11 +52,11 @@ internal class CachingWorkspaceTest { val secondB = workspace.produce("doSecond", Meta { "flag" put true }) val secondC = workspace.produce("doSecond") coroutineScope { - first.startAll(this) - secondA.startAll(this) - secondB.startAll(this) + first.compute(this) + secondA.compute(this) + secondB.compute(this) //repeat to check caching - secondC.startAll(this) + secondC.compute(this) } assertEquals(10, firstCounter) assertEquals(10, secondCounter) diff --git a/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/DataPropagationTest.kt b/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/DataPropagationTest.kt index dbc9cb5e..403b1c51 100644 --- a/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/DataPropagationTest.kt +++ b/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/DataPropagationTest.kt @@ -20,13 +20,13 @@ class DataPropagationTestPlugin : WorkspacePlugin() { val result: Data = selectedData.foldToData(0) { result, data -> result + data.value } - emit("result", result) + data("result", result) } val singleData by task { workspace.data.filterByType()["myData[12]"]?.let { - emit("result", it) + data("result", it) } } @@ -55,12 +55,12 @@ class DataPropagationTest { @Test fun testAllData() = runTest { val node = testWorkspace.produce("Test.allData") - assertEquals(4950, node.asSequence().single().await()) + assertEquals(4950, node.content.asSequence().single().await()) } @Test fun testSingleData() = runTest { val node = testWorkspace.produce("Test.singleData") - assertEquals(12, node.asSequence().single().await()) + assertEquals(12, node.content.asSequence().single().await()) } } \ No newline at end of file diff --git a/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/FileDataTest.kt b/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/FileDataTest.kt index 20d03cd3..1a21fb17 100644 --- a/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/FileDataTest.kt +++ b/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/FileDataTest.kt @@ -9,38 +9,32 @@ import kotlinx.io.writeString import space.kscience.dataforge.context.Context import space.kscience.dataforge.context.Global import space.kscience.dataforge.data.* -import space.kscience.dataforge.io.Envelope -import space.kscience.dataforge.io.IOFormat -import space.kscience.dataforge.io.io -import space.kscience.dataforge.io.readEnvelopeFile +import space.kscience.dataforge.io.* import space.kscience.dataforge.io.yaml.YamlPlugin import space.kscience.dataforge.meta.get import space.kscience.dataforge.misc.DFExperimental import java.nio.file.Files import kotlin.io.path.fileSize import kotlin.io.path.toPath -import kotlin.reflect.KType -import kotlin.reflect.typeOf import kotlin.test.Test import kotlin.test.assertEquals class FileDataTest { - val dataNode = GenericDataTree { - node("dir") { + val dataNode = DataTree { + branch("dir") { static("a", "Some string") { "content" put "Some string" } } static("b", "root data") - meta { - "content" put "This is root meta node" - } +// meta { +// "content" put "This is root meta node" +// } } object StringIOFormat : IOFormat { - override val type: KType get() = typeOf() override fun writeTo(sink: Sink, obj: String) { sink.writeString(obj) @@ -52,11 +46,13 @@ class FileDataTest { @Test @DFExperimental fun testDataWriteRead() = with(Global.io) { + val io = Global.io val dir = Files.createTempDirectory("df_data_node") runBlocking { writeDataDirectory(dir, dataNode, StringIOFormat) println(dir.toUri().toString()) - val reconstructed = readDataDirectory(dir) { _, _ -> StringIOFormat } + val reconstructed = DataTree { files(io, dir) } + .transform { (_, value) -> value.toByteArray().decodeToString() } assertEquals(dataNode["dir.a"]?.meta?.get("content"), reconstructed["dir.a"]?.meta?.get("content")) assertEquals(dataNode["b"]?.await(), reconstructed["b"]?.await()) } @@ -66,14 +62,15 @@ class FileDataTest { @Test @DFExperimental fun testZipWriteRead() = runTest { - with(Global.io) { - val zip = Files.createTempFile("df_data_node", ".zip") - dataNode.writeZip(zip, StringIOFormat) - println(zip.toUri().toString()) - val reconstructed = readDataDirectory(zip) { _, _ -> StringIOFormat } - assertEquals(dataNode["dir.a"]?.meta?.get("content"), reconstructed["dir.a"]?.meta?.get("content")) - assertEquals(dataNode["b"]?.await(), reconstructed["b"]?.await()) - } + val io = Global.io + val zip = Files.createTempFile("df_data_node", ".zip") + dataNode.writeZip(zip, StringIOFormat) + println(zip.toUri().toString()) + val reconstructed = DataTree { files(io, zip) } + .transform { (_, value) -> value.toByteArray().decodeToString() } + assertEquals(dataNode["dir.a"]?.meta?.get("content"), reconstructed["dir.a"]?.meta?.get("content")) + assertEquals(dataNode["b"]?.await(), reconstructed["b"]?.await()) + } @OptIn(DFExperimental::class) diff --git a/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/FileWorkspaceCacheTest.kt b/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/FileWorkspaceCacheTest.kt index 03fda0eb..ee497e1b 100644 --- a/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/FileWorkspaceCacheTest.kt +++ b/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/FileWorkspaceCacheTest.kt @@ -3,7 +3,6 @@ package space.kscience.dataforge.workspace import kotlinx.coroutines.ExperimentalCoroutinesApi import kotlinx.coroutines.test.runTest import org.junit.jupiter.api.Test -import space.kscience.dataforge.data.startAll import space.kscience.dataforge.data.static import space.kscience.dataforge.misc.DFExperimental import java.nio.file.Files @@ -22,13 +21,12 @@ class FileWorkspaceCacheTest { } fileCache(Files.createTempDirectory("dataforge-temporary-cache")) - @Suppress("UNUSED_VARIABLE") val echo by task { transformEach(dataByType()) { arg, _, _ -> arg } } } - workspace.produce("echo").startAll(this) + workspace.produce("echo").compute(this) } } \ No newline at end of file diff --git a/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/SimpleWorkspaceTest.kt b/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/SimpleWorkspaceTest.kt index a41a73bc..91fa2f6c 100644 --- a/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/SimpleWorkspaceTest.kt +++ b/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/SimpleWorkspaceTest.kt @@ -27,8 +27,8 @@ public fun

P.toFactory(): PluginFactory

= object : PluginFactory override val tag: PluginTag = this@toFactory.tag } -public fun Workspace.produceBlocking(task: String, block: MutableMeta.() -> Unit = {}): DataSet = runBlocking { - produce(task, block) +public fun Workspace.produceBlocking(task: String, block: MutableMeta.() -> Unit = {}): DataTree<*> = runBlocking { + produce(task, block).content } @OptIn(DFExperimental::class) @@ -68,8 +68,8 @@ internal class SimpleWorkspaceTest { val filterOne by task { val name by taskMeta.string { error("Name field not defined") } - from(testPluginFactory) { test }.getByType(name)?.let { source -> - data(source.name, source.map { it }) + from(testPluginFactory) { test }[name]?.let { source: Data -> + data(name, source) } } @@ -110,14 +110,14 @@ internal class SimpleWorkspaceTest { } val averageByGroup by task { - val evenSum = workspace.data.filterByType { name, _ -> + val evenSum = workspace.data.filterByType { name, _, _ -> name.toString().toInt() % 2 == 0 }.foldToData(0) { l, r -> l + r.value } data("even", evenSum) - val oddSum = workspace.data.filterByType { name, _ -> + val oddSum = workspace.data.filterByType { name, _, _ -> name.toString().toInt() % 2 == 1 }.foldToData(0) { l, r -> l + r.value @@ -159,7 +159,7 @@ internal class SimpleWorkspaceTest { @Timeout(1) fun testMetaPropagation() = runTest { val node = workspace.produce("sum") { "testFlag" put true } - val res = node.asSequence().single().await() + val res = node.single().await() } @Test @@ -170,20 +170,25 @@ internal class SimpleWorkspaceTest { } @Test - fun testFullSquare() { - runBlocking { - val node = workspace.produce("fullSquare") - println(node.toMeta()) + fun testFullSquare() = runTest { + val result = workspace.produce("fullSquare") + result.forEach { + println( + """ + Name: ${it.name} + Meta: ${it.meta} + Data: ${it.data.await()} + """.trimIndent() + ) } } @Test - fun testFilter() { - runBlocking { - val node = workspace.produce("filterOne") { - "name" put "myData[12]" - } - assertEquals(12, node.single().await()) + fun testFilter() = runTest { + val node = workspace.produce("filterOne") { + "name" put "myData[12]" } + assertEquals(12, node.single().await()) } + } \ No newline at end of file -- 2.34.1 From 466e46098949663d2da39a8403e059e59256c02c Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Sat, 3 Feb 2024 17:25:49 +0300 Subject: [PATCH 18/62] Data tree refactored to a uniform tree instead of sealed class. --- .../dataforge/actions/AbstractAction.kt | 30 +++- .../kscience/dataforge/actions/Action.kt | 18 +-- .../kscience/dataforge/actions/MapAction.kt | 4 +- .../dataforge/actions/ReduceAction.kt | 2 +- .../kscience/dataforge/actions/SplitAction.kt | 4 +- .../kscience/dataforge/data/DataSource.kt | 143 ++++++++++++------ .../kscience/dataforge/data/GroupRule.kt | 4 +- .../kscience/dataforge/data/dataBuilders.kt | 75 ++++----- .../kscience/dataforge/data/dataTransform.kt | 2 +- .../dataforge/data/dataSetBuilderInContext.kt | 19 +-- .../kscience/dataforge/data/ActionsTest.kt | 20 ++- .../dataforge/data/DataTreeBuilderTest.kt | 81 ++++------ .../kscience/dataforge/workspace/Task.kt | 13 +- .../dataforge/workspace/TaskResult.kt | 10 +- .../dataforge/workspace/WorkspaceBuilder.kt | 2 +- .../dataforge/workspace/taskBuilders.kt | 4 +- .../kscience/dataforge/workspace/fileData.kt | 124 ++++++++------- .../kscience/dataforge/workspace/zipData.kt | 69 +++------ .../workspace/CachingWorkspaceTest.kt | 19 +-- .../workspace/DataPropagationTest.kt | 6 +- .../dataforge/workspace/FileDataTest.kt | 28 ++-- .../workspace/FileWorkspaceCacheTest.kt | 6 +- .../workspace/SimpleWorkspaceTest.kt | 16 +- 23 files changed, 353 insertions(+), 346 deletions(-) diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/AbstractAction.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/AbstractAction.kt index 33d4ca05..7cd1ced5 100644 --- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/AbstractAction.kt +++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/AbstractAction.kt @@ -1,9 +1,11 @@ package space.kscience.dataforge.actions -import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.flow.launchIn +import kotlinx.coroutines.flow.onEach import kotlinx.coroutines.launch import space.kscience.dataforge.data.* import space.kscience.dataforge.meta.Meta +import space.kscience.dataforge.misc.DFInternal import space.kscience.dataforge.names.Name import space.kscience.dataforge.names.startsWith import kotlin.reflect.KType @@ -33,26 +35,38 @@ public abstract class AbstractAction( /** * Update part of the data set using provided data + * + * @param source the source data tree in case we need several data items to update */ protected open fun DataSink.update( - allData: DataTree, + source: DataTree, meta: Meta, namedData: NamedData, ){ //by default regenerate the whole data set - generate(allData,meta) + generate(source,meta) } + @OptIn(DFInternal::class) override fun execute( - scope: CoroutineScope, dataSet: DataTree, meta: Meta, - ): ObservableDataTree = MutableDataTree(outputType, scope).apply { - generate(dataSet, meta) - scope.launch { - dataSet.updates().collect { + ): DataTree = if(dataSet.isObservable()) { + MutableDataTree(outputType, dataSet.updatesScope).apply { + generate(dataSet, meta) + dataSet.updates().onEach { update(dataSet, meta, it) + }.launchIn(updatesScope) + + //close updates when the source is closed + updatesScope.launch { + dataSet.awaitClose() + close() } } + } else{ + DataTree(outputType){ + generate(dataSet, meta) + } } } diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/Action.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/Action.kt index ac903aee..5ed60db9 100644 --- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/Action.kt +++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/Action.kt @@ -1,9 +1,6 @@ package space.kscience.dataforge.actions -import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.coroutineScope import space.kscience.dataforge.data.DataTree -import space.kscience.dataforge.data.ObservableDataTree import space.kscience.dataforge.meta.Meta import space.kscience.dataforge.misc.DFExperimental @@ -16,7 +13,7 @@ public fun interface Action { * Transform the data in the node, producing a new node. By default, it is assumed that all calculations are lazy * so not actual computation is started at this moment. */ - public fun execute(scope: CoroutineScope, dataSet: DataTree, meta: Meta): ObservableDataTree + public fun execute(dataSet: DataTree, meta: Meta): DataTree public companion object } @@ -26,20 +23,21 @@ public fun interface Action { */ public fun DataTree.transform( action: Action, - scope: CoroutineScope, meta: Meta = Meta.EMPTY, -): DataTree = action.execute(scope, this, meta) +): DataTree = action.execute(this, meta) /** * Action composition. The result is terminal if one of its parts is terminal */ -public infix fun Action.then(action: Action): Action = - Action { scope, dataSet, meta -> action.execute(scope, this@then.execute(scope, dataSet, meta), meta) } +public infix fun Action.then(action: Action): Action = Action { dataSet, meta -> + action.execute(this@then.execute(dataSet, meta), meta) +} @DFExperimental -public suspend operator fun Action.invoke( +public operator fun Action.invoke( dataSet: DataTree, meta: Meta = Meta.EMPTY, -): DataTree = coroutineScope { execute(this, dataSet, meta) } +): DataTree = execute(dataSet, meta) + diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/MapAction.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/MapAction.kt index 3cf7c788..1f40ed73 100644 --- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/MapAction.kt +++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/MapAction.kt @@ -79,14 +79,14 @@ internal class MapAction( builder.result(env, data.await()) } //setting the data node - data(newName, newData) + put(newName, newData) } override fun DataSink.generate(data: DataTree, meta: Meta) { data.forEach { mapOne(it.name, it.data, meta) } } - override fun DataSink.update(allData: DataTree, meta: Meta, namedData: NamedData) { + override fun DataSink.update(source: DataTree, meta: Meta, namedData: NamedData) { mapOne(namedData.name, namedData.data, namedData.meta) } } diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/ReduceAction.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/ReduceAction.kt index 5b862046..9440be55 100644 --- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/ReduceAction.kt +++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/ReduceAction.kt @@ -103,7 +103,7 @@ internal class ReduceAction( meta = groupMeta ) { group.result.invoke(env, it) } - data(env.name, res) + put(env.name, res) } } } diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/SplitAction.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/SplitAction.kt index 7926ce0b..057419a7 100644 --- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/SplitAction.kt +++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/SplitAction.kt @@ -64,7 +64,7 @@ internal class SplitAction( ).apply(rule) //data.map(outputType, meta = env.meta) { env.result(it) }.named(fragmentName) - data( + put( fragmentName, @Suppress("OPT_IN_USAGE") Data(outputType, meta = env.meta, dependencies = listOf(data)) { env.result(data.await()) @@ -77,7 +77,7 @@ internal class SplitAction( data.forEach { splitOne(it.name, it.data, meta) } } - override fun DataSink.update(allData: DataTree, meta: Meta, namedData: NamedData) { + override fun DataSink.update(source: DataTree, meta: Meta, namedData: NamedData) { splitOne(namedData.name, namedData.data, namedData.meta) } } diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSource.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSource.kt index d6011839..d379d027 100644 --- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSource.kt +++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSource.kt @@ -1,11 +1,11 @@ package space.kscience.dataforge.data -import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.* import kotlinx.coroutines.flow.* -import kotlinx.coroutines.launch import space.kscience.dataforge.meta.Meta import space.kscience.dataforge.misc.DFInternal import space.kscience.dataforge.names.* +import kotlin.contracts.contract import kotlin.reflect.KType import kotlin.reflect.typeOf @@ -66,7 +66,7 @@ public interface GenericDataTree> : DataS } } -public typealias DataTree = GenericDataTree> +public typealias DataTree = GenericDataTree> /** * Return a single data in this tree. Throw error if it is not single. @@ -87,7 +87,7 @@ public operator fun DataTree.get(name: String): Data? = read(name.pars public fun DataTree.asSequence( namePrefix: Name = Name.EMPTY, ): Sequence> = sequence { - data?.let { yield(it.named(Name.EMPTY)) } + data?.let { yield(it.named(namePrefix)) } items.forEach { (token, tree) -> yieldAll(tree.asSequence(namePrefix + token)) } @@ -113,8 +113,8 @@ public fun GenericDataTree<*, *>.isEmpty(): Boolean = data == null && items.isEm @PublishedApi internal class FlatDataTree( override val dataType: KType, - val dataSet: Map>, - val prefix: Name, + private val dataSet: Map>, + private val prefix: Name, ) : GenericDataTree> { override val self: FlatDataTree get() = this override val data: Data? get() = dataSet[prefix] @@ -141,20 +141,56 @@ internal fun Sequence>.toTree(type: KType): DataTree = public inline fun Sequence>.toTree(): DataTree = FlatDataTree(typeOf(), associate { it.name to it.data }, Name.EMPTY) -public interface GenericObservableDataTree> : GenericDataTree, - ObservableDataSource +public interface GenericObservableDataTree> : + GenericDataTree, ObservableDataSource, AutoCloseable { + + /** + * A scope that is used to propagate updates. When this scope is closed, no new updates could arrive. + */ + public val updatesScope: CoroutineScope + + /** + * Close this data tree updates channel + */ + override fun close() { + updatesScope.cancel() + } + +} public typealias ObservableDataTree = GenericObservableDataTree> -public fun DataTree.updates(): Flow> = if (this is GenericObservableDataTree) updates() else emptyFlow() - -public fun interface DataSink { - public fun data(name: Name, data: Data?) +/** + * Check if the [DataTree] is observable + */ +public fun DataTree.isObservable(): Boolean { + contract { + returns(true) implies (this@isObservable is GenericObservableDataTree) + } + return this is GenericObservableDataTree } +/** + * Wait for this data tree to stop spawning updates (updatesScope is closed). + * If this [DataTree] is not observable, return immediately. + */ +public suspend fun DataTree.awaitClose() { + if (isObservable()) { + updatesScope.coroutineContext[Job]?.join() + } +} + +public fun DataTree.updates(): Flow> = + if (this is GenericObservableDataTree) updates() else emptyFlow() + +public fun interface DataSink { + public fun put(name: Name, data: Data?) +} + +@DFInternal public class DataTreeBuilder(private val type: KType) : DataSink { private val map = HashMap>() - override fun data(name: Name, data: Data?) { + override fun put(name: Name, data: Data?) { if (data == null) { map.remove(name) } else { @@ -174,6 +210,7 @@ public inline fun DataTree( /** * Create and a data tree. */ +@OptIn(DFInternal::class) public inline fun DataTree( generator: DataSink.() -> Unit, ): DataTree = DataTreeBuilder(typeOf()).apply(generator).build() @@ -182,77 +219,88 @@ public inline fun DataTree( * A mutable version of [GenericDataTree] */ public interface MutableDataTree : GenericObservableDataTree>, DataSink { - public val scope: CoroutineScope - override var data: Data? override val items: Map> + public fun getOrCreateItem(token: NameToken): MutableDataTree + public operator fun set(token: NameToken, data: Data?) - override fun data(name: Name, data: Data?): Unit = set(name, data) + override fun put(name: Name, data: Data?): Unit = set(name, data) } public tailrec operator fun MutableDataTree.set(name: Name, data: Data?): Unit { when (name.length) { 0 -> this.data = data 1 -> set(name.first(), data) - else -> items[name.first()]?.set(name.cutFirst(), data) + else -> getOrCreateItem(name.first())[name.cutFirst()] = data } } -private class ObservableMutableDataTreeImpl( +private class MutableDataTreeImpl( override val dataType: KType, - override val scope: CoroutineScope, + override val updatesScope: CoroutineScope, ) : MutableDataTree { + private val updates = MutableSharedFlow>() private val children = HashMap>() override var data: Data? = null set(value) { + if (!updatesScope.isActive) error("Can't send updates to closed MutableDataTree") field = value if (value != null) { - scope.launch { + updatesScope.launch { updates.emit(value.named(Name.EMPTY)) } } } override val items: Map> get() = children + + override fun getOrCreateItem(token: NameToken): MutableDataTree = children.getOrPut(token){ + MutableDataTreeImpl(dataType, updatesScope) + } + override val self: MutableDataTree get() = this override fun set(token: NameToken, data: Data?) { - children.getOrPut(token) { - ObservableMutableDataTreeImpl(dataType, scope).also { subTree -> - subTree.updates().onEach { - updates.emit(it.named(token + it.name)) - }.launchIn(scope) - } - }.data = data + if (!updatesScope.isActive) error("Can't send updates to closed MutableDataTree") + val subTree = getOrCreateItem(token) + subTree.updates().onEach { + updates.emit(it.named(token + it.name)) + }.launchIn(updatesScope) + subTree.data = data } - override fun updates(): Flow> = flow { - //emit this node updates - updates.collect { - emit(it) - } - } + override fun updates(): Flow> = updates } +/** + * Create a new [MutableDataTree] + * + * @param parentScope a [CoroutineScope] to control data propagation. By default uses [GlobalScope] + */ +@OptIn(DelicateCoroutinesApi::class) public fun MutableDataTree( type: KType, - scope: CoroutineScope -): MutableDataTree = ObservableMutableDataTreeImpl(type, scope) + parentScope: CoroutineScope = GlobalScope, +): MutableDataTree = MutableDataTreeImpl( + type, + CoroutineScope(parentScope.coroutineContext + Job(parentScope.coroutineContext[Job])) +) /** * Create and initialize a observable mutable data tree. */ +@OptIn(DelicateCoroutinesApi::class) public inline fun MutableDataTree( - scope: CoroutineScope, + parentScope: CoroutineScope = GlobalScope, generator: MutableDataTree.() -> Unit = {}, -): MutableDataTree = MutableDataTree(typeOf(), scope).apply { generator() } +): MutableDataTree = MutableDataTree(typeOf(), parentScope).apply { generator() } //@DFInternal //public fun ObservableDataTree( @@ -262,18 +310,21 @@ public inline fun MutableDataTree( //): ObservableDataTree = MutableDataTree(type, scope.coroutineContext).apply(generator) public inline fun ObservableDataTree( - scope: CoroutineScope, + parentScope: CoroutineScope, generator: MutableDataTree.() -> Unit = {}, -): ObservableDataTree = MutableDataTree(typeOf(), scope).apply(generator) +): ObservableDataTree = MutableDataTree(typeOf(), parentScope).apply(generator) /** * Collect a [Sequence] into an observable tree with additional [updates] */ -public fun Sequence>.toObservableTree(dataType: KType, scope: CoroutineScope, updates: Flow>): ObservableDataTree = - MutableDataTree(dataType, scope).apply { - emitAll(this@toObservableTree) - updates.onEach { - data(it.name, it.data) - }.launchIn(scope) - } +public fun Sequence>.toObservableTree( + dataType: KType, + parentScope: CoroutineScope, + updates: Flow>, +): ObservableDataTree = MutableDataTree(dataType, parentScope).apply { + this.putAll(this@toObservableTree) + updates.onEach { + put(it.name, it.data) + }.launchIn(updatesScope) +} diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/GroupRule.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/GroupRule.kt index 1a1c66b2..90486d85 100644 --- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/GroupRule.kt +++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/GroupRule.kt @@ -17,6 +17,7 @@ package space.kscience.dataforge.data import space.kscience.dataforge.meta.get import space.kscience.dataforge.meta.string +import space.kscience.dataforge.misc.DFInternal public interface GroupRule { public fun gather(set: DataTree): Map> @@ -30,6 +31,7 @@ public interface GroupRule { * @param defaultTagValue * @return */ + @OptIn(DFInternal::class) public fun byMetaValue( key: String, defaultTagValue: String, @@ -42,7 +44,7 @@ public interface GroupRule { set.forEach { data -> val tagValue: String = data.meta[key]?.string ?: defaultTagValue - map.getOrPut(tagValue) { DataTreeBuilder(set.dataType) }.data(data.name,data.data) + map.getOrPut(tagValue) { DataTreeBuilder(set.dataType) }.put(data.name, data.data) } diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataBuilders.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataBuilders.kt index 72b1bb33..8e8b6eaa 100644 --- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataBuilders.kt +++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataBuilders.kt @@ -12,16 +12,12 @@ import space.kscience.dataforge.names.isEmpty import space.kscience.dataforge.names.plus -public fun DataSink.data(value: NamedData) { - data(value.name, value.data) -} - -public fun DataSink.emitAll(sequence: Sequence>) { - sequence.forEach { data(it) } +public fun DataSink.put(value: NamedData) { + put(value.name, value.data) } public fun DataSink.branch(dataTree: DataTree) { - emitAll(dataTree.asSequence()) + putAll(dataTree.asSequence()) } public inline fun DataSink.branch( @@ -32,7 +28,7 @@ public inline fun DataSink.branch( apply(block) } else { val proxyDataSink = DataSink { nameWithoutPrefix, data -> - this.data(prefix + nameWithoutPrefix, data) + this.put(prefix + nameWithoutPrefix, data) } proxyDataSink.apply(block) @@ -45,69 +41,69 @@ public inline fun DataSink.branch( ): Unit = branch(prefix.asName(), block) -public fun DataSink.data(name: String, value: Data) { - data(Name.parse(name), value) +public fun DataSink.put(name: String, value: Data) { + put(Name.parse(name), value) } public fun DataSink.branch(name: Name, set: DataTree) { - branch(name) { emitAll(set.asSequence()) } + branch(name) { putAll(set.asSequence()) } } public fun DataSink.branch(name: String, set: DataTree) { - branch(Name.parse(name)) { emitAll(set.asSequence()) } + branch(Name.parse(name)) { putAll(set.asSequence()) } } /** * Produce lazy [Data] and emit it into the [MutableDataTree] */ -public inline fun DataSink.data( +public inline fun DataSink.put( name: String, meta: Meta = Meta.EMPTY, noinline producer: suspend () -> T, ) { val data = Data(meta, block = producer) - data(name, data) + put(name, data) } -public inline fun DataSink.data( +public inline fun DataSink.put( name: Name, meta: Meta = Meta.EMPTY, noinline producer: suspend () -> T, ) { val data = Data(meta, block = producer) - data(name, data) + put(name, data) } /** * Emit static data with the fixed value */ -public inline fun DataSink.static( +public inline fun DataSink.wrap( name: String, data: T, meta: Meta = Meta.EMPTY, -): Unit = data(name, Data.static(data, meta)) +): Unit = put(name, Data.static(data, meta)) -public inline fun DataSink.static( +public inline fun DataSink.wrap( name: Name, data: T, meta: Meta = Meta.EMPTY, -): Unit = data(name, Data.static(data, meta)) +): Unit = put(name, Data.static(data, meta)) -public inline fun DataSink.static( +public inline fun DataSink.wrap( name: String, data: T, mutableMeta: MutableMeta.() -> Unit, -): Unit = data(Name.parse(name), Data.static(data, Meta(mutableMeta))) +): Unit = put(Name.parse(name), Data.static(data, Meta(mutableMeta))) -public fun DataSink.populateFrom(sequence: Sequence>) { +public fun DataSink.putAll(sequence: Sequence>) { sequence.forEach { - data(it.name, it.data) + put(it.name, it.data) } } -public fun DataSink.populateFrom(tree: DataTree) { - populateFrom(tree.asSequence()) +public fun DataSink.putAll(tree: DataTree) { + this.putAll(tree.asSequence()) } @@ -115,13 +111,22 @@ public fun DataSink.populateFrom(tree: DataTree) { * Update data with given node data and meta with node meta. */ @DFExperimental -public fun MutableDataTree.populateFrom(flow: ObservableDataSource): Job = flow.updates().onEach { - //TODO check if the place is occupied - data(it.name, it.data) -}.launchIn(scope) +public fun MutableDataTree.putAll(source: DataTree) { + source.forEach { + put(it.name, it.data) + } +} -//public fun DataSetBuilder.populateFrom(flow: Flow>) { -// flow.collect { -// data(it.name, it.data) -// } -//} +/** + * Copy given data set and mirror its changes to this [DataSink] in [this@setAndObserve]. Returns an update [Job] + */ +public fun DataSink.watchBranch( + name: Name, + dataSet: ObservableDataTree, +): Job { + branch(name, dataSet) + return dataSet.updates().onEach { + put(name + it.name, it.data) + }.launchIn(dataSet.updatesScope) + +} \ No newline at end of file diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataTransform.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataTransform.kt index f33ec1f0..8c7ce70e 100644 --- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataTransform.kt +++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataTransform.kt @@ -194,7 +194,7 @@ public suspend fun DataTree.transform( val d = Data(outputType, newMeta, coroutineContext, listOf(namedData)) { block(namedData.awaitWithMeta()) } - data(namedData.name, d) + put(namedData.name, d) } } diff --git a/dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/dataSetBuilderInContext.kt b/dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/dataSetBuilderInContext.kt index 5f8791d0..cfccb02b 100644 --- a/dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/dataSetBuilderInContext.kt +++ b/dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/dataSetBuilderInContext.kt @@ -1,10 +1,6 @@ package space.kscience.dataforge.data -import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.Job -import kotlinx.coroutines.launch import space.kscience.dataforge.names.Name -import space.kscience.dataforge.names.plus /** @@ -12,7 +8,7 @@ import space.kscience.dataforge.names.plus */ context(DataSink) public infix fun String.put(data: Data): Unit = - data(Name.parse(this), data) + put(Name.parse(this), data) /** * Append node @@ -29,16 +25,3 @@ public infix fun String.put( block: DataSink.() -> Unit, ): Unit = branch(Name.parse(this), block) -/** - * Copy given data set and mirror its changes to this [LegacyDataTreeBuilder] in [this@setAndObserve]. Returns an update [Job] - */ -context(DataSink) -public fun CoroutineScope.setAndWatch( - name: Name, - dataSet: DataTree, -): Job = launch { - branch(name, dataSet) - dataSet.updates().collect { - data(name + it.name, it.data) - } -} \ No newline at end of file diff --git a/dataforge-data/src/jvmTest/kotlin/space/kscience/dataforge/data/ActionsTest.kt b/dataforge-data/src/jvmTest/kotlin/space/kscience/dataforge/data/ActionsTest.kt index 17549b6b..4aa6e6d4 100644 --- a/dataforge-data/src/jvmTest/kotlin/space/kscience/dataforge/data/ActionsTest.kt +++ b/dataforge-data/src/jvmTest/kotlin/space/kscience/dataforge/data/ActionsTest.kt @@ -1,7 +1,7 @@ package space.kscience.dataforge.data -import kotlinx.coroutines.ExperimentalCoroutinesApi import kotlinx.coroutines.delay +import kotlinx.coroutines.runBlocking import kotlinx.coroutines.test.runTest import org.junit.jupiter.api.Test import space.kscience.dataforge.actions.Action @@ -10,13 +10,13 @@ import space.kscience.dataforge.actions.mapping import space.kscience.dataforge.misc.DFExperimental import kotlin.test.assertEquals -@OptIn(DFExperimental::class, ExperimentalCoroutinesApi::class) +@OptIn(DFExperimental::class) internal class ActionsTest { @Test fun testStaticMapAction() = runTest { val data: DataTree = DataTree { repeat(10) { - static(it.toString(), it) + wrap(it.toString(), it) } } @@ -28,20 +28,24 @@ internal class ActionsTest { } @Test - fun testDynamicMapAction() = runTest { - val data: MutableDataTree = MutableDataTree(this) + fun testDynamicMapAction() = runBlocking { + val source: MutableDataTree = MutableDataTree() val plusOne = Action.mapping { result { it + 1 } } - val result = plusOne(data) + val result = plusOne(source) + repeat(10) { - data.static(it.toString(), it) + source.wrap(it.toString(), it) } - delay(20) + delay(10) + + source.close() + result.awaitClose() assertEquals(2, result["1"]?.await()) } diff --git a/dataforge-data/src/jvmTest/kotlin/space/kscience/dataforge/data/DataTreeBuilderTest.kt b/dataforge-data/src/jvmTest/kotlin/space/kscience/dataforge/data/DataTreeBuilderTest.kt index c5c450f7..95b7a7bd 100644 --- a/dataforge-data/src/jvmTest/kotlin/space/kscience/dataforge/data/DataTreeBuilderTest.kt +++ b/dataforge-data/src/jvmTest/kotlin/space/kscience/dataforge/data/DataTreeBuilderTest.kt @@ -1,6 +1,8 @@ package space.kscience.dataforge.data -import kotlinx.coroutines.* +import kotlinx.coroutines.delay +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.test.runTest import space.kscience.dataforge.misc.DFExperimental import space.kscience.dataforge.names.asName import kotlin.test.Test @@ -9,26 +11,25 @@ import kotlin.test.assertEquals internal class DataTreeBuilderTest { @Test - fun testTreeBuild() = runBlocking { + fun testTreeBuild() = runTest { val node = DataTree { "primary" put { - static("a", "a") - static("b", "b") + wrap("a", "a") + wrap("b", "b") } - static("c.d", "c.d") - static("c.f", "c.f") - } - runBlocking { - assertEquals("a", node["primary.a"]?.await()) - assertEquals("b", node["primary.b"]?.await()) - assertEquals("c.d", node["c.d"]?.await()) - assertEquals("c.f", node["c.f"]?.await()) + wrap("c.d", "c.d") + wrap("c.f", "c.f") } + assertEquals("a", node["primary.a"]?.await()) + assertEquals("b", node["primary.b"]?.await()) + assertEquals("c.d", node["c.d"]?.await()) + assertEquals("c.f", node["c.f"]?.await()) + } @OptIn(DFExperimental::class) @Test - fun testDataUpdate() = runBlocking { + fun testDataUpdate() = runTest { val updateData = DataTree { "update" put { "a" put Data.static("a") @@ -38,54 +39,30 @@ internal class DataTreeBuilderTest { val node = DataTree { "primary" put { - static("a", "a") - static("b", "b") + wrap("a", "a") + wrap("b", "b") } - static("root", "root") - populateFrom(updateData) + wrap("root", "root") + putAll(updateData) } - runBlocking { - assertEquals("a", node["update.a"]?.await()) - assertEquals("a", node["primary.a"]?.await()) - } + assertEquals("a", node["update.a"]?.await()) + assertEquals("a", node["primary.a"]?.await()) } @Test fun testDynamicUpdates() = runBlocking { - try { - lateinit var updateJob: Job - supervisorScope { - val subNode = ObservableDataTree(this) { - updateJob = launch { - repeat(10) { - delay(10) - static("value", it) - } - delay(10) - } - } - launch { - subNode.updates().collect { - println(it) - } - } - val rootNode = ObservableDataTree(this) { - setAndWatch("sub".asName(), subNode) - } + val subNode = MutableDataTree() - launch { - rootNode.updates().collect { - println(it) - } - } - updateJob.join() - assertEquals(9, rootNode["sub.value"]?.await()) - cancel() - } - } catch (t: Throwable) { - if (t !is CancellationException) throw t + val rootNode = MutableDataTree { + watchBranch("sub".asName(), subNode) } + repeat(10) { + subNode.wrap("value[$it]", it) + } + + delay(20) + assertEquals(9, rootNode["sub.value[9]"]?.await()) } } \ No newline at end of file diff --git a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/Task.kt b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/Task.kt index af2520eb..e636de49 100644 --- a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/Task.kt +++ b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/Task.kt @@ -88,12 +88,17 @@ public fun Task( workspace: Workspace, taskName: Name, taskMeta: Meta, - ): TaskResult = withContext(GoalExecutionRestriction() + workspace.goalLogger) { + ): TaskResult { //TODO use safe builder and check for external data on add and detects cycles - val dataset = MutableDataTree(resultType, this).apply { - TaskResultBuilder(workspace, taskName, taskMeta, this).apply { builder() } + val dataset = MutableDataTree(resultType, workspace.context).apply { + TaskResultBuilder(workspace, taskName, taskMeta, this).apply { + withContext(GoalExecutionRestriction() + workspace.goalLogger) { + builder() + } + } } - workspace.wrapResult(dataset, taskName, taskMeta) + return workspace.wrapResult(dataset, taskName, taskMeta) + } } diff --git a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/TaskResult.kt b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/TaskResult.kt index 94da4383..7aa94101 100644 --- a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/TaskResult.kt +++ b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/TaskResult.kt @@ -2,6 +2,7 @@ package space.kscience.dataforge.workspace import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Job +import kotlinx.coroutines.joinAll import kotlinx.coroutines.launch import space.kscience.dataforge.data.ObservableDataTree import space.kscience.dataforge.data.asSequence @@ -20,7 +21,7 @@ public data class TaskResult( public val workspace: Workspace, public val taskName: Name, public val taskMeta: Meta, -): ObservableDataTree by content +) : ObservableDataTree by content /** * Wrap data into [TaskResult] @@ -32,8 +33,9 @@ public fun Workspace.wrapResult(data: ObservableDataTree, taskName: Name, * Start computation for all data elements of this node. * The resulting [Job] is completed only when all of them are completed. */ -public fun TaskResult<*>.compute(scope: CoroutineScope): Job = scope.launch { - asSequence().forEach { +public fun TaskResult<*>.launch(scope: CoroutineScope): Job { + val jobs = asSequence().map { it.data.launch(scope) - } + }.toList() + return scope.launch { jobs.joinAll() } } \ No newline at end of file diff --git a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/WorkspaceBuilder.kt b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/WorkspaceBuilder.kt index 6ccdcb1c..4705c3b0 100644 --- a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/WorkspaceBuilder.kt +++ b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/WorkspaceBuilder.kt @@ -102,7 +102,7 @@ public inline fun TaskContainer.action( noinline descriptorBuilder: MetaDescriptorBuilder.() -> Unit = {}, ): PropertyDelegateProvider>> = task(MetaDescriptor(descriptorBuilder)) { - result(action.execute(workspace.context, from(selector), taskMeta.copy(metaTransform))) + result(action.execute(from(selector), taskMeta.copy(metaTransform))) } public class WorkspaceBuilder( diff --git a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/taskBuilders.kt b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/taskBuilders.kt index 95e7d481..1900ff23 100644 --- a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/taskBuilders.kt +++ b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/taskBuilders.kt @@ -93,7 +93,7 @@ public suspend inline fun TaskResultBuilder.transformEach( action(it, data.name, meta) } - data(data.name, res) + put(data.name, res) } } @@ -113,7 +113,7 @@ public suspend inline fun TaskResultBuilder.actionFrom( action: Action, dependencyMeta: Meta = defaultDependencyMeta, ) { - branch(action.execute(workspace.context, from(selector, dependencyMeta), dependencyMeta)) + branch(action.execute(from(selector, dependencyMeta), dependencyMeta)) } diff --git a/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/fileData.kt b/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/fileData.kt index 4eba363b..51c9a5e8 100644 --- a/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/fileData.kt +++ b/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/fileData.kt @@ -11,7 +11,6 @@ import space.kscience.dataforge.names.Name import space.kscience.dataforge.names.NameToken import space.kscience.dataforge.names.asName import space.kscience.dataforge.names.plus -import space.kscience.dataforge.workspace.FileData.defaultPathToName import java.nio.file.Files import java.nio.file.Path import java.nio.file.StandardWatchEventKinds @@ -36,20 +35,6 @@ public object FileData { public const val DF_FILE_EXTENSION: String = "df" public val DEFAULT_IGNORE_EXTENSIONS: Set = setOf(DF_FILE_EXTENSION) - /** - * Transform file name into DataForg name. Ignores DataForge file extensions. - */ - public val defaultPathToName: (Path) -> Name = { path -> - Name( - path.map { segment -> - if (segment.isRegularFile() && segment.extension in DEFAULT_IGNORE_EXTENSIONS) { - NameToken(path.nameWithoutExtension) - } else { - NameToken(path.name) - } - } - ) - } } @@ -77,51 +62,68 @@ public fun IOPlugin.readFileData( ) } -public fun DataSink.file(io: IOPlugin, path: Path, name: Name) { +public fun DataSink.file(io: IOPlugin, name: Name, path: Path) { if (!path.isRegularFile()) error("Only regular files could be handled by this function") - data(name, io.readFileData(path)) + put(name, io.readFileData(path)) } public fun DataSink.directory( io: IOPlugin, + name: Name, path: Path, - pathToName: (Path) -> Name = defaultPathToName, ) { if (!path.isDirectory()) error("Only directories could be handled by this function") - val metaFile = path.resolve(IOPlugin.META_FILE_NAME) - val dataFile = path.resolve(IOPlugin.DATA_FILE_NAME) //process root data - if (metaFile.exists() || dataFile.exists()) { - data( - Name.EMPTY, + + var dataBinary: Binary? = null + var meta: Meta? = null + Files.list(path).forEach { childPath -> + val fileName = childPath.fileName.toString() + if (fileName == IOPlugin.DATA_FILE_NAME) { + dataBinary = childPath.asBinary() + } else if (fileName.startsWith(IOPlugin.META_FILE_NAME)) { + meta = io.readMetaFileOrNull(childPath) + } else if (!fileName.startsWith("@")) { + val token = if (childPath.isRegularFile() && childPath.extension in FileData.DEFAULT_IGNORE_EXTENSIONS) { + NameToken(childPath.nameWithoutExtension) + } else { + NameToken(childPath.name) + } + + files(io, name + token, childPath) + } + } + + //set data if it is relevant + if (dataBinary != null || meta != null) { + put( + name, StaticData( typeOf(), - dataFile.takeIf { it.exists() }?.asBinary() ?: Binary.EMPTY, - io.readMetaFileOrNull(metaFile) ?: Meta.EMPTY + dataBinary ?: Binary.EMPTY, + meta ?: Meta.EMPTY ) ) } - Files.list(path).forEach { childPath -> - val fileName = childPath.fileName.toString() - if (!fileName.startsWith("@")) { - files(io, childPath, pathToName) - } - } } -public fun DataSink.files(io: IOPlugin, path: Path, pathToName: (Path) -> Name = defaultPathToName) { +public fun DataSink.files( + io: IOPlugin, + name: Name, + path: Path, +) { if (path.isRegularFile() && path.extension == "zip") { //Using explicit Zip file system to avoid bizarre compatibility bugs val fsProvider = FileSystemProvider.installedProviders().find { it.scheme == "jar" } ?: error("Zip file system provider not found") val fs = fsProvider.newFileSystem(path, mapOf("create" to "true")) - return files(io, fs.rootDirectories.first(), pathToName) + files(io, name, fs.rootDirectories.first()) } if (path.isRegularFile()) { - file(io, path, pathToName(path)) + file(io, name, path) } else { - directory(io, path, pathToName) + directory(io, name, path) } } @@ -132,11 +134,11 @@ private fun Path.toName() = Name(map { NameToken.parse(it.nameWithoutExtension) @DFExperimental public fun DataSink.monitorFiles( io: IOPlugin, + name: Name, path: Path, - pathToName: (Path) -> Name = defaultPathToName, scope: CoroutineScope = io.context, ): Job { - files(io, path, pathToName) + files(io, name, path) return scope.launch(Dispatchers.IO) { val watchService = path.fileSystem.newWatchService() @@ -153,11 +155,11 @@ public fun DataSink.monitorFiles( for (event: WatchEvent<*> in key.pollEvents()) { val eventPath = event.context() as Path if (event.kind() == StandardWatchEventKinds.ENTRY_DELETE) { - data(eventPath.toName(), null) + put(eventPath.toName(), null) } else { val fileName = eventPath.fileName.toString() if (!fileName.startsWith("@")) { - files(io, eventPath, pathToName) + files(io, name, eventPath) } } } @@ -179,29 +181,24 @@ public suspend fun IOPlugin.writeDataDirectory( dataSet: DataTree, format: IOWriter, envelopeFormat: EnvelopeFormat? = null, - nameToPath: (name: Name, data: Data) -> Path = { name, _ -> - Path(name.tokens.joinToString("/") { token -> token.toStringUnescaped() }) - }, -) { - withContext(Dispatchers.IO) { - if (!Files.exists(path)) { - Files.createDirectories(path) - } else if (!Files.isDirectory(path)) { - error("Can't write a node into file") - } - dataSet.forEach { (name, data) -> - val childPath = path.resolve(nameToPath(name, data)) - childPath.parent.createDirectories() - val envelope = data.toEnvelope(format) - if (envelopeFormat != null) { - writeEnvelopeFile(childPath, envelope, envelopeFormat) - } else { - writeEnvelopeDirectory(childPath, envelope) - } - } - dataSet.meta?.let { writeMetaFile(path, it) } - +): Unit = withContext(Dispatchers.IO) { + if (!Files.exists(path)) { + Files.createDirectories(path) + } else if (!Files.isDirectory(path)) { + error("Can't write a node into file") } + dataSet.forEach { (name, data) -> + val childPath = path.resolve(name.tokens.joinToString("/") { token -> token.toStringUnescaped() }) + childPath.parent.createDirectories() + val envelope = data.toEnvelope(format) + if (envelopeFormat != null) { + writeEnvelopeFile(childPath, envelope, envelopeFormat) + } else { + writeEnvelopeDirectory(childPath, envelope) + } + } + dataSet.meta?.let { writeMetaFile(path, it) } + } /** @@ -212,15 +209,12 @@ public suspend fun IOPlugin.writeDataDirectory( public fun DataSink.resources( io: IOPlugin, vararg resources: String, - pathToName: (Path) -> Name = defaultPathToName, classLoader: ClassLoader = Thread.currentThread().contextClassLoader, ) { resources.forEach { resource -> val path = classLoader.getResource(resource)?.toURI()?.toPath() ?: error( "Resource with name $resource is not resolved" ) - branch(resource.asName()) { - files(io, path, pathToName) - } + files(io, resource.asName(), path) } } diff --git a/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/zipData.kt b/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/zipData.kt index 83b617cb..9d175bd3 100644 --- a/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/zipData.kt +++ b/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/zipData.kt @@ -3,67 +3,36 @@ package space.kscience.dataforge.workspace import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.withContext import space.kscience.dataforge.data.DataTree -import space.kscience.dataforge.io.* +import space.kscience.dataforge.io.EnvelopeFormat +import space.kscience.dataforge.io.IOPlugin +import space.kscience.dataforge.io.IOWriter import space.kscience.dataforge.misc.DFExperimental -import java.nio.file.Files import java.nio.file.Path -import java.nio.file.StandardOpenOption -import java.util.zip.ZipEntry -import java.util.zip.ZipOutputStream - - -private suspend fun ZipOutputStream.writeNode( - name: String, - tree: DataTree, - dataFormat: IOFormat, - envelopeFormat: EnvelopeFormat = TaggedEnvelopeFormat, -): Unit = withContext(Dispatchers.IO) { - //TODO add directory-based envelope writer - tree.data?.let { - val envelope = it.toEnvelope(dataFormat) - val entry = ZipEntry(name) - putNextEntry(entry) - - //TODO remove additional copy - val bytes = ByteArray { - writeWith(envelopeFormat, envelope) - } - write(bytes) - } - - - val entry = ZipEntry("$name/") - putNextEntry(entry) - closeEntry() - tree.items.forEach { (token, item) -> - val childName = "$name/$token" - writeNode(childName, item, dataFormat, envelopeFormat) - } - -} +import java.nio.file.spi.FileSystemProvider +import kotlin.io.path.exists +import kotlin.io.path.extension /** * Write this [DataTree] as a zip archive */ @DFExperimental -public suspend fun DataTree.writeZip( +public suspend fun IOPlugin.writeZip( path: Path, - format: IOFormat, - envelopeFormat: EnvelopeFormat = TaggedEnvelopeFormat, + dataSet: DataTree, + format: IOWriter, + envelopeFormat: EnvelopeFormat? = null, ): Unit = withContext(Dispatchers.IO) { - val actualFile = if (path.toString().endsWith(".zip")) { + if (path.exists()) error("Can't override existing zip data file $path") + val actualFile = if (path.extension == "zip") { path } else { path.resolveSibling(path.fileName.toString() + ".zip") } - val fos = Files.newOutputStream( - actualFile, - StandardOpenOption.WRITE, - StandardOpenOption.CREATE, - StandardOpenOption.TRUNCATE_EXISTING - ) - val zos = ZipOutputStream(fos) - zos.use { - it.writeNode("", this@writeZip, format, envelopeFormat) + val fsProvider = FileSystemProvider.installedProviders().find { it.scheme == "jar" } + ?: error("Zip file system provider not found") + //val fs = FileSystems.newFileSystem(actualFile, mapOf("create" to true)) + val fs = fsProvider.newFileSystem(actualFile, mapOf("create" to true)) + fs.use { + writeDataDirectory(fs.rootDirectories.first(), dataSet, format, envelopeFormat) } -} \ No newline at end of file +} diff --git a/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/CachingWorkspaceTest.kt b/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/CachingWorkspaceTest.kt index 78ce853e..e5c2c230 100644 --- a/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/CachingWorkspaceTest.kt +++ b/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/CachingWorkspaceTest.kt @@ -1,17 +1,16 @@ package space.kscience.dataforge.workspace -import kotlinx.coroutines.ExperimentalCoroutinesApi import kotlinx.coroutines.coroutineScope import kotlinx.coroutines.test.runTest import org.junit.jupiter.api.Test -import space.kscience.dataforge.data.static +import space.kscience.dataforge.data.wrap import space.kscience.dataforge.meta.Meta import space.kscience.dataforge.meta.boolean import space.kscience.dataforge.meta.get import space.kscience.dataforge.misc.DFExperimental import kotlin.test.assertEquals -@OptIn(ExperimentalCoroutinesApi::class, DFExperimental::class) +@OptIn(DFExperimental::class) internal class CachingWorkspaceTest { @Test @@ -23,7 +22,7 @@ internal class CachingWorkspaceTest { data { //statically initialize data repeat(5) { - static("myData[$it]", it) + wrap("myData[$it]", it) } } @@ -39,7 +38,7 @@ internal class CachingWorkspaceTest { val doSecond by task { transformEach( doFirst, - dependencyMeta = if(taskMeta["flag"].boolean == true) taskMeta else Meta.EMPTY + dependencyMeta = if (taskMeta["flag"].boolean == true) taskMeta else Meta.EMPTY ) { _, name, _ -> secondCounter++ println("Done second on $name with flag=${taskMeta["flag"].boolean ?: false}") @@ -51,13 +50,15 @@ internal class CachingWorkspaceTest { val secondA = workspace.produce("doSecond") val secondB = workspace.produce("doSecond", Meta { "flag" put true }) val secondC = workspace.produce("doSecond") + //use coroutineScope to wait for the result coroutineScope { - first.compute(this) - secondA.compute(this) - secondB.compute(this) + first.launch(this) + secondA.launch(this) + secondB.launch(this) //repeat to check caching - secondC.compute(this) + secondC.launch(this) } + assertEquals(10, firstCounter) assertEquals(10, secondCounter) } diff --git a/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/DataPropagationTest.kt b/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/DataPropagationTest.kt index 403b1c51..d611b1c8 100644 --- a/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/DataPropagationTest.kt +++ b/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/DataPropagationTest.kt @@ -20,13 +20,13 @@ class DataPropagationTestPlugin : WorkspacePlugin() { val result: Data = selectedData.foldToData(0) { result, data -> result + data.value } - data("result", result) + put("result", result) } val singleData by task { workspace.data.filterByType()["myData[12]"]?.let { - data("result", it) + put("result", it) } } @@ -47,7 +47,7 @@ class DataPropagationTest { } data { repeat(100) { - static("myData[$it]", it) + wrap("myData[$it]", it) } } } diff --git a/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/FileDataTest.kt b/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/FileDataTest.kt index 1a21fb17..10a1c268 100644 --- a/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/FileDataTest.kt +++ b/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/FileDataTest.kt @@ -1,6 +1,5 @@ package space.kscience.dataforge.workspace -import kotlinx.coroutines.runBlocking import kotlinx.coroutines.test.runTest import kotlinx.io.Sink import kotlinx.io.Source @@ -13,7 +12,9 @@ import space.kscience.dataforge.io.* import space.kscience.dataforge.io.yaml.YamlPlugin import space.kscience.dataforge.meta.get import space.kscience.dataforge.misc.DFExperimental +import space.kscience.dataforge.names.Name import java.nio.file.Files +import kotlin.io.path.deleteExisting import kotlin.io.path.fileSize import kotlin.io.path.toPath import kotlin.test.Test @@ -23,11 +24,11 @@ import kotlin.test.assertEquals class FileDataTest { val dataNode = DataTree { branch("dir") { - static("a", "Some string") { + wrap("a", "Some string") { "content" put "Some string" } } - static("b", "root data") + wrap("b", "root data") // meta { // "content" put "This is root meta node" // } @@ -45,17 +46,17 @@ class FileDataTest { @Test @DFExperimental - fun testDataWriteRead() = with(Global.io) { + fun testDataWriteRead() = runTest { val io = Global.io val dir = Files.createTempDirectory("df_data_node") - runBlocking { - writeDataDirectory(dir, dataNode, StringIOFormat) - println(dir.toUri().toString()) - val reconstructed = DataTree { files(io, dir) } - .transform { (_, value) -> value.toByteArray().decodeToString() } - assertEquals(dataNode["dir.a"]?.meta?.get("content"), reconstructed["dir.a"]?.meta?.get("content")) - assertEquals(dataNode["b"]?.await(), reconstructed["b"]?.await()) + io.writeDataDirectory(dir, dataNode, StringIOFormat) + println(dir.toUri().toString()) + val data = DataTree { + files(io, Name.EMPTY, dir) } + val reconstructed = data.transform { (_, value) -> value.toByteArray().decodeToString() } + assertEquals(dataNode["dir.a"]?.meta?.get("content"), reconstructed["dir.a"]?.meta?.get("content")) + assertEquals(dataNode["b"]?.await(), reconstructed["b"]?.await()) } @@ -64,9 +65,10 @@ class FileDataTest { fun testZipWriteRead() = runTest { val io = Global.io val zip = Files.createTempFile("df_data_node", ".zip") - dataNode.writeZip(zip, StringIOFormat) + zip.deleteExisting() + io.writeZip(zip, dataNode, StringIOFormat) println(zip.toUri().toString()) - val reconstructed = DataTree { files(io, zip) } + val reconstructed = DataTree { files(io, Name.EMPTY, zip) } .transform { (_, value) -> value.toByteArray().decodeToString() } assertEquals(dataNode["dir.a"]?.meta?.get("content"), reconstructed["dir.a"]?.meta?.get("content")) assertEquals(dataNode["b"]?.await(), reconstructed["b"]?.await()) diff --git a/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/FileWorkspaceCacheTest.kt b/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/FileWorkspaceCacheTest.kt index ee497e1b..0f16b1c8 100644 --- a/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/FileWorkspaceCacheTest.kt +++ b/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/FileWorkspaceCacheTest.kt @@ -3,7 +3,7 @@ package space.kscience.dataforge.workspace import kotlinx.coroutines.ExperimentalCoroutinesApi import kotlinx.coroutines.test.runTest import org.junit.jupiter.api.Test -import space.kscience.dataforge.data.static +import space.kscience.dataforge.data.wrap import space.kscience.dataforge.misc.DFExperimental import java.nio.file.Files @@ -16,7 +16,7 @@ class FileWorkspaceCacheTest { data { //statically initialize data repeat(5) { - static("myData[$it]", it) + wrap("myData[$it]", it) } } fileCache(Files.createTempDirectory("dataforge-temporary-cache")) @@ -26,7 +26,7 @@ class FileWorkspaceCacheTest { } } - workspace.produce("echo").compute(this) + workspace.produce("echo").launch(this) } } \ No newline at end of file diff --git a/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/SimpleWorkspaceTest.kt b/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/SimpleWorkspaceTest.kt index 91fa2f6c..b49b9d54 100644 --- a/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/SimpleWorkspaceTest.kt +++ b/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/SimpleWorkspaceTest.kt @@ -62,14 +62,14 @@ internal class SimpleWorkspaceTest { data { //statically initialize data repeat(100) { - static("myData[$it]", it) + wrap("myData[$it]", it) } } val filterOne by task { val name by taskMeta.string { error("Name field not defined") } from(testPluginFactory) { test }[name]?.let { source: Data -> - data(name, source) + put(name, source) } } @@ -97,7 +97,7 @@ internal class SimpleWorkspaceTest { val newData: Data = data.combine(linearData[data.name]!!) { l, r -> l + r } - data(data.name, newData) + put(data.name, newData) } } @@ -106,7 +106,7 @@ internal class SimpleWorkspaceTest { val res = from(square).foldToData(0) { l, r -> l + r.value } - data("sum", res) + put("sum", res) } val averageByGroup by task { @@ -116,13 +116,13 @@ internal class SimpleWorkspaceTest { l + r.value } - data("even", evenSum) + put("even", evenSum) val oddSum = workspace.data.filterByType { name, _, _ -> name.toString().toInt() % 2 == 1 }.foldToData(0) { l, r -> l + r.value } - data("odd", oddSum) + put("odd", oddSum) } val delta by task { @@ -132,7 +132,7 @@ internal class SimpleWorkspaceTest { val res = even.combine(odd) { l, r -> l - r } - data("res", res) + put("res", res) } val customPipe by task { @@ -140,7 +140,7 @@ internal class SimpleWorkspaceTest { val meta = data.meta.toMutableMeta().apply { "newValue" put 22 } - data(data.name + "new", data.transform { (data.meta["value"].int ?: 0) + it }) + put(data.name + "new", data.transform { (data.meta["value"].int ?: 0) + it }) } } -- 2.34.1 From 6e209ab5cd5e569f46f4264d9702d532ac6c1568 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Sat, 3 Feb 2024 17:34:19 +0300 Subject: [PATCH 19/62] Data tree refactored to a uniform tree instead of sealed class. --- dataforge-io/build.gradle.kts | 2 +- .../{fileData.kt => readFileData.kt} | 42 ++--------- .../dataforge/workspace/writeFileData.kt | 72 +++++++++++++++++++ .../kscience/dataforge/workspace/zipData.kt | 38 ---------- 4 files changed, 77 insertions(+), 77 deletions(-) rename dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/{fileData.kt => readFileData.kt} (81%) create mode 100644 dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/writeFileData.kt delete mode 100644 dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/zipData.kt diff --git a/dataforge-io/build.gradle.kts b/dataforge-io/build.gradle.kts index f7197197..5be52e61 100644 --- a/dataforge-io/build.gradle.kts +++ b/dataforge-io/build.gradle.kts @@ -4,7 +4,7 @@ plugins { description = "IO module" -val ioVersion = "0.3.0" +val ioVersion = "0.3.1" kscience { jvm() diff --git a/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/fileData.kt b/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/readFileData.kt similarity index 81% rename from dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/fileData.kt rename to dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/readFileData.kt index 51c9a5e8..1815c3e4 100644 --- a/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/fileData.kt +++ b/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/readFileData.kt @@ -1,7 +1,9 @@ package space.kscience.dataforge.workspace import kotlinx.coroutines.* -import space.kscience.dataforge.data.* +import space.kscience.dataforge.data.Data +import space.kscience.dataforge.data.DataSink +import space.kscience.dataforge.data.StaticData import space.kscience.dataforge.io.* import space.kscience.dataforge.meta.Meta import space.kscience.dataforge.meta.copy @@ -21,11 +23,6 @@ import kotlin.io.path.* import kotlin.reflect.typeOf -//public typealias FileFormatResolver = (Path, Meta) -> IOFormat - -public typealias FileFormatResolver = (path: Path, meta: Meta) -> IOReader? - - public object FileData { public val FILE_KEY: Name = "file".asName() public val FILE_PATH_KEY: Name = FILE_KEY + "path" @@ -116,7 +113,7 @@ public fun DataSink.files( //Using explicit Zip file system to avoid bizarre compatibility bugs val fsProvider = FileSystemProvider.installedProviders().find { it.scheme == "jar" } ?: error("Zip file system provider not found") - val fs = fsProvider.newFileSystem(path, mapOf("create" to "true")) + val fs = fsProvider.newFileSystem(path, emptyMap()) files(io, name, fs.rootDirectories.first()) } @@ -170,37 +167,6 @@ public fun DataSink.monitorFiles( } -/** - * Write the data tree to existing directory or create a new one using default [java.nio.file.FileSystem] provider - * - * @param nameToPath a [Name] to [Path] converter used to create - */ -@DFExperimental -public suspend fun IOPlugin.writeDataDirectory( - path: Path, - dataSet: DataTree, - format: IOWriter, - envelopeFormat: EnvelopeFormat? = null, -): Unit = withContext(Dispatchers.IO) { - if (!Files.exists(path)) { - Files.createDirectories(path) - } else if (!Files.isDirectory(path)) { - error("Can't write a node into file") - } - dataSet.forEach { (name, data) -> - val childPath = path.resolve(name.tokens.joinToString("/") { token -> token.toStringUnescaped() }) - childPath.parent.createDirectories() - val envelope = data.toEnvelope(format) - if (envelopeFormat != null) { - writeEnvelopeFile(childPath, envelope, envelopeFormat) - } else { - writeEnvelopeDirectory(childPath, envelope) - } - } - dataSet.meta?.let { writeMetaFile(path, it) } - -} - /** * @param resources The names of the resources to read. * @param classLoader The class loader to use for loading the resources. By default, it uses the current thread's context class loader. diff --git a/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/writeFileData.kt b/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/writeFileData.kt new file mode 100644 index 00000000..379a79dd --- /dev/null +++ b/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/writeFileData.kt @@ -0,0 +1,72 @@ +package space.kscience.dataforge.workspace + +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext +import space.kscience.dataforge.data.* +import space.kscience.dataforge.io.* +import space.kscience.dataforge.misc.DFExperimental +import space.kscience.dataforge.names.Name +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.spi.FileSystemProvider +import kotlin.io.path.Path +import kotlin.io.path.createDirectories +import kotlin.io.path.exists +import kotlin.io.path.extension + + +/** + * Write the data tree to existing directory or create a new one using default [java.nio.file.FileSystem] provider + * + * @param nameToPath a [Name] to [Path] converter used to create + */ +@DFExperimental +public suspend fun IOPlugin.writeDataDirectory( + path: Path, + dataSet: DataTree, + format: IOWriter, + envelopeFormat: EnvelopeFormat? = null, +): Unit = withContext(Dispatchers.IO) { + if (!Files.exists(path)) { + Files.createDirectories(path) + } else if (!Files.isDirectory(path)) { + error("Can't write a node into file") + } + dataSet.forEach { (name, data) -> + val childPath = path.resolve(name.tokens.joinToString("/") { token -> token.toStringUnescaped() }) + childPath.parent.createDirectories() + val envelope = data.toEnvelope(format) + if (envelopeFormat != null) { + writeEnvelopeFile(childPath, envelope, envelopeFormat) + } else { + writeEnvelopeDirectory(childPath, envelope) + } + } + dataSet.meta?.let { writeMetaFile(path, it) } + +} + +/** + * Write this [DataTree] as a zip archive + */ +@DFExperimental +public suspend fun IOPlugin.writeZip( + path: Path, + dataSet: DataTree, + format: IOWriter, + envelopeFormat: EnvelopeFormat? = null, +): Unit = withContext(Dispatchers.IO) { + if (path.exists()) error("Can't override existing zip data file $path") + val actualFile = if (path.extension == "zip") { + path + } else { + path.resolveSibling(path.fileName.toString() + ".zip") + } + val fsProvider = FileSystemProvider.installedProviders().find { it.scheme == "jar" } + ?: error("Zip file system provider not found") + //val fs = FileSystems.newFileSystem(actualFile, mapOf("create" to true)) + val fs = fsProvider.newFileSystem(actualFile, mapOf("create" to true)) + fs.use { + writeDataDirectory(fs.rootDirectories.first(), dataSet, format, envelopeFormat) + } +} \ No newline at end of file diff --git a/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/zipData.kt b/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/zipData.kt deleted file mode 100644 index 9d175bd3..00000000 --- a/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/zipData.kt +++ /dev/null @@ -1,38 +0,0 @@ -package space.kscience.dataforge.workspace - -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.withContext -import space.kscience.dataforge.data.DataTree -import space.kscience.dataforge.io.EnvelopeFormat -import space.kscience.dataforge.io.IOPlugin -import space.kscience.dataforge.io.IOWriter -import space.kscience.dataforge.misc.DFExperimental -import java.nio.file.Path -import java.nio.file.spi.FileSystemProvider -import kotlin.io.path.exists -import kotlin.io.path.extension - -/** - * Write this [DataTree] as a zip archive - */ -@DFExperimental -public suspend fun IOPlugin.writeZip( - path: Path, - dataSet: DataTree, - format: IOWriter, - envelopeFormat: EnvelopeFormat? = null, -): Unit = withContext(Dispatchers.IO) { - if (path.exists()) error("Can't override existing zip data file $path") - val actualFile = if (path.extension == "zip") { - path - } else { - path.resolveSibling(path.fileName.toString() + ".zip") - } - val fsProvider = FileSystemProvider.installedProviders().find { it.scheme == "jar" } - ?: error("Zip file system provider not found") - //val fs = FileSystems.newFileSystem(actualFile, mapOf("create" to true)) - val fs = fsProvider.newFileSystem(actualFile, mapOf("create" to true)) - fs.use { - writeDataDirectory(fs.rootDirectories.first(), dataSet, format, envelopeFormat) - } -} -- 2.34.1 From 6524c6c69182b51eddaf7ae73e55b997e7de8b34 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Sat, 3 Feb 2024 17:36:49 +0300 Subject: [PATCH 20/62] Add .kotlin to gitignore --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 17a319a4..53b55cd4 100644 --- a/.gitignore +++ b/.gitignore @@ -5,5 +5,7 @@ out/ .gradle build/ +.kotlin + !gradle-wrapper.jar \ No newline at end of file -- 2.34.1 From 297847663bbbeab041750d9fd2c7ec3aa31e2a78 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Sat, 3 Feb 2024 19:16:18 +0300 Subject: [PATCH 21/62] Update documentation --- CHANGELOG.md | 23 +- build.gradle.kts | 2 +- dataforge-context/README.md | 6 +- dataforge-context/api/dataforge-context.api | 21 +- dataforge-data/README.md | 6 +- dataforge-data/build.gradle.kts | 1 + dataforge-io/README.md | 6 +- dataforge-io/dataforge-io-yaml/README.md | 6 +- dataforge-meta/README.md | 6 +- dataforge-meta/api/dataforge-meta.api | 315 +++++++++--------- .../meta/descriptors/MetaDescriptor.kt | 1 + dataforge-scripting/README.md | 6 +- dataforge-workspace/README.md | 6 +- 13 files changed, 211 insertions(+), 194 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4469f802..b9d3b6a9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,27 +3,42 @@ ## Unreleased ### Added + +### Changed + +### Deprecated + +### Removed + +### Fixed + +### Security + +## 0.8.0 - 2024-02-03 + +### Added + - Wasm artifacts - Add automatic MetaConverter for serializeable objects - Add Meta and MutableMeta delegates for convertable and serializeable - Meta mapping for data. ### Changed + - Descriptor `children` renamed to `nodes` - `MetaConverter` now inherits `MetaSpec` (former `Specifiction`). So `MetaConverter` could be used more universally. - Meta copy and modification now use lightweight non-observable meta builders. +- Full refactor of Data API. DataTree now works similar to Meta: contains optional anonymous root element and data items. Updates are available for `ObservaleDataSource` and `ObservableDataTree` variants. ### Deprecated + - `node(key,converter)` in favor of `serializable` delegate -### Removed - ### Fixed + - Partially fixed a bug with `MutableMeta` observable wrappers. - `valueSequence` now include root value. So `meta.update` works properly. -### Security - ## 0.7.0 - 2023-11-26 ### Added diff --git a/build.gradle.kts b/build.gradle.kts index 4a336e25..b9349868 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -8,7 +8,7 @@ plugins { allprojects { group = "space.kscience" - version = "0.8.0-dev-1" + version = "0.8.0" } subprojects { diff --git a/dataforge-context/README.md b/dataforge-context/README.md index 905171b1..f0aff459 100644 --- a/dataforge-context/README.md +++ b/dataforge-context/README.md @@ -6,18 +6,16 @@ Context and provider definitions ## Artifact: -The Maven coordinates of this project are `space.kscience:dataforge-context:0.7.0`. +The Maven coordinates of this project are `space.kscience:dataforge-context:0.8.0`. **Gradle Kotlin DSL:** ```kotlin repositories { maven("https://repo.kotlin.link") - //uncomment to access development builds - //maven("https://maven.pkg.jetbrains.space/spc/p/sci/dev") mavenCentral() } dependencies { - implementation("space.kscience:dataforge-context:0.7.0") + implementation("space.kscience:dataforge-context:0.8.0") } ``` diff --git a/dataforge-context/api/dataforge-context.api b/dataforge-context/api/dataforge-context.api index eac8e52c..12bca176 100644 --- a/dataforge-context/api/dataforge-context.api +++ b/dataforge-context/api/dataforge-context.api @@ -249,10 +249,27 @@ public final class space/kscience/dataforge/context/SlfLogManager$Companion : sp public fun getTag ()Lspace/kscience/dataforge/context/PluginTag; } -public final class space/kscience/dataforge/properties/PropertyKt { +public abstract interface annotation class space/kscience/dataforge/descriptors/Description : java/lang/annotation/Annotation { + public abstract fun value ()Ljava/lang/String; } -public final class space/kscience/dataforge/properties/SchemePropertyKt { +public abstract interface annotation class space/kscience/dataforge/descriptors/DescriptorResource : java/lang/annotation/Annotation { + public abstract fun resourceName ()Ljava/lang/String; +} + +public abstract interface annotation class space/kscience/dataforge/descriptors/DescriptorUrl : java/lang/annotation/Annotation { + public abstract fun url ()Ljava/lang/String; +} + +public abstract interface annotation class space/kscience/dataforge/descriptors/Multiple : java/lang/annotation/Annotation { +} + +public final class space/kscience/dataforge/descriptors/ReflectiveDescriptorsKt { + public static final fun forClass (Lspace/kscience/dataforge/meta/descriptors/MetaDescriptor$Companion;Lkotlin/reflect/KClass;Lkotlin/jvm/functions/Function1;)Lspace/kscience/dataforge/meta/descriptors/MetaDescriptor; + public static synthetic fun forClass$default (Lspace/kscience/dataforge/meta/descriptors/MetaDescriptor$Companion;Lkotlin/reflect/KClass;Lkotlin/jvm/functions/Function1;ILjava/lang/Object;)Lspace/kscience/dataforge/meta/descriptors/MetaDescriptor; +} + +public final class space/kscience/dataforge/properties/MetaAsFlowKt { } public final class space/kscience/dataforge/provider/DfTypeKt { diff --git a/dataforge-data/README.md b/dataforge-data/README.md index 5935af6e..d77ed1b9 100644 --- a/dataforge-data/README.md +++ b/dataforge-data/README.md @@ -6,18 +6,16 @@ ## Artifact: -The Maven coordinates of this project are `space.kscience:dataforge-data:0.7.0`. +The Maven coordinates of this project are `space.kscience:dataforge-data:0.8.0`. **Gradle Kotlin DSL:** ```kotlin repositories { maven("https://repo.kotlin.link") - //uncomment to access development builds - //maven("https://maven.pkg.jetbrains.space/spc/p/sci/dev") mavenCentral() } dependencies { - implementation("space.kscience:dataforge-data:0.7.0") + implementation("space.kscience:dataforge-data:0.8.0") } ``` diff --git a/dataforge-data/build.gradle.kts b/dataforge-data/build.gradle.kts index ea542290..99314ea3 100644 --- a/dataforge-data/build.gradle.kts +++ b/dataforge-data/build.gradle.kts @@ -11,6 +11,7 @@ kscience{ dependencies { api(spclibs.atomicfu) api(projects.dataforgeMeta) + //Remove after subtype moved to stdlib api(kotlin("reflect")) } } diff --git a/dataforge-io/README.md b/dataforge-io/README.md index ec431a04..85e49e5a 100644 --- a/dataforge-io/README.md +++ b/dataforge-io/README.md @@ -6,18 +6,16 @@ IO module ## Artifact: -The Maven coordinates of this project are `space.kscience:dataforge-io:0.7.0`. +The Maven coordinates of this project are `space.kscience:dataforge-io:0.8.0`. **Gradle Kotlin DSL:** ```kotlin repositories { maven("https://repo.kotlin.link") - //uncomment to access development builds - //maven("https://maven.pkg.jetbrains.space/spc/p/sci/dev") mavenCentral() } dependencies { - implementation("space.kscience:dataforge-io:0.7.0") + implementation("space.kscience:dataforge-io:0.8.0") } ``` diff --git a/dataforge-io/dataforge-io-yaml/README.md b/dataforge-io/dataforge-io-yaml/README.md index 06a0efb4..20f5b4f6 100644 --- a/dataforge-io/dataforge-io-yaml/README.md +++ b/dataforge-io/dataforge-io-yaml/README.md @@ -6,18 +6,16 @@ YAML meta IO ## Artifact: -The Maven coordinates of this project are `space.kscience:dataforge-io-yaml:0.7.0`. +The Maven coordinates of this project are `space.kscience:dataforge-io-yaml:0.8.0`. **Gradle Kotlin DSL:** ```kotlin repositories { maven("https://repo.kotlin.link") - //uncomment to access development builds - //maven("https://maven.pkg.jetbrains.space/spc/p/sci/dev") mavenCentral() } dependencies { - implementation("space.kscience:dataforge-io-yaml:0.7.0") + implementation("space.kscience:dataforge-io-yaml:0.8.0") } ``` diff --git a/dataforge-meta/README.md b/dataforge-meta/README.md index 5f214640..bd11ebf1 100644 --- a/dataforge-meta/README.md +++ b/dataforge-meta/README.md @@ -6,18 +6,16 @@ Meta definition and basic operations on meta ## Artifact: -The Maven coordinates of this project are `space.kscience:dataforge-meta:0.7.0`. +The Maven coordinates of this project are `space.kscience:dataforge-meta:0.8.0`. **Gradle Kotlin DSL:** ```kotlin repositories { maven("https://repo.kotlin.link") - //uncomment to access development builds - //maven("https://maven.pkg.jetbrains.space/spc/p/sci/dev") mavenCentral() } dependencies { - implementation("space.kscience:dataforge-meta:0.7.0") + implementation("space.kscience:dataforge-meta:0.8.0") } ``` diff --git a/dataforge-meta/api/dataforge-meta.api b/dataforge-meta/api/dataforge-meta.api index 1700ca7d..7564da80 100644 --- a/dataforge-meta/api/dataforge-meta.api +++ b/dataforge-meta/api/dataforge-meta.api @@ -56,6 +56,20 @@ public final class space/kscience/dataforge/meta/JsonMetaKt { public static final fun toValue (Lkotlinx/serialization/json/JsonPrimitive;Lspace/kscience/dataforge/meta/descriptors/MetaDescriptor;)Lspace/kscience/dataforge/meta/Value; } +public final class space/kscience/dataforge/meta/KeepTransformationRule : space/kscience/dataforge/meta/TransformationRule { + public fun (Lkotlin/jvm/functions/Function1;)V + public final fun component1 ()Lkotlin/jvm/functions/Function1; + public final fun copy (Lkotlin/jvm/functions/Function1;)Lspace/kscience/dataforge/meta/KeepTransformationRule; + public static synthetic fun copy$default (Lspace/kscience/dataforge/meta/KeepTransformationRule;Lkotlin/jvm/functions/Function1;ILjava/lang/Object;)Lspace/kscience/dataforge/meta/KeepTransformationRule; + public fun equals (Ljava/lang/Object;)Z + public final fun getSelector ()Lkotlin/jvm/functions/Function1; + public fun hashCode ()I + public fun matches (Lspace/kscience/dataforge/names/Name;Lspace/kscience/dataforge/meta/Meta;)Z + public fun selectItems (Lspace/kscience/dataforge/meta/Meta;)Lkotlin/sequences/Sequence; + public fun toString ()Ljava/lang/String; + public fun transformItem (Lspace/kscience/dataforge/names/Name;Lspace/kscience/dataforge/meta/Meta;Lspace/kscience/dataforge/meta/MutableMeta;)V +} + public final class space/kscience/dataforge/meta/Laminate : space/kscience/dataforge/meta/TypedMeta { public static final field Companion Lspace/kscience/dataforge/meta/Laminate$Companion; public fun equals (Ljava/lang/Object;)Z @@ -159,6 +173,32 @@ public final class space/kscience/dataforge/meta/MetaBuilder : space/kscience/da public abstract interface annotation class space/kscience/dataforge/meta/MetaBuilderMarker : java/lang/annotation/Annotation { } +public abstract interface class space/kscience/dataforge/meta/MetaConverter : space/kscience/dataforge/meta/MetaSpec { + public static final field Companion Lspace/kscience/dataforge/meta/MetaConverter$Companion; + public abstract fun convert (Ljava/lang/Object;)Lspace/kscience/dataforge/meta/Meta; + public fun getDescriptor ()Lspace/kscience/dataforge/meta/descriptors/MetaDescriptor; + public fun read (Lspace/kscience/dataforge/meta/Meta;)Ljava/lang/Object; + public abstract fun readOrNull (Lspace/kscience/dataforge/meta/Meta;)Ljava/lang/Object; +} + +public final class space/kscience/dataforge/meta/MetaConverter$Companion { + public final fun getBoolean ()Lspace/kscience/dataforge/meta/MetaConverter; + public final fun getDouble ()Lspace/kscience/dataforge/meta/MetaConverter; + public final fun getFloat ()Lspace/kscience/dataforge/meta/MetaConverter; + public final fun getInt ()Lspace/kscience/dataforge/meta/MetaConverter; + public final fun getLong ()Lspace/kscience/dataforge/meta/MetaConverter; + public final fun getMeta ()Lspace/kscience/dataforge/meta/MetaConverter; + public final fun getNumber ()Lspace/kscience/dataforge/meta/MetaConverter; + public final fun getString ()Lspace/kscience/dataforge/meta/MetaConverter; + public final fun getValue ()Lspace/kscience/dataforge/meta/MetaConverter; + public final fun valueList (Lkotlin/jvm/functions/Function1;Lkotlin/jvm/functions/Function1;)Lspace/kscience/dataforge/meta/MetaConverter; + public static synthetic fun valueList$default (Lspace/kscience/dataforge/meta/MetaConverter$Companion;Lkotlin/jvm/functions/Function1;Lkotlin/jvm/functions/Function1;ILjava/lang/Object;)Lspace/kscience/dataforge/meta/MetaConverter; +} + +public final class space/kscience/dataforge/meta/MetaConverterKt { + public static final fun convertNullable (Lspace/kscience/dataforge/meta/MetaConverter;Ljava/lang/Object;)Lspace/kscience/dataforge/meta/Meta; +} + public final class space/kscience/dataforge/meta/MetaDelegateKt { public static final fun boolean (Lspace/kscience/dataforge/meta/MetaProvider;Lspace/kscience/dataforge/names/Name;)Lkotlin/properties/ReadOnlyProperty; public static final fun boolean (Lspace/kscience/dataforge/meta/MetaProvider;Lspace/kscience/dataforge/names/Name;Lkotlin/jvm/functions/Function0;)Lkotlin/properties/ReadOnlyProperty; @@ -178,20 +218,24 @@ public final class space/kscience/dataforge/meta/MetaDelegateKt { public static final fun int (Lspace/kscience/dataforge/meta/MetaProvider;Lspace/kscience/dataforge/names/Name;)Lkotlin/properties/ReadOnlyProperty; public static synthetic fun int$default (Lspace/kscience/dataforge/meta/MetaProvider;ILspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lkotlin/properties/ReadOnlyProperty; public static synthetic fun int$default (Lspace/kscience/dataforge/meta/MetaProvider;Lspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lkotlin/properties/ReadOnlyProperty; + public static final fun listOfSpec (Lspace/kscience/dataforge/meta/Meta;Lspace/kscience/dataforge/meta/MetaSpec;Lspace/kscience/dataforge/names/Name;)Lkotlin/properties/ReadOnlyProperty; + public static synthetic fun listOfSpec$default (Lspace/kscience/dataforge/meta/Meta;Lspace/kscience/dataforge/meta/MetaSpec;Lspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lkotlin/properties/ReadOnlyProperty; public static final fun long (Lspace/kscience/dataforge/meta/MetaProvider;JLspace/kscience/dataforge/names/Name;)Lkotlin/properties/ReadOnlyProperty; public static final fun long (Lspace/kscience/dataforge/meta/MetaProvider;Lspace/kscience/dataforge/names/Name;)Lkotlin/properties/ReadOnlyProperty; public static synthetic fun long$default (Lspace/kscience/dataforge/meta/MetaProvider;JLspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lkotlin/properties/ReadOnlyProperty; public static synthetic fun long$default (Lspace/kscience/dataforge/meta/MetaProvider;Lspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lkotlin/properties/ReadOnlyProperty; public static final fun node (Lspace/kscience/dataforge/meta/MetaProvider;Lspace/kscience/dataforge/names/Name;)Lkotlin/properties/ReadOnlyProperty; - public static final fun node (Lspace/kscience/dataforge/meta/MetaProvider;Lspace/kscience/dataforge/names/Name;Lspace/kscience/dataforge/meta/transformations/MetaConverter;)Lkotlin/properties/ReadOnlyProperty; + public static final fun node (Lspace/kscience/dataforge/meta/MetaProvider;Lspace/kscience/dataforge/names/Name;Lspace/kscience/dataforge/meta/MetaSpec;)Lkotlin/properties/ReadOnlyProperty; public static synthetic fun node$default (Lspace/kscience/dataforge/meta/MetaProvider;Lspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lkotlin/properties/ReadOnlyProperty; - public static synthetic fun node$default (Lspace/kscience/dataforge/meta/MetaProvider;Lspace/kscience/dataforge/names/Name;Lspace/kscience/dataforge/meta/transformations/MetaConverter;ILjava/lang/Object;)Lkotlin/properties/ReadOnlyProperty; + public static synthetic fun node$default (Lspace/kscience/dataforge/meta/MetaProvider;Lspace/kscience/dataforge/names/Name;Lspace/kscience/dataforge/meta/MetaSpec;ILjava/lang/Object;)Lkotlin/properties/ReadOnlyProperty; public static final fun number (Lspace/kscience/dataforge/meta/MetaProvider;Ljava/lang/Number;Lspace/kscience/dataforge/names/Name;)Lkotlin/properties/ReadOnlyProperty; public static final fun number (Lspace/kscience/dataforge/meta/MetaProvider;Lspace/kscience/dataforge/names/Name;)Lkotlin/properties/ReadOnlyProperty; public static final fun number (Lspace/kscience/dataforge/meta/MetaProvider;Lspace/kscience/dataforge/names/Name;Lkotlin/jvm/functions/Function0;)Lkotlin/properties/ReadOnlyProperty; public static synthetic fun number$default (Lspace/kscience/dataforge/meta/MetaProvider;Ljava/lang/Number;Lspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lkotlin/properties/ReadOnlyProperty; public static synthetic fun number$default (Lspace/kscience/dataforge/meta/MetaProvider;Lspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lkotlin/properties/ReadOnlyProperty; public static synthetic fun number$default (Lspace/kscience/dataforge/meta/MetaProvider;Lspace/kscience/dataforge/names/Name;Lkotlin/jvm/functions/Function0;ILjava/lang/Object;)Lkotlin/properties/ReadOnlyProperty; + public static final fun spec (Lspace/kscience/dataforge/meta/MetaProvider;Lspace/kscience/dataforge/meta/MetaSpec;Lspace/kscience/dataforge/names/Name;)Lkotlin/properties/ReadOnlyProperty; + public static synthetic fun spec$default (Lspace/kscience/dataforge/meta/MetaProvider;Lspace/kscience/dataforge/meta/MetaSpec;Lspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lkotlin/properties/ReadOnlyProperty; public static final fun string (Lspace/kscience/dataforge/meta/MetaProvider;Ljava/lang/String;Lspace/kscience/dataforge/names/Name;)Lkotlin/properties/ReadOnlyProperty; public static final fun string (Lspace/kscience/dataforge/meta/MetaProvider;Lspace/kscience/dataforge/names/Name;)Lkotlin/properties/ReadOnlyProperty; public static final fun string (Lspace/kscience/dataforge/meta/MetaProvider;Lspace/kscience/dataforge/names/Name;Lkotlin/jvm/functions/Function0;)Lkotlin/properties/ReadOnlyProperty; @@ -252,6 +296,47 @@ public final class space/kscience/dataforge/meta/MetaSerializer : kotlinx/serial public fun serialize (Lkotlinx/serialization/encoding/Encoder;Lspace/kscience/dataforge/meta/Meta;)V } +public abstract interface class space/kscience/dataforge/meta/MetaSpec : space/kscience/dataforge/meta/descriptors/Described { + public fun read (Lspace/kscience/dataforge/meta/Meta;)Ljava/lang/Object; + public abstract fun readOrNull (Lspace/kscience/dataforge/meta/Meta;)Ljava/lang/Object; +} + +public final class space/kscience/dataforge/meta/MetaSpecKt { + public static final fun readNullable (Lspace/kscience/dataforge/meta/MetaSpec;Lspace/kscience/dataforge/meta/Meta;)Ljava/lang/Object; + public static final fun readValue (Lspace/kscience/dataforge/meta/MetaSpec;Lspace/kscience/dataforge/meta/Value;)Ljava/lang/Object; +} + +public final class space/kscience/dataforge/meta/MetaTransformation { + public static final field Companion Lspace/kscience/dataforge/meta/MetaTransformation$Companion; + public static final fun apply-impl (Ljava/util/Collection;Lspace/kscience/dataforge/meta/Meta;)Lspace/kscience/dataforge/meta/Meta; + public static final fun bind-impl (Ljava/util/Collection;Lspace/kscience/dataforge/meta/ObservableMeta;Lspace/kscience/dataforge/meta/MutableMeta;)V + public static final synthetic fun box-impl (Ljava/util/Collection;)Lspace/kscience/dataforge/meta/MetaTransformation; + public static fun constructor-impl (Ljava/util/Collection;)Ljava/util/Collection; + public fun equals (Ljava/lang/Object;)Z + public static fun equals-impl (Ljava/util/Collection;Ljava/lang/Object;)Z + public static final fun equals-impl0 (Ljava/util/Collection;Ljava/util/Collection;)Z + public static final fun generate-impl (Ljava/util/Collection;Lspace/kscience/dataforge/meta/Meta;)Lspace/kscience/dataforge/meta/Meta; + public fun hashCode ()I + public static fun hashCode-impl (Ljava/util/Collection;)I + public fun toString ()Ljava/lang/String; + public static fun toString-impl (Ljava/util/Collection;)Ljava/lang/String; + public final synthetic fun unbox-impl ()Ljava/util/Collection; +} + +public final class space/kscience/dataforge/meta/MetaTransformation$Companion { + public final fun make--mWxz5M (Lkotlin/jvm/functions/Function1;)Ljava/util/Collection; +} + +public final class space/kscience/dataforge/meta/MetaTransformationBuilder { + public fun ()V + public final fun build-m6Fha10 ()Ljava/util/Collection; + public final fun keep (Ljava/lang/String;)V + public final fun keep (Lkotlin/jvm/functions/Function1;)V + public final fun keep (Lspace/kscience/dataforge/names/Name;)V + public final fun move (Lspace/kscience/dataforge/names/Name;Lspace/kscience/dataforge/names/Name;Lkotlin/jvm/functions/Function1;)V + public static synthetic fun move$default (Lspace/kscience/dataforge/meta/MetaTransformationBuilder;Lspace/kscience/dataforge/names/Name;Lspace/kscience/dataforge/names/Name;Lkotlin/jvm/functions/Function1;ILjava/lang/Object;)V +} + public abstract interface class space/kscience/dataforge/meta/MutableMeta : space/kscience/dataforge/meta/Meta, space/kscience/dataforge/meta/MutableMetaProvider { public static final field Companion Lspace/kscience/dataforge/meta/MutableMeta$Companion; public synthetic fun get (Lspace/kscience/dataforge/names/Name;)Lspace/kscience/dataforge/meta/Meta; @@ -293,6 +378,8 @@ public final class space/kscience/dataforge/meta/MutableMetaDelegateKt { public static synthetic fun boolean$default (Lspace/kscience/dataforge/meta/MutableMetaProvider;Lspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lkotlin/properties/ReadWriteProperty; public static synthetic fun boolean$default (Lspace/kscience/dataforge/meta/MutableMetaProvider;Lspace/kscience/dataforge/names/Name;Lkotlin/jvm/functions/Function0;ILjava/lang/Object;)Lkotlin/properties/ReadWriteProperty; public static synthetic fun boolean$default (Lspace/kscience/dataforge/meta/MutableMetaProvider;ZLspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lkotlin/properties/ReadWriteProperty; + public static final fun convertable (Lspace/kscience/dataforge/meta/MutableMetaProvider;Lspace/kscience/dataforge/meta/MetaConverter;Lspace/kscience/dataforge/names/Name;)Lkotlin/properties/ReadWriteProperty; + public static synthetic fun convertable$default (Lspace/kscience/dataforge/meta/MutableMetaProvider;Lspace/kscience/dataforge/meta/MetaConverter;Lspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lkotlin/properties/ReadWriteProperty; public static final fun double (Lspace/kscience/dataforge/meta/MutableMetaProvider;DLspace/kscience/dataforge/names/Name;)Lkotlin/properties/ReadWriteProperty; public static final fun double (Lspace/kscience/dataforge/meta/MutableMetaProvider;Lspace/kscience/dataforge/names/Name;)Lkotlin/properties/ReadWriteProperty; public static synthetic fun double$default (Lspace/kscience/dataforge/meta/MutableMetaProvider;DLspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lkotlin/properties/ReadWriteProperty; @@ -307,6 +394,8 @@ public final class space/kscience/dataforge/meta/MutableMetaDelegateKt { public static final fun int (Lspace/kscience/dataforge/meta/MutableMetaProvider;Lspace/kscience/dataforge/names/Name;)Lkotlin/properties/ReadWriteProperty; public static synthetic fun int$default (Lspace/kscience/dataforge/meta/MutableMetaProvider;ILspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lkotlin/properties/ReadWriteProperty; public static synthetic fun int$default (Lspace/kscience/dataforge/meta/MutableMetaProvider;Lspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lkotlin/properties/ReadWriteProperty; + public static final fun listOfConvertable (Lspace/kscience/dataforge/meta/MutableMeta;Lspace/kscience/dataforge/meta/MetaConverter;Lspace/kscience/dataforge/names/Name;)Lkotlin/properties/ReadWriteProperty; + public static synthetic fun listOfConvertable$default (Lspace/kscience/dataforge/meta/MutableMeta;Lspace/kscience/dataforge/meta/MetaConverter;Lspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lkotlin/properties/ReadWriteProperty; public static final fun listValue (Lspace/kscience/dataforge/meta/MutableMetaProvider;Lspace/kscience/dataforge/names/Name;Lkotlin/jvm/functions/Function1;Lkotlin/jvm/functions/Function1;)Lkotlin/properties/ReadWriteProperty; public static synthetic fun listValue$default (Lspace/kscience/dataforge/meta/MutableMetaProvider;Lspace/kscience/dataforge/names/Name;Lkotlin/jvm/functions/Function1;Lkotlin/jvm/functions/Function1;ILjava/lang/Object;)Lkotlin/properties/ReadWriteProperty; public static final fun long (Lspace/kscience/dataforge/meta/MutableMetaProvider;JLspace/kscience/dataforge/names/Name;)Lkotlin/properties/ReadWriteProperty; @@ -314,9 +403,9 @@ public final class space/kscience/dataforge/meta/MutableMetaDelegateKt { public static synthetic fun long$default (Lspace/kscience/dataforge/meta/MutableMetaProvider;JLspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lkotlin/properties/ReadWriteProperty; public static synthetic fun long$default (Lspace/kscience/dataforge/meta/MutableMetaProvider;Lspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lkotlin/properties/ReadWriteProperty; public static final fun node (Lspace/kscience/dataforge/meta/MutableMetaProvider;Lspace/kscience/dataforge/names/Name;)Lkotlin/properties/ReadWriteProperty; - public static final fun node (Lspace/kscience/dataforge/meta/MutableMetaProvider;Lspace/kscience/dataforge/names/Name;Lspace/kscience/dataforge/meta/transformations/MetaConverter;)Lkotlin/properties/ReadWriteProperty; + public static final fun node (Lspace/kscience/dataforge/meta/MutableMetaProvider;Lspace/kscience/dataforge/names/Name;Lspace/kscience/dataforge/meta/MetaConverter;)Lkotlin/properties/ReadWriteProperty; public static synthetic fun node$default (Lspace/kscience/dataforge/meta/MutableMetaProvider;Lspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lkotlin/properties/ReadWriteProperty; - public static synthetic fun node$default (Lspace/kscience/dataforge/meta/MutableMetaProvider;Lspace/kscience/dataforge/names/Name;Lspace/kscience/dataforge/meta/transformations/MetaConverter;ILjava/lang/Object;)Lkotlin/properties/ReadWriteProperty; + public static synthetic fun node$default (Lspace/kscience/dataforge/meta/MutableMetaProvider;Lspace/kscience/dataforge/names/Name;Lspace/kscience/dataforge/meta/MetaConverter;ILjava/lang/Object;)Lkotlin/properties/ReadWriteProperty; public static final fun number (Lspace/kscience/dataforge/meta/MutableMetaProvider;Ljava/lang/Number;Lspace/kscience/dataforge/names/Name;)Lkotlin/properties/ReadWriteProperty; public static final fun number (Lspace/kscience/dataforge/meta/MutableMetaProvider;Lspace/kscience/dataforge/names/Name;)Lkotlin/properties/ReadWriteProperty; public static final fun number (Lspace/kscience/dataforge/meta/MutableMetaProvider;Lspace/kscience/dataforge/names/Name;Lkotlin/jvm/functions/Function0;)Lkotlin/properties/ReadWriteProperty; @@ -365,7 +454,7 @@ public final class space/kscience/dataforge/meta/MutableMetaKt { public static final fun set (Lspace/kscience/dataforge/meta/MutableValueProvider;Lspace/kscience/dataforge/names/Name;Lspace/kscience/dataforge/meta/Value;)V public static final fun setIndexed (Lspace/kscience/dataforge/meta/MutableMetaProvider;Lspace/kscience/dataforge/names/Name;Ljava/lang/Iterable;Lkotlin/jvm/functions/Function2;)V public static synthetic fun setIndexed$default (Lspace/kscience/dataforge/meta/MutableMetaProvider;Lspace/kscience/dataforge/names/Name;Ljava/lang/Iterable;Lkotlin/jvm/functions/Function2;ILjava/lang/Object;)V - public static final fun toMutableMeta (Lspace/kscience/dataforge/meta/Meta;)Lspace/kscience/dataforge/meta/ObservableMutableMeta; + public static final fun toMutableMeta (Lspace/kscience/dataforge/meta/Meta;)Lspace/kscience/dataforge/meta/MutableMeta; public static final fun update (Lspace/kscience/dataforge/meta/MutableMetaProvider;Lspace/kscience/dataforge/meta/Meta;)V public static final fun withDefault (Lspace/kscience/dataforge/meta/MutableMeta;Lspace/kscience/dataforge/meta/MetaProvider;)Lspace/kscience/dataforge/meta/MutableMeta; } @@ -419,11 +508,6 @@ public abstract interface class space/kscience/dataforge/meta/ObservableMeta : s public abstract fun removeListener (Ljava/lang/Object;)V } -public final class space/kscience/dataforge/meta/ObservableMetaKt { - public static final fun useProperty (Lspace/kscience/dataforge/meta/Scheme;Lkotlin/reflect/KProperty1;Ljava/lang/Object;Lkotlin/jvm/functions/Function2;)V - public static synthetic fun useProperty$default (Lspace/kscience/dataforge/meta/Scheme;Lkotlin/reflect/KProperty1;Ljava/lang/Object;Lkotlin/jvm/functions/Function2;ILjava/lang/Object;)V -} - public final class space/kscience/dataforge/meta/ObservableMetaWrapperKt { public static final fun asObservable (Lspace/kscience/dataforge/meta/MutableMeta;)Lspace/kscience/dataforge/meta/ObservableMutableMeta; } @@ -437,10 +521,19 @@ public abstract interface class space/kscience/dataforge/meta/ObservableMutableM public abstract fun getOrCreate (Lspace/kscience/dataforge/names/Name;)Lspace/kscience/dataforge/meta/ObservableMutableMeta; } -public abstract interface class space/kscience/dataforge/meta/ReadOnlySpecification : space/kscience/dataforge/meta/descriptors/Described { - public abstract fun empty ()Ljava/lang/Object; - public fun invoke (Lkotlin/jvm/functions/Function1;)Ljava/lang/Object; - public abstract fun read (Lspace/kscience/dataforge/meta/Meta;)Ljava/lang/Object; +public final class space/kscience/dataforge/meta/RegexItemTransformationRule : space/kscience/dataforge/meta/TransformationRule { + public fun (Lkotlin/text/Regex;Lkotlin/jvm/functions/Function4;)V + public final fun component1 ()Lkotlin/text/Regex; + public final fun component2 ()Lkotlin/jvm/functions/Function4; + public final fun copy (Lkotlin/text/Regex;Lkotlin/jvm/functions/Function4;)Lspace/kscience/dataforge/meta/RegexItemTransformationRule; + public static synthetic fun copy$default (Lspace/kscience/dataforge/meta/RegexItemTransformationRule;Lkotlin/text/Regex;Lkotlin/jvm/functions/Function4;ILjava/lang/Object;)Lspace/kscience/dataforge/meta/RegexItemTransformationRule; + public fun equals (Ljava/lang/Object;)Z + public final fun getFrom ()Lkotlin/text/Regex; + public final fun getTransform ()Lkotlin/jvm/functions/Function4; + public fun hashCode ()I + public fun matches (Lspace/kscience/dataforge/names/Name;Lspace/kscience/dataforge/meta/Meta;)Z + public fun toString ()Ljava/lang/String; + public fun transformItem (Lspace/kscience/dataforge/names/Name;Lspace/kscience/dataforge/meta/Meta;Lspace/kscience/dataforge/meta/MutableMeta;)V } public class space/kscience/dataforge/meta/Scheme : space/kscience/dataforge/meta/Configurable, space/kscience/dataforge/meta/MetaRepr, space/kscience/dataforge/meta/MutableMetaProvider, space/kscience/dataforge/meta/descriptors/Described { @@ -454,6 +547,7 @@ public class space/kscience/dataforge/meta/Scheme : space/kscience/dataforge/met public fun setValue (Lspace/kscience/dataforge/names/Name;Lspace/kscience/dataforge/meta/Value;)V public fun toMeta ()Lspace/kscience/dataforge/meta/Laminate; public synthetic fun toMeta ()Lspace/kscience/dataforge/meta/Meta; + public fun toString ()Ljava/lang/String; public fun validate (Lspace/kscience/dataforge/names/Name;Lspace/kscience/dataforge/meta/Meta;)Z } @@ -461,20 +555,30 @@ public final class space/kscience/dataforge/meta/SchemeKt { public static final fun copy (Lspace/kscience/dataforge/meta/Scheme;Lspace/kscience/dataforge/meta/SchemeSpec;Lkotlin/jvm/functions/Function1;)Lspace/kscience/dataforge/meta/Scheme; public static synthetic fun copy$default (Lspace/kscience/dataforge/meta/Scheme;Lspace/kscience/dataforge/meta/SchemeSpec;Lkotlin/jvm/functions/Function1;ILjava/lang/Object;)Lspace/kscience/dataforge/meta/Scheme; public static final fun invoke (Lspace/kscience/dataforge/meta/Scheme;Lkotlin/jvm/functions/Function1;)Lspace/kscience/dataforge/meta/Scheme; - public static final fun retarget (Lspace/kscience/dataforge/meta/Scheme;Lspace/kscience/dataforge/meta/MutableMeta;)Lspace/kscience/dataforge/meta/Scheme; + public static final fun scheme (Lspace/kscience/dataforge/meta/MutableMeta;Lspace/kscience/dataforge/meta/SchemeSpec;Lspace/kscience/dataforge/names/Name;)Lkotlin/properties/ReadWriteProperty; + public static final fun scheme (Lspace/kscience/dataforge/meta/Scheme;Lspace/kscience/dataforge/meta/SchemeSpec;Lspace/kscience/dataforge/names/Name;)Lkotlin/properties/ReadWriteProperty; + public static synthetic fun scheme$default (Lspace/kscience/dataforge/meta/MutableMeta;Lspace/kscience/dataforge/meta/SchemeSpec;Lspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lkotlin/properties/ReadWriteProperty; + public static synthetic fun scheme$default (Lspace/kscience/dataforge/meta/Scheme;Lspace/kscience/dataforge/meta/SchemeSpec;Lspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lkotlin/properties/ReadWriteProperty; + public static final fun schemeOrNull (Lspace/kscience/dataforge/meta/MutableMeta;Lspace/kscience/dataforge/meta/SchemeSpec;Lspace/kscience/dataforge/names/Name;)Lkotlin/properties/ReadWriteProperty; + public static final fun schemeOrNull (Lspace/kscience/dataforge/meta/Scheme;Lspace/kscience/dataforge/meta/SchemeSpec;Lspace/kscience/dataforge/names/Name;)Lkotlin/properties/ReadWriteProperty; + public static synthetic fun schemeOrNull$default (Lspace/kscience/dataforge/meta/MutableMeta;Lspace/kscience/dataforge/meta/SchemeSpec;Lspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lkotlin/properties/ReadWriteProperty; + public static synthetic fun schemeOrNull$default (Lspace/kscience/dataforge/meta/Scheme;Lspace/kscience/dataforge/meta/SchemeSpec;Lspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lkotlin/properties/ReadWriteProperty; + public static final fun updateWith (Lspace/kscience/dataforge/meta/Configurable;Lspace/kscience/dataforge/meta/SchemeSpec;Lkotlin/jvm/functions/Function1;)Lspace/kscience/dataforge/meta/Scheme; + public static final fun updateWith (Lspace/kscience/dataforge/meta/MutableMeta;Lspace/kscience/dataforge/meta/SchemeSpec;Lkotlin/jvm/functions/Function1;)Lspace/kscience/dataforge/meta/Scheme; + public static final fun useProperty (Lspace/kscience/dataforge/meta/Scheme;Lkotlin/reflect/KProperty1;Ljava/lang/Object;Lkotlin/jvm/functions/Function2;)V + public static synthetic fun useProperty$default (Lspace/kscience/dataforge/meta/Scheme;Lkotlin/reflect/KProperty1;Ljava/lang/Object;Lkotlin/jvm/functions/Function2;ILjava/lang/Object;)V } -public class space/kscience/dataforge/meta/SchemeSpec : space/kscience/dataforge/meta/Specification { +public class space/kscience/dataforge/meta/SchemeSpec : space/kscience/dataforge/meta/MetaConverter { public fun (Lkotlin/jvm/functions/Function0;)V - public synthetic fun empty ()Ljava/lang/Object; - public fun empty ()Lspace/kscience/dataforge/meta/Scheme; + public synthetic fun convert (Ljava/lang/Object;)Lspace/kscience/dataforge/meta/Meta; + public fun convert (Lspace/kscience/dataforge/meta/Scheme;)Lspace/kscience/dataforge/meta/Meta; + public final fun empty ()Lspace/kscience/dataforge/meta/Scheme; public fun getDescriptor ()Lspace/kscience/dataforge/meta/descriptors/MetaDescriptor; - public synthetic fun invoke (Lkotlin/jvm/functions/Function1;)Ljava/lang/Object; public final fun invoke (Lkotlin/jvm/functions/Function1;)Lspace/kscience/dataforge/meta/Scheme; - public synthetic fun read (Lspace/kscience/dataforge/meta/Meta;)Ljava/lang/Object; - public fun read (Lspace/kscience/dataforge/meta/Meta;)Lspace/kscience/dataforge/meta/Scheme; - public synthetic fun write (Lspace/kscience/dataforge/meta/MutableMeta;)Ljava/lang/Object; - public fun write (Lspace/kscience/dataforge/meta/MutableMeta;)Lspace/kscience/dataforge/meta/Scheme; + public synthetic fun readOrNull (Lspace/kscience/dataforge/meta/Meta;)Ljava/lang/Object; + public fun readOrNull (Lspace/kscience/dataforge/meta/Meta;)Lspace/kscience/dataforge/meta/Scheme; + public final fun write (Lspace/kscience/dataforge/meta/MutableMeta;)Lspace/kscience/dataforge/meta/Scheme; } public final class space/kscience/dataforge/meta/SealedMeta : space/kscience/dataforge/meta/TypedMeta { @@ -515,21 +619,20 @@ public final class space/kscience/dataforge/meta/SealedMetaKt { public static final fun seal (Lspace/kscience/dataforge/meta/Meta;)Lspace/kscience/dataforge/meta/SealedMeta; } -public abstract interface class space/kscience/dataforge/meta/Specification : space/kscience/dataforge/meta/ReadOnlySpecification { - public abstract fun write (Lspace/kscience/dataforge/meta/MutableMeta;)Ljava/lang/Object; -} - -public final class space/kscience/dataforge/meta/SpecificationKt { - public static final fun spec (Lspace/kscience/dataforge/meta/MutableMeta;Lspace/kscience/dataforge/meta/Specification;Lspace/kscience/dataforge/names/Name;)Lkotlin/properties/ReadWriteProperty; - public static final fun spec (Lspace/kscience/dataforge/meta/Scheme;Lspace/kscience/dataforge/meta/Specification;Lspace/kscience/dataforge/names/Name;)Lkotlin/properties/ReadWriteProperty; - public static synthetic fun spec$default (Lspace/kscience/dataforge/meta/MutableMeta;Lspace/kscience/dataforge/meta/Specification;Lspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lkotlin/properties/ReadWriteProperty; - public static synthetic fun spec$default (Lspace/kscience/dataforge/meta/Scheme;Lspace/kscience/dataforge/meta/Specification;Lspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lkotlin/properties/ReadWriteProperty; - public static final fun specOrNull (Lspace/kscience/dataforge/meta/MutableMeta;Lspace/kscience/dataforge/meta/Specification;Lspace/kscience/dataforge/names/Name;)Lkotlin/properties/ReadWriteProperty; - public static final fun specOrNull (Lspace/kscience/dataforge/meta/Scheme;Lspace/kscience/dataforge/meta/Specification;Lspace/kscience/dataforge/names/Name;)Lkotlin/properties/ReadWriteProperty; - public static synthetic fun specOrNull$default (Lspace/kscience/dataforge/meta/MutableMeta;Lspace/kscience/dataforge/meta/Specification;Lspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lkotlin/properties/ReadWriteProperty; - public static synthetic fun specOrNull$default (Lspace/kscience/dataforge/meta/Scheme;Lspace/kscience/dataforge/meta/Specification;Lspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lkotlin/properties/ReadWriteProperty; - public static final fun updateWith (Lspace/kscience/dataforge/meta/Configurable;Lspace/kscience/dataforge/meta/Specification;Lkotlin/jvm/functions/Function1;)Ljava/lang/Object; - public static final fun updateWith (Lspace/kscience/dataforge/meta/MutableMeta;Lspace/kscience/dataforge/meta/Specification;Lkotlin/jvm/functions/Function1;)Ljava/lang/Object; +public final class space/kscience/dataforge/meta/SingleItemTransformationRule : space/kscience/dataforge/meta/TransformationRule { + public fun (Lspace/kscience/dataforge/names/Name;Lkotlin/jvm/functions/Function3;)V + public final fun component1 ()Lspace/kscience/dataforge/names/Name; + public final fun component2 ()Lkotlin/jvm/functions/Function3; + public final fun copy (Lspace/kscience/dataforge/names/Name;Lkotlin/jvm/functions/Function3;)Lspace/kscience/dataforge/meta/SingleItemTransformationRule; + public static synthetic fun copy$default (Lspace/kscience/dataforge/meta/SingleItemTransformationRule;Lspace/kscience/dataforge/names/Name;Lkotlin/jvm/functions/Function3;ILjava/lang/Object;)Lspace/kscience/dataforge/meta/SingleItemTransformationRule; + public fun equals (Ljava/lang/Object;)Z + public final fun getFrom ()Lspace/kscience/dataforge/names/Name; + public final fun getTransform ()Lkotlin/jvm/functions/Function3; + public fun hashCode ()I + public fun matches (Lspace/kscience/dataforge/names/Name;Lspace/kscience/dataforge/meta/Meta;)Z + public fun selectItems (Lspace/kscience/dataforge/meta/Meta;)Lkotlin/sequences/Sequence; + public fun toString ()Ljava/lang/String; + public fun transformItem (Lspace/kscience/dataforge/names/Name;Lspace/kscience/dataforge/meta/Meta;Lspace/kscience/dataforge/meta/MutableMeta;)V } public final class space/kscience/dataforge/meta/StringValue : space/kscience/dataforge/meta/Value { @@ -550,6 +653,12 @@ public final class space/kscience/dataforge/meta/StringValue : space/kscience/da public final synthetic fun unbox-impl ()Ljava/lang/String; } +public abstract interface class space/kscience/dataforge/meta/TransformationRule { + public abstract fun matches (Lspace/kscience/dataforge/names/Name;Lspace/kscience/dataforge/meta/Meta;)Z + public fun selectItems (Lspace/kscience/dataforge/meta/Meta;)Lkotlin/sequences/Sequence; + public abstract fun transformItem (Lspace/kscience/dataforge/names/Name;Lspace/kscience/dataforge/meta/Meta;Lspace/kscience/dataforge/meta/MutableMeta;)V +} + public final class space/kscience/dataforge/meta/True : space/kscience/dataforge/meta/Value { public static final field INSTANCE Lspace/kscience/dataforge/meta/True; public fun equals (Ljava/lang/Object;)Z @@ -692,6 +801,7 @@ public final class space/kscience/dataforge/meta/descriptors/MetaDescriptor { public final fun getDescription ()Ljava/lang/String; public final fun getIndexKey ()Ljava/lang/String; public final fun getMultiple ()Z + public final fun getNodes ()Ljava/util/Map; public final fun getValueRestriction ()Lspace/kscience/dataforge/meta/descriptors/ValueRestriction; public final fun getValueTypes ()Ljava/util/List; public fun hashCode ()I @@ -720,6 +830,7 @@ public final class space/kscience/dataforge/meta/descriptors/MetaDescriptorBuild public final fun attributes (Lkotlin/jvm/functions/Function1;)V public final fun build ()Lspace/kscience/dataforge/meta/descriptors/MetaDescriptor; public final fun default (Ljava/lang/Object;)V + public final fun from (Lspace/kscience/dataforge/meta/descriptors/MetaDescriptor;)V public final fun getAllowedValues ()Ljava/util/List; public final fun getAttributes ()Lspace/kscience/dataforge/meta/MutableMeta; public final fun getChildren ()Ljava/util/Map; @@ -730,10 +841,6 @@ public final class space/kscience/dataforge/meta/descriptors/MetaDescriptorBuild public final fun getMultiple ()Z public final fun getValueRestriction ()Lspace/kscience/dataforge/meta/descriptors/ValueRestriction; public final fun getValueTypes ()Ljava/util/List; - public final fun item (Lspace/kscience/dataforge/names/Name;Lkotlin/jvm/functions/Function1;)Lspace/kscience/dataforge/meta/descriptors/MetaDescriptorBuilder; - public static synthetic fun item$default (Lspace/kscience/dataforge/meta/descriptors/MetaDescriptorBuilder;Lspace/kscience/dataforge/names/Name;Lkotlin/jvm/functions/Function1;ILjava/lang/Object;)Lspace/kscience/dataforge/meta/descriptors/MetaDescriptorBuilder; - public final fun node (Lspace/kscience/dataforge/names/Name;Lspace/kscience/dataforge/meta/descriptors/MetaDescriptor;Lkotlin/jvm/functions/Function1;)Lspace/kscience/dataforge/meta/descriptors/MetaDescriptorBuilder; - public static synthetic fun node$default (Lspace/kscience/dataforge/meta/descriptors/MetaDescriptorBuilder;Lspace/kscience/dataforge/names/Name;Lspace/kscience/dataforge/meta/descriptors/MetaDescriptor;Lkotlin/jvm/functions/Function1;ILjava/lang/Object;)Lspace/kscience/dataforge/meta/descriptors/MetaDescriptorBuilder; public final fun setAllowedValues (Ljava/util/List;)V public final fun setAttributes (Lspace/kscience/dataforge/meta/MutableMeta;)V public final fun setChildren (Ljava/util/Map;)V @@ -751,16 +858,16 @@ public final class space/kscience/dataforge/meta/descriptors/MetaDescriptorBuild public static final fun MetaDescriptor (Lkotlin/jvm/functions/Function1;)Lspace/kscience/dataforge/meta/descriptors/MetaDescriptor; public static final fun copy (Lspace/kscience/dataforge/meta/descriptors/MetaDescriptor;Lkotlin/jvm/functions/Function1;)Lspace/kscience/dataforge/meta/descriptors/MetaDescriptor; public static synthetic fun copy$default (Lspace/kscience/dataforge/meta/descriptors/MetaDescriptor;Lkotlin/jvm/functions/Function1;ILjava/lang/Object;)Lspace/kscience/dataforge/meta/descriptors/MetaDescriptor; - public static final fun item (Lspace/kscience/dataforge/meta/descriptors/MetaDescriptorBuilder;Ljava/lang/String;Lkotlin/jvm/functions/Function1;)Lspace/kscience/dataforge/meta/descriptors/MetaDescriptorBuilder; public static final fun node (Lspace/kscience/dataforge/meta/descriptors/MetaDescriptorBuilder;Ljava/lang/String;Lkotlin/jvm/functions/Function1;)V public static final fun node (Lspace/kscience/dataforge/meta/descriptors/MetaDescriptorBuilder;Ljava/lang/String;Lspace/kscience/dataforge/meta/descriptors/Described;Lkotlin/jvm/functions/Function1;)V - public static final fun node (Lspace/kscience/dataforge/meta/descriptors/MetaDescriptorBuilder;Lspace/kscience/dataforge/names/Name;Lkotlin/jvm/functions/Function1;)Lspace/kscience/dataforge/meta/descriptors/MetaDescriptorBuilder; + public static final fun node (Lspace/kscience/dataforge/meta/descriptors/MetaDescriptorBuilder;Ljava/lang/String;Lspace/kscience/dataforge/meta/descriptors/MetaDescriptor;)V + public static final fun node (Lspace/kscience/dataforge/meta/descriptors/MetaDescriptorBuilder;Lspace/kscience/dataforge/names/Name;Lkotlin/jvm/functions/Function1;)V public static synthetic fun node$default (Lspace/kscience/dataforge/meta/descriptors/MetaDescriptorBuilder;Ljava/lang/String;Lspace/kscience/dataforge/meta/descriptors/Described;Lkotlin/jvm/functions/Function1;ILjava/lang/Object;)V public static final fun required (Lspace/kscience/dataforge/meta/descriptors/MetaDescriptorBuilder;)V - public static final fun value (Lspace/kscience/dataforge/meta/descriptors/MetaDescriptorBuilder;Ljava/lang/String;Lspace/kscience/dataforge/meta/ValueType;[Lspace/kscience/dataforge/meta/ValueType;Lkotlin/jvm/functions/Function1;)Lspace/kscience/dataforge/meta/descriptors/MetaDescriptorBuilder; - public static final fun value (Lspace/kscience/dataforge/meta/descriptors/MetaDescriptorBuilder;Lspace/kscience/dataforge/names/Name;Lspace/kscience/dataforge/meta/ValueType;[Lspace/kscience/dataforge/meta/ValueType;Lkotlin/jvm/functions/Function1;)Lspace/kscience/dataforge/meta/descriptors/MetaDescriptorBuilder; - public static synthetic fun value$default (Lspace/kscience/dataforge/meta/descriptors/MetaDescriptorBuilder;Ljava/lang/String;Lspace/kscience/dataforge/meta/ValueType;[Lspace/kscience/dataforge/meta/ValueType;Lkotlin/jvm/functions/Function1;ILjava/lang/Object;)Lspace/kscience/dataforge/meta/descriptors/MetaDescriptorBuilder; - public static synthetic fun value$default (Lspace/kscience/dataforge/meta/descriptors/MetaDescriptorBuilder;Lspace/kscience/dataforge/names/Name;Lspace/kscience/dataforge/meta/ValueType;[Lspace/kscience/dataforge/meta/ValueType;Lkotlin/jvm/functions/Function1;ILjava/lang/Object;)Lspace/kscience/dataforge/meta/descriptors/MetaDescriptorBuilder; + public static final fun value (Lspace/kscience/dataforge/meta/descriptors/MetaDescriptorBuilder;Ljava/lang/String;Lspace/kscience/dataforge/meta/ValueType;[Lspace/kscience/dataforge/meta/ValueType;Lkotlin/jvm/functions/Function1;)V + public static final fun value (Lspace/kscience/dataforge/meta/descriptors/MetaDescriptorBuilder;Lspace/kscience/dataforge/names/Name;Lspace/kscience/dataforge/meta/ValueType;[Lspace/kscience/dataforge/meta/ValueType;Lkotlin/jvm/functions/Function1;)V + public static synthetic fun value$default (Lspace/kscience/dataforge/meta/descriptors/MetaDescriptorBuilder;Ljava/lang/String;Lspace/kscience/dataforge/meta/ValueType;[Lspace/kscience/dataforge/meta/ValueType;Lkotlin/jvm/functions/Function1;ILjava/lang/Object;)V + public static synthetic fun value$default (Lspace/kscience/dataforge/meta/descriptors/MetaDescriptorBuilder;Lspace/kscience/dataforge/names/Name;Lspace/kscience/dataforge/meta/ValueType;[Lspace/kscience/dataforge/meta/ValueType;Lkotlin/jvm/functions/Function1;ILjava/lang/Object;)V } public final class space/kscience/dataforge/meta/descriptors/MetaDescriptorKt { @@ -781,117 +888,6 @@ public final class space/kscience/dataforge/meta/descriptors/ValueRestriction : public static fun values ()[Lspace/kscience/dataforge/meta/descriptors/ValueRestriction; } -public final class space/kscience/dataforge/meta/transformations/KeepTransformationRule : space/kscience/dataforge/meta/transformations/TransformationRule { - public fun (Lkotlin/jvm/functions/Function1;)V - public final fun component1 ()Lkotlin/jvm/functions/Function1; - public final fun copy (Lkotlin/jvm/functions/Function1;)Lspace/kscience/dataforge/meta/transformations/KeepTransformationRule; - public static synthetic fun copy$default (Lspace/kscience/dataforge/meta/transformations/KeepTransformationRule;Lkotlin/jvm/functions/Function1;ILjava/lang/Object;)Lspace/kscience/dataforge/meta/transformations/KeepTransformationRule; - public fun equals (Ljava/lang/Object;)Z - public final fun getSelector ()Lkotlin/jvm/functions/Function1; - public fun hashCode ()I - public fun matches (Lspace/kscience/dataforge/names/Name;Lspace/kscience/dataforge/meta/Meta;)Z - public fun selectItems (Lspace/kscience/dataforge/meta/Meta;)Lkotlin/sequences/Sequence; - public fun toString ()Ljava/lang/String; - public fun transformItem (Lspace/kscience/dataforge/names/Name;Lspace/kscience/dataforge/meta/Meta;Lspace/kscience/dataforge/meta/MutableMeta;)V -} - -public abstract interface class space/kscience/dataforge/meta/transformations/MetaConverter { - public static final field Companion Lspace/kscience/dataforge/meta/transformations/MetaConverter$Companion; - public fun getDescriptor ()Lspace/kscience/dataforge/meta/descriptors/MetaDescriptor; - public abstract fun getType ()Lkotlin/reflect/KType; - public fun metaToObject (Lspace/kscience/dataforge/meta/Meta;)Ljava/lang/Object; - public abstract fun metaToObjectOrNull (Lspace/kscience/dataforge/meta/Meta;)Ljava/lang/Object; - public abstract fun objectToMeta (Ljava/lang/Object;)Lspace/kscience/dataforge/meta/Meta; -} - -public final class space/kscience/dataforge/meta/transformations/MetaConverter$Companion { - public final fun getBoolean ()Lspace/kscience/dataforge/meta/transformations/MetaConverter; - public final fun getDouble ()Lspace/kscience/dataforge/meta/transformations/MetaConverter; - public final fun getFloat ()Lspace/kscience/dataforge/meta/transformations/MetaConverter; - public final fun getInt ()Lspace/kscience/dataforge/meta/transformations/MetaConverter; - public final fun getLong ()Lspace/kscience/dataforge/meta/transformations/MetaConverter; - public final fun getMeta ()Lspace/kscience/dataforge/meta/transformations/MetaConverter; - public final fun getNumber ()Lspace/kscience/dataforge/meta/transformations/MetaConverter; - public final fun getString ()Lspace/kscience/dataforge/meta/transformations/MetaConverter; - public final fun getValue ()Lspace/kscience/dataforge/meta/transformations/MetaConverter; - public final fun valueList (Lkotlin/jvm/functions/Function1;Lkotlin/jvm/functions/Function1;)Lspace/kscience/dataforge/meta/transformations/MetaConverter; - public static synthetic fun valueList$default (Lspace/kscience/dataforge/meta/transformations/MetaConverter$Companion;Lkotlin/jvm/functions/Function1;Lkotlin/jvm/functions/Function1;ILjava/lang/Object;)Lspace/kscience/dataforge/meta/transformations/MetaConverter; -} - -public final class space/kscience/dataforge/meta/transformations/MetaConverterKt { - public static final fun nullableMetaToObject (Lspace/kscience/dataforge/meta/transformations/MetaConverter;Lspace/kscience/dataforge/meta/Meta;)Ljava/lang/Object; - public static final fun nullableObjectToMeta (Lspace/kscience/dataforge/meta/transformations/MetaConverter;Ljava/lang/Object;)Lspace/kscience/dataforge/meta/Meta; - public static final fun valueToObject (Lspace/kscience/dataforge/meta/transformations/MetaConverter;Lspace/kscience/dataforge/meta/Value;)Ljava/lang/Object; -} - -public final class space/kscience/dataforge/meta/transformations/MetaTransformation { - public static final field Companion Lspace/kscience/dataforge/meta/transformations/MetaTransformation$Companion; - public static final fun apply-impl (Ljava/util/Collection;Lspace/kscience/dataforge/meta/Meta;)Lspace/kscience/dataforge/meta/Meta; - public static final fun bind-impl (Ljava/util/Collection;Lspace/kscience/dataforge/meta/ObservableMeta;Lspace/kscience/dataforge/meta/MutableMeta;)V - public static final synthetic fun box-impl (Ljava/util/Collection;)Lspace/kscience/dataforge/meta/transformations/MetaTransformation; - public static fun constructor-impl (Ljava/util/Collection;)Ljava/util/Collection; - public fun equals (Ljava/lang/Object;)Z - public static fun equals-impl (Ljava/util/Collection;Ljava/lang/Object;)Z - public static final fun equals-impl0 (Ljava/util/Collection;Ljava/util/Collection;)Z - public static final fun generate-impl (Ljava/util/Collection;Lspace/kscience/dataforge/meta/Meta;)Lspace/kscience/dataforge/meta/Meta; - public fun hashCode ()I - public static fun hashCode-impl (Ljava/util/Collection;)I - public fun toString ()Ljava/lang/String; - public static fun toString-impl (Ljava/util/Collection;)Ljava/lang/String; - public final synthetic fun unbox-impl ()Ljava/util/Collection; -} - -public final class space/kscience/dataforge/meta/transformations/MetaTransformation$Companion { - public final fun make-XNaMui4 (Lkotlin/jvm/functions/Function1;)Ljava/util/Collection; -} - -public final class space/kscience/dataforge/meta/transformations/MetaTransformationBuilder { - public fun ()V - public final fun build-050menU ()Ljava/util/Collection; - public final fun keep (Ljava/lang/String;)V - public final fun keep (Lkotlin/jvm/functions/Function1;)V - public final fun keep (Lspace/kscience/dataforge/names/Name;)V - public final fun move (Lspace/kscience/dataforge/names/Name;Lspace/kscience/dataforge/names/Name;Lkotlin/jvm/functions/Function1;)V - public static synthetic fun move$default (Lspace/kscience/dataforge/meta/transformations/MetaTransformationBuilder;Lspace/kscience/dataforge/names/Name;Lspace/kscience/dataforge/names/Name;Lkotlin/jvm/functions/Function1;ILjava/lang/Object;)V -} - -public final class space/kscience/dataforge/meta/transformations/RegexItemTransformationRule : space/kscience/dataforge/meta/transformations/TransformationRule { - public fun (Lkotlin/text/Regex;Lkotlin/jvm/functions/Function4;)V - public final fun component1 ()Lkotlin/text/Regex; - public final fun component2 ()Lkotlin/jvm/functions/Function4; - public final fun copy (Lkotlin/text/Regex;Lkotlin/jvm/functions/Function4;)Lspace/kscience/dataforge/meta/transformations/RegexItemTransformationRule; - public static synthetic fun copy$default (Lspace/kscience/dataforge/meta/transformations/RegexItemTransformationRule;Lkotlin/text/Regex;Lkotlin/jvm/functions/Function4;ILjava/lang/Object;)Lspace/kscience/dataforge/meta/transformations/RegexItemTransformationRule; - public fun equals (Ljava/lang/Object;)Z - public final fun getFrom ()Lkotlin/text/Regex; - public final fun getTransform ()Lkotlin/jvm/functions/Function4; - public fun hashCode ()I - public fun matches (Lspace/kscience/dataforge/names/Name;Lspace/kscience/dataforge/meta/Meta;)Z - public fun toString ()Ljava/lang/String; - public fun transformItem (Lspace/kscience/dataforge/names/Name;Lspace/kscience/dataforge/meta/Meta;Lspace/kscience/dataforge/meta/MutableMeta;)V -} - -public final class space/kscience/dataforge/meta/transformations/SingleItemTransformationRule : space/kscience/dataforge/meta/transformations/TransformationRule { - public fun (Lspace/kscience/dataforge/names/Name;Lkotlin/jvm/functions/Function3;)V - public final fun component1 ()Lspace/kscience/dataforge/names/Name; - public final fun component2 ()Lkotlin/jvm/functions/Function3; - public final fun copy (Lspace/kscience/dataforge/names/Name;Lkotlin/jvm/functions/Function3;)Lspace/kscience/dataforge/meta/transformations/SingleItemTransformationRule; - public static synthetic fun copy$default (Lspace/kscience/dataforge/meta/transformations/SingleItemTransformationRule;Lspace/kscience/dataforge/names/Name;Lkotlin/jvm/functions/Function3;ILjava/lang/Object;)Lspace/kscience/dataforge/meta/transformations/SingleItemTransformationRule; - public fun equals (Ljava/lang/Object;)Z - public final fun getFrom ()Lspace/kscience/dataforge/names/Name; - public final fun getTransform ()Lkotlin/jvm/functions/Function3; - public fun hashCode ()I - public fun matches (Lspace/kscience/dataforge/names/Name;Lspace/kscience/dataforge/meta/Meta;)Z - public fun selectItems (Lspace/kscience/dataforge/meta/Meta;)Lkotlin/sequences/Sequence; - public fun toString ()Ljava/lang/String; - public fun transformItem (Lspace/kscience/dataforge/names/Name;Lspace/kscience/dataforge/meta/Meta;Lspace/kscience/dataforge/meta/MutableMeta;)V -} - -public abstract interface class space/kscience/dataforge/meta/transformations/TransformationRule { - public abstract fun matches (Lspace/kscience/dataforge/names/Name;Lspace/kscience/dataforge/meta/Meta;)Z - public fun selectItems (Lspace/kscience/dataforge/meta/Meta;)Lkotlin/sequences/Sequence; - public abstract fun transformItem (Lspace/kscience/dataforge/names/Name;Lspace/kscience/dataforge/meta/Meta;Lspace/kscience/dataforge/meta/MutableMeta;)V -} - public final class space/kscience/dataforge/misc/CastJvmKt { public static final fun unsafeCast (Ljava/lang/Object;)Ljava/lang/Object; } @@ -972,6 +968,7 @@ public final class space/kscience/dataforge/names/NameKt { public static final fun startsWith (Lspace/kscience/dataforge/names/Name;Ljava/lang/String;)Z public static final fun startsWith (Lspace/kscience/dataforge/names/Name;Lspace/kscience/dataforge/names/Name;)Z public static final fun startsWith (Lspace/kscience/dataforge/names/Name;Lspace/kscience/dataforge/names/NameToken;)Z + public static final fun toStringUnescaped (Lspace/kscience/dataforge/names/Name;)Ljava/lang/String; public static final fun withIndex (Lspace/kscience/dataforge/names/Name;Ljava/lang/String;)Lspace/kscience/dataforge/names/Name; } diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/descriptors/MetaDescriptor.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/descriptors/MetaDescriptor.kt index 12bbd5d4..45954985 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/descriptors/MetaDescriptor.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/descriptors/MetaDescriptor.kt @@ -7,6 +7,7 @@ import space.kscience.dataforge.names.* /** * Restrictions on value in the node */ +@Serializable public enum class ValueRestriction { /** * No restrictions diff --git a/dataforge-scripting/README.md b/dataforge-scripting/README.md index af79cc8f..fbc5cb69 100644 --- a/dataforge-scripting/README.md +++ b/dataforge-scripting/README.md @@ -6,18 +6,16 @@ ## Artifact: -The Maven coordinates of this project are `space.kscience:dataforge-scripting:0.7.0`. +The Maven coordinates of this project are `space.kscience:dataforge-scripting:0.8.0`. **Gradle Kotlin DSL:** ```kotlin repositories { maven("https://repo.kotlin.link") - //uncomment to access development builds - //maven("https://maven.pkg.jetbrains.space/spc/p/sci/dev") mavenCentral() } dependencies { - implementation("space.kscience:dataforge-scripting:0.7.0") + implementation("space.kscience:dataforge-scripting:0.8.0") } ``` diff --git a/dataforge-workspace/README.md b/dataforge-workspace/README.md index c096699f..cea37368 100644 --- a/dataforge-workspace/README.md +++ b/dataforge-workspace/README.md @@ -6,18 +6,16 @@ ## Artifact: -The Maven coordinates of this project are `space.kscience:dataforge-workspace:0.7.0`. +The Maven coordinates of this project are `space.kscience:dataforge-workspace:0.8.0`. **Gradle Kotlin DSL:** ```kotlin repositories { maven("https://repo.kotlin.link") - //uncomment to access development builds - //maven("https://maven.pkg.jetbrains.space/spc/p/sci/dev") mavenCentral() } dependencies { - implementation("space.kscience:dataforge-workspace:0.7.0") + implementation("space.kscience:dataforge-workspace:0.8.0") } ``` -- 2.34.1 From 196b394278afdb67bc3993618ad362ced04b2c4b Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Sat, 3 Feb 2024 19:34:01 +0300 Subject: [PATCH 22/62] Update API --- dataforge-meta/api/dataforge-meta.api | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/dataforge-meta/api/dataforge-meta.api b/dataforge-meta/api/dataforge-meta.api index 7564da80..49f28259 100644 --- a/dataforge-meta/api/dataforge-meta.api +++ b/dataforge-meta/api/dataforge-meta.api @@ -881,6 +881,7 @@ public final class space/kscience/dataforge/meta/descriptors/MetaDescriptorKt { public final class space/kscience/dataforge/meta/descriptors/ValueRestriction : java/lang/Enum { public static final field ABSENT Lspace/kscience/dataforge/meta/descriptors/ValueRestriction; + public static final field Companion Lspace/kscience/dataforge/meta/descriptors/ValueRestriction$Companion; public static final field NONE Lspace/kscience/dataforge/meta/descriptors/ValueRestriction; public static final field REQUIRED Lspace/kscience/dataforge/meta/descriptors/ValueRestriction; public static fun getEntries ()Lkotlin/enums/EnumEntries; @@ -888,6 +889,10 @@ public final class space/kscience/dataforge/meta/descriptors/ValueRestriction : public static fun values ()[Lspace/kscience/dataforge/meta/descriptors/ValueRestriction; } +public final class space/kscience/dataforge/meta/descriptors/ValueRestriction$Companion { + public final fun serializer ()Lkotlinx/serialization/KSerializer; +} + public final class space/kscience/dataforge/misc/CastJvmKt { public static final fun unsafeCast (Ljava/lang/Object;)Ljava/lang/Object; } -- 2.34.1 From 66ce15ae6d90d1fce9648748a73a9959274dc436 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Sun, 4 Feb 2024 13:54:08 +0300 Subject: [PATCH 23/62] Hot-fix for scheme initialization --- build.gradle.kts | 2 +- .../kscience/dataforge/data/ActionsTest.kt | 2 +- dataforge-meta/api/dataforge-meta.api | 9 ++++- .../kscience/dataforge/meta/MutableMeta.kt | 5 +++ .../space/kscience/dataforge/meta/Scheme.kt | 40 +++++++++---------- .../meta/descriptors/MetaDescriptorBuilder.kt | 3 -- .../meta/descriptors/schemeDescriptor.kt | 9 +++-- .../kscience/dataforge/meta/SchemeTest.kt | 23 ++++++++++- 8 files changed, 61 insertions(+), 32 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index b9349868..8be3a7b0 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -8,7 +8,7 @@ plugins { allprojects { group = "space.kscience" - version = "0.8.0" + version = "0.8.1" } subprojects { diff --git a/dataforge-data/src/jvmTest/kotlin/space/kscience/dataforge/data/ActionsTest.kt b/dataforge-data/src/jvmTest/kotlin/space/kscience/dataforge/data/ActionsTest.kt index 4aa6e6d4..f6eae12c 100644 --- a/dataforge-data/src/jvmTest/kotlin/space/kscience/dataforge/data/ActionsTest.kt +++ b/dataforge-data/src/jvmTest/kotlin/space/kscience/dataforge/data/ActionsTest.kt @@ -42,7 +42,7 @@ internal class ActionsTest { source.wrap(it.toString(), it) } - delay(10) + delay(20) source.close() result.awaitClose() diff --git a/dataforge-meta/api/dataforge-meta.api b/dataforge-meta/api/dataforge-meta.api index 49f28259..f9834c54 100644 --- a/dataforge-meta/api/dataforge-meta.api +++ b/dataforge-meta/api/dataforge-meta.api @@ -433,6 +433,7 @@ public final class space/kscience/dataforge/meta/MutableMetaDelegateKt { public final class space/kscience/dataforge/meta/MutableMetaKt { public static final fun ObservableMutableMeta ()Lspace/kscience/dataforge/meta/ObservableMutableMeta; public static final fun ObservableMutableMeta (Lkotlin/jvm/functions/Function1;)Lspace/kscience/dataforge/meta/ObservableMutableMeta; + public static final fun ObservableMutableMeta (Lspace/kscience/dataforge/meta/Meta;)Lspace/kscience/dataforge/meta/ObservableMutableMeta; public static synthetic fun ObservableMutableMeta$default (Lkotlin/jvm/functions/Function1;ILjava/lang/Object;)Lspace/kscience/dataforge/meta/ObservableMutableMeta; public static final fun append (Lspace/kscience/dataforge/meta/MutableMeta;Ljava/lang/String;Lspace/kscience/dataforge/meta/Meta;)V public static final fun append (Lspace/kscience/dataforge/meta/MutableMeta;Ljava/lang/String;Lspace/kscience/dataforge/meta/Value;)V @@ -538,6 +539,8 @@ public final class space/kscience/dataforge/meta/RegexItemTransformationRule : s public class space/kscience/dataforge/meta/Scheme : space/kscience/dataforge/meta/Configurable, space/kscience/dataforge/meta/MetaRepr, space/kscience/dataforge/meta/MutableMetaProvider, space/kscience/dataforge/meta/descriptors/Described { public fun ()V + public fun (Lspace/kscience/dataforge/meta/Meta;Lspace/kscience/dataforge/meta/descriptors/MetaDescriptor;)V + public synthetic fun (Lspace/kscience/dataforge/meta/Meta;Lspace/kscience/dataforge/meta/descriptors/MetaDescriptor;ILkotlin/jvm/internal/DefaultConstructorMarker;)V public synthetic fun get (Lspace/kscience/dataforge/names/Name;)Lspace/kscience/dataforge/meta/Meta; public fun get (Lspace/kscience/dataforge/names/Name;)Lspace/kscience/dataforge/meta/MutableMeta; public final fun getDescriptor ()Lspace/kscience/dataforge/meta/descriptors/MetaDescriptor; @@ -555,6 +558,10 @@ public final class space/kscience/dataforge/meta/SchemeKt { public static final fun copy (Lspace/kscience/dataforge/meta/Scheme;Lspace/kscience/dataforge/meta/SchemeSpec;Lkotlin/jvm/functions/Function1;)Lspace/kscience/dataforge/meta/Scheme; public static synthetic fun copy$default (Lspace/kscience/dataforge/meta/Scheme;Lspace/kscience/dataforge/meta/SchemeSpec;Lkotlin/jvm/functions/Function1;ILjava/lang/Object;)Lspace/kscience/dataforge/meta/Scheme; public static final fun invoke (Lspace/kscience/dataforge/meta/Scheme;Lkotlin/jvm/functions/Function1;)Lspace/kscience/dataforge/meta/Scheme; + public static final fun listOfScheme (Lspace/kscience/dataforge/meta/MutableMeta;Lspace/kscience/dataforge/meta/SchemeSpec;Lspace/kscience/dataforge/names/Name;)Lkotlin/properties/ReadWriteProperty; + public static final fun listOfScheme (Lspace/kscience/dataforge/meta/Scheme;Lspace/kscience/dataforge/meta/SchemeSpec;Lspace/kscience/dataforge/names/Name;)Lkotlin/properties/ReadWriteProperty; + public static synthetic fun listOfScheme$default (Lspace/kscience/dataforge/meta/MutableMeta;Lspace/kscience/dataforge/meta/SchemeSpec;Lspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lkotlin/properties/ReadWriteProperty; + public static synthetic fun listOfScheme$default (Lspace/kscience/dataforge/meta/Scheme;Lspace/kscience/dataforge/meta/SchemeSpec;Lspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lkotlin/properties/ReadWriteProperty; public static final fun scheme (Lspace/kscience/dataforge/meta/MutableMeta;Lspace/kscience/dataforge/meta/SchemeSpec;Lspace/kscience/dataforge/names/Name;)Lkotlin/properties/ReadWriteProperty; public static final fun scheme (Lspace/kscience/dataforge/meta/Scheme;Lspace/kscience/dataforge/meta/SchemeSpec;Lspace/kscience/dataforge/names/Name;)Lkotlin/properties/ReadWriteProperty; public static synthetic fun scheme$default (Lspace/kscience/dataforge/meta/MutableMeta;Lspace/kscience/dataforge/meta/SchemeSpec;Lspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lkotlin/properties/ReadWriteProperty; @@ -837,7 +844,6 @@ public final class space/kscience/dataforge/meta/descriptors/MetaDescriptorBuild public final fun getDefault ()Lspace/kscience/dataforge/meta/Value; public final fun getDescription ()Ljava/lang/String; public final fun getIndexKey ()Ljava/lang/String; - public final fun getInfo ()Ljava/lang/String; public final fun getMultiple ()Z public final fun getValueRestriction ()Lspace/kscience/dataforge/meta/descriptors/ValueRestriction; public final fun getValueTypes ()Ljava/util/List; @@ -847,7 +853,6 @@ public final class space/kscience/dataforge/meta/descriptors/MetaDescriptorBuild public final fun setDefault (Lspace/kscience/dataforge/meta/Value;)V public final fun setDescription (Ljava/lang/String;)V public final fun setIndexKey (Ljava/lang/String;)V - public final fun setInfo (Ljava/lang/String;)V public final fun setMultiple (Z)V public final fun setValueRestriction (Lspace/kscience/dataforge/meta/descriptors/ValueRestriction;)V public final fun setValueTypes (Ljava/util/List;)V diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MutableMeta.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MutableMeta.kt index 231f9e54..d3453c4d 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MutableMeta.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MutableMeta.kt @@ -378,6 +378,11 @@ public fun Meta.asMutableMeta(): MutableMeta = (this as? MutableMeta) ?: toMutab @JsName("newObservableMutableMeta") public fun ObservableMutableMeta(): ObservableMutableMeta = MutableMetaImpl(null) +/** + * Create a pre-filled [ObservableMutableMeta] + */ +public fun ObservableMutableMeta(content: Meta): ObservableMutableMeta = ObservableMutableMeta { update(content) } + /** * Build a [MutableMeta] using given transformation */ diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/Scheme.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/Scheme.kt index 2e9edc1d..c330de99 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/Scheme.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/Scheme.kt @@ -13,32 +13,28 @@ import kotlin.reflect.KProperty1 /** * A base for delegate-based or descriptor-based scheme. [Scheme] has an empty constructor to simplify usage from [MetaSpec]. - * Default item provider and [MetaDescriptor] are optional + * + * @param prototype default values provided by this scheme */ -public open class Scheme : Described, MetaRepr, MutableMetaProvider, Configurable { +public open class Scheme( + private var prototype: Meta? = null, + descriptor: MetaDescriptor? = null +) : Described, MetaRepr, MutableMetaProvider, Configurable { /** * Meta to be mutated by this scheme */ - private var target: MutableMeta? = null - get() { - // automatic initialization of target if it is missing - if (field == null) { - field = MutableMeta() - } - return field - } + internal var target: MutableMeta = MutableMeta() /** - * Default values provided by this scheme + * A descriptor of this scheme */ - private var prototype: Meta? = null + final override var descriptor: MetaDescriptor? = descriptor + private set + final override val meta: ObservableMutableMeta = SchemeMeta(Name.EMPTY) - final override var descriptor: MetaDescriptor? = null - private set - /** * This method must be called before the scheme could be used */ @@ -90,7 +86,7 @@ public open class Scheme : Described, MetaRepr, MutableMetaProvider, Configurabl ?: descriptor?.get(pathName)?.defaultValue set(value) { val oldValue = target[pathName]?.value - target!![pathName] = value + target[pathName] = value if (oldValue != value) { invalidate(Name.EMPTY) } @@ -126,7 +122,7 @@ public open class Scheme : Described, MetaRepr, MutableMetaProvider, Configurabl override fun hashCode(): Int = Meta.hashCode(this) override fun set(name: Name, node: Meta?) { - target!![name] = node + target[name] = node invalidate(name) } @@ -176,15 +172,19 @@ public open class SchemeSpec( it.initialize(MutableMeta(), source, descriptor) } + /** + * Write changes made to the [Scheme] to target [MutableMeta]. If the empty [Scheme] contains any data it is copied to the target. + */ public fun write(target: MutableMeta): T = empty().also { + target.update(it.meta) it.initialize(target, Meta.EMPTY, descriptor) } /** - * Generate an empty object + * Generate a blank object. The object could contain some elements if they are defined in a constructor */ public fun empty(): T = builder().also { - it.initialize(MutableMeta(), Meta.EMPTY, descriptor) + it.initialize(MutableMeta(), it.target, descriptor) } override fun convert(obj: T): Meta = obj.meta @@ -269,7 +269,6 @@ public fun Scheme.schemeOrNull( * If children are mutable, the changes in list elements are reflected on them. * The list is a snapshot of children state, so change in structure is not reflected on its composition. */ -@DFExperimental public fun MutableMeta.listOfScheme( spec: SchemeSpec, key: Name? = null, @@ -286,7 +285,6 @@ public fun MutableMeta.listOfScheme( } -@DFExperimental public fun Scheme.listOfScheme( spec: SchemeSpec, key: Name? = null, diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/descriptors/MetaDescriptorBuilder.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/descriptors/MetaDescriptorBuilder.kt index 5d4d81ad..2291e3d3 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/descriptors/MetaDescriptorBuilder.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/descriptors/MetaDescriptorBuilder.kt @@ -11,9 +11,6 @@ import kotlin.collections.set public class MetaDescriptorBuilder @PublishedApi internal constructor() { public var description: String? = null - @Deprecated("Replace by description", ReplaceWith("description")) - public var info: String? by ::description - public var children: MutableMap = linkedMapOf() public var multiple: Boolean = false public var valueRestriction: ValueRestriction = ValueRestriction.NONE diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/descriptors/schemeDescriptor.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/descriptors/schemeDescriptor.kt index a2a77182..c7b73508 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/descriptors/schemeDescriptor.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/descriptors/schemeDescriptor.kt @@ -3,11 +3,12 @@ package space.kscience.dataforge.meta.descriptors import space.kscience.dataforge.meta.Scheme import space.kscience.dataforge.meta.SchemeSpec import space.kscience.dataforge.meta.ValueType -import space.kscience.dataforge.misc.DFExperimental import kotlin.reflect.KProperty1 import kotlin.reflect.typeOf -@DFExperimental +/** + * Add a value item to a [MetaDescriptor] inferring some of its properties from the type + */ public inline fun MetaDescriptorBuilder.value( property: KProperty1, noinline block: MetaDescriptorBuilder.() -> Unit = {}, @@ -39,7 +40,9 @@ public inline fun MetaDescriptorBuilder.value( else -> node(property.name, block) } -@DFExperimental +/** + * Add a schem-based branch to a [MetaDescriptor] + */ public inline fun MetaDescriptorBuilder.scheme( property: KProperty1, spec: SchemeSpec, diff --git a/dataforge-meta/src/commonTest/kotlin/space/kscience/dataforge/meta/SchemeTest.kt b/dataforge-meta/src/commonTest/kotlin/space/kscience/dataforge/meta/SchemeTest.kt index bb2736ce..eaa766d8 100644 --- a/dataforge-meta/src/commonTest/kotlin/space/kscience/dataforge/meta/SchemeTest.kt +++ b/dataforge-meta/src/commonTest/kotlin/space/kscience/dataforge/meta/SchemeTest.kt @@ -5,7 +5,18 @@ import kotlin.test.Test import kotlin.test.assertEquals import kotlin.test.assertNotNull -@DFExperimental + +private class SchemeWithInit: Scheme(){ + init { + set("initial", "initialValue") + } + + var initial by string() + companion object: SchemeSpec(::SchemeWithInit) +} + + + class SchemeTest { @Test fun testSchemeWrappingBeforeEdit() { @@ -15,6 +26,7 @@ class SchemeTest { assertEquals(29, config["a"].int) } + @OptIn(DFExperimental::class) @Test fun testSchemeWrappingAfterEdit() { val scheme = TestScheme.empty() @@ -46,4 +58,13 @@ class SchemeTest { scheme.v = ListValue(1.0, 2.0, 3.0) assertNotNull(value) } + + + @Test + fun testSchemeWithInit(){ + val scheme = SchemeWithInit() + assertEquals("initialValue", scheme.initial) + scheme.initial = "none" + assertEquals("none", scheme.initial) + } } \ No newline at end of file -- 2.34.1 From 78641f6f877b1e521e662e6596192e9da214f6e0 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Sun, 4 Feb 2024 13:54:44 +0300 Subject: [PATCH 24/62] Roll-back version --- build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle.kts b/build.gradle.kts index 8be3a7b0..b9349868 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -8,7 +8,7 @@ plugins { allprojects { group = "space.kscience" - version = "0.8.1" + version = "0.8.0" } subprojects { -- 2.34.1 From 0e5a31db9f2b232baed5b475cdd5052f1923cad1 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Tue, 6 Feb 2024 10:21:05 +0300 Subject: [PATCH 25/62] Hot fix for observable meta wrapper --- .../space/kscience/dataforge/meta/ObservableMetaWrapper.kt | 4 ++-- .../space/kscience/dataforge/meta/ObservableMetaTest.kt | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/ObservableMetaWrapper.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/ObservableMetaWrapper.kt index 71e15aa9..abd2deb6 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/ObservableMetaWrapper.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/ObservableMetaWrapper.kt @@ -15,9 +15,9 @@ private class ObservableMetaWrapper( val listeners: MutableSet, ) : ObservableMutableMeta { override val items: Map - get() = root.items.keys.associateWith { + get() = root[nodeName]?.items?.keys?.associateWith { ObservableMetaWrapper(root, nodeName + it, listeners) - } + } ?: emptyMap() override fun get(name: Name): ObservableMutableMeta? = if (root[nodeName + name] == null) { null diff --git a/dataforge-meta/src/commonTest/kotlin/space/kscience/dataforge/meta/ObservableMetaTest.kt b/dataforge-meta/src/commonTest/kotlin/space/kscience/dataforge/meta/ObservableMetaTest.kt index 4681ec12..2d6cc36f 100644 --- a/dataforge-meta/src/commonTest/kotlin/space/kscience/dataforge/meta/ObservableMetaTest.kt +++ b/dataforge-meta/src/commonTest/kotlin/space/kscience/dataforge/meta/ObservableMetaTest.kt @@ -17,6 +17,8 @@ class ObservableMetaTest { } }.asObservable() + println(meta) + assertEquals("scatter", meta["data.type"].string) } -- 2.34.1 From 4197b4bb6165f5847d1d84359a67702894b337a6 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Tue, 6 Feb 2024 14:36:20 +0300 Subject: [PATCH 26/62] Add ordering to name indices --- CHANGELOG.md | 2 ++ build.gradle.kts | 2 +- .../kscience/dataforge/io/EnvelopeParts.kt | 3 +- .../dataforge/meta/MutableMetaDelegate.kt | 3 +- .../space/kscience/dataforge/meta/Scheme.kt | 7 ++--- .../dataforge/names/NameIndexComparator.kt | 30 +++++++++++++++++++ .../dataforge/meta/ObservableMetaTest.kt | 2 -- 7 files changed, 40 insertions(+), 9 deletions(-) create mode 100644 dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/names/NameIndexComparator.kt diff --git a/CHANGELOG.md b/CHANGELOG.md index b9d3b6a9..e5d85a2f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## Unreleased ### Added +- Name index comparator ### Changed @@ -11,6 +12,7 @@ ### Removed ### Fixed +- `listOfScheme` and `listOfConvertable` delegates provides correct items order ### Security diff --git a/build.gradle.kts b/build.gradle.kts index b9349868..d41918ac 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -8,7 +8,7 @@ plugins { allprojects { group = "space.kscience" - version = "0.8.0" + version = "0.8.1-dev-1" } subprojects { diff --git a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/EnvelopeParts.kt b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/EnvelopeParts.kt index 183e7b03..88231899 100644 --- a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/EnvelopeParts.kt +++ b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/EnvelopeParts.kt @@ -11,6 +11,7 @@ import space.kscience.dataforge.io.PartDescriptor.Companion.PARTS_KEY import space.kscience.dataforge.io.PartDescriptor.Companion.SEPARATOR_KEY import space.kscience.dataforge.meta.* import space.kscience.dataforge.names.asName +import space.kscience.dataforge.names.getIndexedList import space.kscience.dataforge.names.plus private class PartDescriptor : Scheme() { @@ -84,7 +85,7 @@ public fun EnvelopeBuilder.envelopes( public fun Envelope.parts(): EnvelopeParts { if (data == null) return emptyList() //TODO add zip folder reader - val parts = meta.getIndexed(PARTS_KEY).values.map { + val parts = meta.getIndexedList(PARTS_KEY).map { PartDescriptor.read(it) } return if (parts.isEmpty()) { diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MutableMetaDelegate.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MutableMetaDelegate.kt index f6e96109..70c4aceb 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MutableMetaDelegate.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MutableMetaDelegate.kt @@ -4,6 +4,7 @@ import space.kscience.dataforge.meta.descriptors.MetaDescriptor import space.kscience.dataforge.misc.DFExperimental import space.kscience.dataforge.names.Name import space.kscience.dataforge.names.asName +import space.kscience.dataforge.names.getIndexedList import kotlin.properties.ReadWriteProperty import kotlin.reflect.KProperty @@ -65,7 +66,7 @@ public fun MutableMeta.listOfConvertable( ): ReadWriteProperty> = object : ReadWriteProperty> { override fun getValue(thisRef: Any?, property: KProperty<*>): List { val name = key ?: property.name.asName() - return getIndexed(name).values.map { converter.read(it) } + return getIndexedList(name).map { converter.read(it) } } override fun setValue(thisRef: Any?, property: KProperty<*>, value: List) { diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/Scheme.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/Scheme.kt index c330de99..96b6df49 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/Scheme.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/Scheme.kt @@ -18,7 +18,7 @@ import kotlin.reflect.KProperty1 */ public open class Scheme( private var prototype: Meta? = null, - descriptor: MetaDescriptor? = null + descriptor: MetaDescriptor? = null, ) : Described, MetaRepr, MutableMetaProvider, Configurable { /** @@ -187,7 +187,7 @@ public open class SchemeSpec( it.initialize(MutableMeta(), it.target, descriptor) } - override fun convert(obj: T): Meta = obj.meta + override fun convert(obj: T): Meta = obj.meta /** * A convenience method to use specifications in builders @@ -197,7 +197,6 @@ public open class SchemeSpec( } - /** * Update a [MutableMeta] using given specification */ @@ -275,7 +274,7 @@ public fun MutableMeta.listOfScheme( ): ReadWriteProperty> = object : ReadWriteProperty> { override fun getValue(thisRef: Any?, property: KProperty<*>): List { val name = key ?: property.name.asName() - return getIndexed(name).values.map { spec.write(it as MutableMeta) } + return getIndexedList(name).map { spec.write(it as MutableMeta) } } override fun setValue(thisRef: Any?, property: KProperty<*>, value: List) { diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/names/NameIndexComparator.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/names/NameIndexComparator.kt new file mode 100644 index 00000000..742f8ebb --- /dev/null +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/names/NameIndexComparator.kt @@ -0,0 +1,30 @@ +package space.kscience.dataforge.names + +import space.kscience.dataforge.meta.Meta +import space.kscience.dataforge.meta.getIndexed + + +/** + * A comparator for indices in a [Name]. If both indices are integers, compare them as integers. + * Null always stays "before" non-null index. + */ +public object NameIndexComparator : Comparator { + override fun compare(a: String?, b: String?): Int { + if (a == b) return 0 + if (a == null) return 1 + if (b == null) return -1 + val aInt = a.toIntOrNull() + val bInt = b.toIntOrNull() + return if (aInt != null && bInt != null) { + aInt.compareTo(bInt) + } else { + a.compareTo(b) + } + } + +} + +public fun Meta.getIndexedList(name: Name): List = getIndexed(name).entries.sortedWith( + //sort by index + compareBy(space.kscience.dataforge.names.NameIndexComparator) { it.key } +).map{it.value} \ No newline at end of file diff --git a/dataforge-meta/src/commonTest/kotlin/space/kscience/dataforge/meta/ObservableMetaTest.kt b/dataforge-meta/src/commonTest/kotlin/space/kscience/dataforge/meta/ObservableMetaTest.kt index 2d6cc36f..4681ec12 100644 --- a/dataforge-meta/src/commonTest/kotlin/space/kscience/dataforge/meta/ObservableMetaTest.kt +++ b/dataforge-meta/src/commonTest/kotlin/space/kscience/dataforge/meta/ObservableMetaTest.kt @@ -17,8 +17,6 @@ class ObservableMetaTest { } }.asObservable() - println(meta) - assertEquals("scatter", meta["data.type"].string) } -- 2.34.1 From e850ca4145535ce8b230e888c303c4346ca1d55d Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Sun, 18 Feb 2024 18:34:26 +0300 Subject: [PATCH 27/62] minor refactoring --- .../dataforge/actions/AbstractAction.kt | 14 ++++---- .../kscience/dataforge/actions/MapAction.kt | 4 +-- .../kscience/dataforge/actions/SplitAction.kt | 4 +-- .../kscience/dataforge/data/NamedData.kt | 2 +- .../kscience/dataforge/meta/MutableMeta.kt | 33 +++++++++++++++---- .../kscience/dataforge/meta/SealedMeta.kt | 4 ++- .../space/kscience/dataforge/meta/MetaTest.kt | 29 ++++++++++++++++ gradle/wrapper/gradle-wrapper.properties | 2 +- 8 files changed, 72 insertions(+), 20 deletions(-) diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/AbstractAction.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/AbstractAction.kt index 7cd1ced5..1802e488 100644 --- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/AbstractAction.kt +++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/AbstractAction.kt @@ -37,21 +37,23 @@ public abstract class AbstractAction( * Update part of the data set using provided data * * @param source the source data tree in case we need several data items to update + * @param meta the metadata used for the whole data tree + * @param updatedData an updated item */ protected open fun DataSink.update( source: DataTree, meta: Meta, - namedData: NamedData, - ){ + updatedData: NamedData, + ) { //by default regenerate the whole data set - generate(source,meta) + generate(source, meta) } @OptIn(DFInternal::class) override fun execute( dataSet: DataTree, meta: Meta, - ): DataTree = if(dataSet.isObservable()) { + ): DataTree = if (dataSet.isObservable()) { MutableDataTree(outputType, dataSet.updatesScope).apply { generate(dataSet, meta) dataSet.updates().onEach { @@ -64,8 +66,8 @@ public abstract class AbstractAction( close() } } - } else{ - DataTree(outputType){ + } else { + DataTree(outputType) { generate(dataSet, meta) } } diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/MapAction.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/MapAction.kt index 1f40ed73..fafb2d55 100644 --- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/MapAction.kt +++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/MapAction.kt @@ -86,8 +86,8 @@ internal class MapAction( data.forEach { mapOne(it.name, it.data, meta) } } - override fun DataSink.update(source: DataTree, meta: Meta, namedData: NamedData) { - mapOne(namedData.name, namedData.data, namedData.meta) + override fun DataSink.update(source: DataTree, meta: Meta, updatedData: NamedData) { + mapOne(updatedData.name, updatedData.data, updatedData.meta) } } diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/SplitAction.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/SplitAction.kt index 057419a7..cacbf1ee 100644 --- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/SplitAction.kt +++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/SplitAction.kt @@ -77,8 +77,8 @@ internal class SplitAction( data.forEach { splitOne(it.name, it.data, meta) } } - override fun DataSink.update(source: DataTree, meta: Meta, namedData: NamedData) { - splitOne(namedData.name, namedData.data, namedData.meta) + override fun DataSink.update(source: DataTree, meta: Meta, updatedData: NamedData) { + splitOne(updatedData.name, updatedData.data, updatedData.meta) } } diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/NamedData.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/NamedData.kt index 63e36a3f..bf65292c 100644 --- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/NamedData.kt +++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/NamedData.kt @@ -10,7 +10,7 @@ public interface NamedData : Named, Data { } public operator fun NamedData<*>.component1(): Name = name -public operator fun NamedData.component2(): Data = data +public operator fun NamedData.component2(): Data = data private class NamedDataImpl( override val name: Name, diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MutableMeta.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MutableMeta.kt index d3453c4d..6e4a5daa 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MutableMeta.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MutableMeta.kt @@ -19,6 +19,10 @@ public annotation class MetaBuilderMarker public interface MutableMetaProvider : MetaProvider, MutableValueProvider { override fun get(name: Name): MutableMeta? public operator fun set(name: Name, node: Meta?) + + /** + * Set value with the given name. Does nothing if value is not changed. + */ override fun setValue(name: Name, value: Value?) } @@ -48,11 +52,13 @@ public interface MutableMeta : Meta, MutableMetaProvider { } override fun setValue(name: Name, value: Value?) { - getOrCreate(name).value = value + if (value != getValue(name)) { + getOrCreate(name).value = value + } } /** - * Get existing node or create a new one + * Get an existing node or create a new one */ public fun getOrCreate(name: Name): MutableMeta @@ -198,10 +204,8 @@ public operator fun MutableMetaProvider.set(key: String, metas: Iterable): /** - * Update existing mutable node with another node. The rules are following: - * * value replaces anything - * * node updates node and replaces anything but node - * * node list updates node list if number of nodes in the list is the same and replaces anything otherwise + * Update the existing mutable node with another node. + * Values that are present in the current provider and are missing in [meta] are kept. */ public fun MutableMetaProvider.update(meta: Meta) { meta.valueSequence().forEach { (name, value) -> @@ -222,7 +226,7 @@ public fun > MutableTypedMeta.edit(name: Name, builde getOrCreate(name).apply(builder) /** - * Set a value at a given [name]. If node does not exist, create it. + * Set a value at a given [name]. If a node does not exist, create it. */ public operator fun > MutableTypedMeta.set(name: Name, value: Value?) { edit(name) { @@ -367,6 +371,21 @@ public fun MutableMeta.append(name: Name, value: Value): Unit = append(name, Met public fun MutableMeta.append(key: String, value: Value): Unit = append(Name.parse(key), value) +/** + * Update all items that exist in the [newMeta] and remove existing items that are missing in [newMeta]. + * This produces the same result as clearing all items and updating blank meta with a [newMeta], but does not + * produce unnecessary invalidation events (if they are supported). + */ +public fun MutableMeta.reset(newMeta: Meta) { + //remove old items + (items.keys - newMeta.items.keys).forEach { + remove(it.asName()) + } + newMeta.items.forEach { (token, item)-> + set(token, item) + } +} + /** * Create a mutable copy of this meta. The copy is created even if the Meta is already mutable */ diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/SealedMeta.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/SealedMeta.kt index e842b990..b3fdf062 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/SealedMeta.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/SealedMeta.kt @@ -76,6 +76,8 @@ internal class MetaBuilder( override fun set(name: Name, node: Meta?) { + //skip setting if value has not changed + if(node == get(name)) return when (name.length) { 0 -> error("Can't set a meta with empty name") 1 -> { @@ -89,7 +91,7 @@ internal class MetaBuilder( } else -> { - getOrCreate(name.first().asName()).set(name.cutFirst(), node) + getOrCreate(name.first().asName())[name.cutFirst()] = node } } } diff --git a/dataforge-meta/src/commonTest/kotlin/space/kscience/dataforge/meta/MetaTest.kt b/dataforge-meta/src/commonTest/kotlin/space/kscience/dataforge/meta/MetaTest.kt index 85db7bd6..78b5cdb4 100644 --- a/dataforge-meta/src/commonTest/kotlin/space/kscience/dataforge/meta/MetaTest.kt +++ b/dataforge-meta/src/commonTest/kotlin/space/kscience/dataforge/meta/MetaTest.kt @@ -61,4 +61,33 @@ class MetaTest { assertEquals(null, indexed["8"]) assertEquals(12, indexed["12"].int) } + + @Test + fun reset() { + val oldMeta = MutableMeta { + "a" put { + "value" put "aValue" + } + "b" put { + "value" put "bValue" + } + "c" put { + "value" put "cValue" + } + } + val newMeta = Meta { + "a" put { + "value" put "aValue" + } + "b" put { + "value" put "bValue" + } + "d" put { + "value" put "dValue" + } + } + oldMeta.reset(newMeta) + println(oldMeta) + assertEquals(setOf("a", "b", "d"), oldMeta.items.keys.map { it.toString() }.toSet()) + } } \ No newline at end of file diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index e411586a..17655d0e 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-8.4-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.6-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists -- 2.34.1 From aa4c745819fd0e216df833f5a703023ec671b1fc Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Sun, 18 Feb 2024 18:43:33 +0300 Subject: [PATCH 28/62] remove unnecessary type limitations in actions --- .../space/kscience/dataforge/actions/AbstractAction.kt | 2 +- .../kotlin/space/kscience/dataforge/actions/MapAction.kt | 4 ++-- .../kotlin/space/kscience/dataforge/actions/ReduceAction.kt | 6 +++--- .../kotlin/space/kscience/dataforge/actions/SplitAction.kt | 6 +++--- .../kotlin/space/kscience/dataforge/data/GroupRule.kt | 4 ++-- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/AbstractAction.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/AbstractAction.kt index 1802e488..00201a90 100644 --- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/AbstractAction.kt +++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/AbstractAction.kt @@ -21,7 +21,7 @@ internal fun MutableMap.removeWhatStartsWith(name: Name) { /** * An action that caches results on-demand and recalculates them on source push */ -public abstract class AbstractAction( +public abstract class AbstractAction( public val outputType: KType, ) : Action { diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/MapAction.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/MapAction.kt index fafb2d55..ffa1d93d 100644 --- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/MapAction.kt +++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/MapAction.kt @@ -50,7 +50,7 @@ public class MapActionBuilder( } @PublishedApi -internal class MapAction( +internal class MapAction( outputType: KType, private val block: MapActionBuilder.() -> Unit, ) : AbstractAction(outputType) { @@ -96,7 +96,7 @@ internal class MapAction( * A one-to-one mapping action */ @DFExperimental -public inline fun Action.Companion.mapping( +public inline fun Action.Companion.mapping( noinline builder: MapActionBuilder.() -> Unit, ): Action = MapAction(typeOf(), builder) diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/ReduceAction.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/ReduceAction.kt index 9440be55..0da9b61c 100644 --- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/ReduceAction.kt +++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/ReduceAction.kt @@ -12,7 +12,7 @@ import kotlin.reflect.KType import kotlin.reflect.typeOf -public class JoinGroup( +public class JoinGroup( public var name: String, internal val set: DataTree, @PublishedApi internal var outputType: KType, @@ -35,7 +35,7 @@ public class JoinGroup( } @DFBuilder -public class ReduceGroupBuilder( +public class ReduceGroupBuilder( public val actionMeta: Meta, private val outputType: KType, ) { @@ -79,7 +79,7 @@ public class ReduceGroupBuilder( } @PublishedApi -internal class ReduceAction( +internal class ReduceAction( outputType: KType, private val action: ReduceGroupBuilder.() -> Unit, ) : AbstractAction(outputType) { diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/SplitAction.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/SplitAction.kt index cacbf1ee..b2937515 100644 --- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/SplitAction.kt +++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/SplitAction.kt @@ -13,9 +13,9 @@ import kotlin.reflect.KType import kotlin.reflect.typeOf -public class SplitBuilder(public val name: Name, public val meta: Meta) { +public class SplitBuilder(public val name: Name, public val meta: Meta) { - public class FragmentRule( + public class FragmentRule( public val name: Name, public var meta: MutableMeta, @PublishedApi internal var outputType: KType, @@ -44,7 +44,7 @@ public class SplitBuilder(public val name: Name, public val me * Action that splits each incoming element into a number of fragments defined in builder */ @PublishedApi -internal class SplitAction( +internal class SplitAction( outputType: KType, private val action: SplitBuilder.() -> Unit, ) : AbstractAction(outputType) { diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/GroupRule.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/GroupRule.kt index 90486d85..ff7c94a8 100644 --- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/GroupRule.kt +++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/GroupRule.kt @@ -20,7 +20,7 @@ import space.kscience.dataforge.meta.string import space.kscience.dataforge.misc.DFInternal public interface GroupRule { - public fun gather(set: DataTree): Map> + public fun gather(set: DataTree): Map> public companion object { /** @@ -37,7 +37,7 @@ public interface GroupRule { defaultTagValue: String, ): GroupRule = object : GroupRule { - override fun gather( + override fun gather( set: DataTree, ): Map> { val map = HashMap>() -- 2.34.1 From f95e278b2db7171a7e1bda6ffafa00d0f633f3f5 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Sun, 18 Feb 2024 20:11:53 +0300 Subject: [PATCH 29/62] Replace `branch` data builder with `putAll` --- CHANGELOG.md | 1 + .../kscience/dataforge/data/DataSource.kt | 8 ++-- .../kscience/dataforge/data/dataBuilders.kt | 38 +++++++++++++------ .../dataforge/data/dataSetBuilderInContext.kt | 4 +- .../dataforge/data/DataTreeBuilderTest.kt | 2 +- .../kscience/dataforge/workspace/Workspace.kt | 2 +- .../dataforge/workspace/taskBuilders.kt | 6 +-- .../dataforge/workspace/FileDataTest.kt | 2 +- 8 files changed, 39 insertions(+), 24 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e5d85a2f..2849fce6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ - Name index comparator ### Changed +- DataSink `branch` is replaced with `putAll` to avoid confusion with DataTree methods ### Deprecated diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSource.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSource.kt index d379d027..40086ca2 100644 --- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSource.kt +++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSource.kt @@ -98,15 +98,15 @@ public val DataTree<*>.meta: Meta? get() = data?.meta /** * Provide subtree if it exists */ -public tailrec fun > GenericDataTree.branch(name: Name): TR? = +public tailrec fun > GenericDataTree.putAll(name: Name): TR? = when (name.length) { 0 -> self 1 -> items[name.first()] - else -> items[name.first()]?.branch(name.cutFirst()) + else -> items[name.first()]?.putAll(name.cutFirst()) } -public fun > GenericDataTree.branch(name: String): TR? = - branch(name.parseAsName()) +public fun > GenericDataTree.putAll(name: String): TR? = + this@branch.putAll(name.parseAsName()) public fun GenericDataTree<*, *>.isEmpty(): Boolean = data == null && items.isEmpty() diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataBuilders.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataBuilders.kt index 8e8b6eaa..bdeb2798 100644 --- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataBuilders.kt +++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataBuilders.kt @@ -16,11 +16,7 @@ public fun DataSink.put(value: NamedData) { put(value.name, value.data) } -public fun DataSink.branch(dataTree: DataTree) { - putAll(dataTree.asSequence()) -} - -public inline fun DataSink.branch( +public inline fun DataSink.putAll( prefix: Name, block: DataSink.() -> Unit, ) { @@ -35,24 +31,42 @@ public inline fun DataSink.branch( } } +@Deprecated("Use putAll", ReplaceWith("putAll(prefix, block)")) +public inline fun DataSink.branch( + prefix: Name, + block: DataSink.() -> Unit, +): Unit = putAll(prefix, block) + + +public inline fun DataSink.putAll( + prefix: String, + block: DataSink.() -> Unit, +): Unit = putAll(prefix.asName(), block) + +@Deprecated("Use putAll", ReplaceWith("putAll(prefix, block)")) public inline fun DataSink.branch( prefix: String, block: DataSink.() -> Unit, -): Unit = branch(prefix.asName(), block) - +): Unit = putAll(prefix, block) public fun DataSink.put(name: String, value: Data) { put(Name.parse(name), value) } -public fun DataSink.branch(name: Name, set: DataTree) { - branch(name) { putAll(set.asSequence()) } +public fun DataSink.putAll(name: Name, tree: DataTree) { + putAll(name) { putAll(tree.asSequence()) } } -public fun DataSink.branch(name: String, set: DataTree) { - branch(Name.parse(name)) { putAll(set.asSequence()) } +@Deprecated("Use putAll", ReplaceWith("putAll(name, tree)")) +public fun DataSink.branch(name: Name, tree: DataTree): Unit = putAll(name,tree) + +public fun DataSink.putAll(name: String, tree: DataTree) { + putAll(Name.parse(name)) { putAll(tree.asSequence()) } } +@Deprecated("Use putAll", ReplaceWith("putAll(name, tree)")) +public fun DataSink.branch(name: String, tree: DataTree): Unit = putAll(name,tree) + /** * Produce lazy [Data] and emit it into the [MutableDataTree] */ @@ -124,7 +138,7 @@ public fun DataSink.watchBranch( name: Name, dataSet: ObservableDataTree, ): Job { - branch(name, dataSet) + putAll(name, dataSet) return dataSet.updates().onEach { put(name + it.name, it.data) }.launchIn(dataSet.updatesScope) diff --git a/dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/dataSetBuilderInContext.kt b/dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/dataSetBuilderInContext.kt index cfccb02b..6055a5fa 100644 --- a/dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/dataSetBuilderInContext.kt +++ b/dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/dataSetBuilderInContext.kt @@ -15,7 +15,7 @@ public infix fun String.put(data: Data): Unit = */ context(DataSink) public infix fun String.put(dataSet: DataTree): Unit = - branch(this, dataSet) + this.putAll(this, dataSet) /** * Build and append node @@ -23,5 +23,5 @@ public infix fun String.put(dataSet: DataTree): Unit = context(DataSink) public infix fun String.put( block: DataSink.() -> Unit, -): Unit = branch(Name.parse(this), block) +): Unit = this.putAll(Name.parse(this), block) diff --git a/dataforge-data/src/jvmTest/kotlin/space/kscience/dataforge/data/DataTreeBuilderTest.kt b/dataforge-data/src/jvmTest/kotlin/space/kscience/dataforge/data/DataTreeBuilderTest.kt index 95b7a7bd..a4e732dc 100644 --- a/dataforge-data/src/jvmTest/kotlin/space/kscience/dataforge/data/DataTreeBuilderTest.kt +++ b/dataforge-data/src/jvmTest/kotlin/space/kscience/dataforge/data/DataTreeBuilderTest.kt @@ -43,7 +43,7 @@ internal class DataTreeBuilderTest { wrap("b", "b") } wrap("root", "root") - putAll(updateData) + this.putAll(updateData) } assertEquals("a", node["update.a"]?.await()) diff --git a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/Workspace.kt b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/Workspace.kt index f3ea322c..e4e315fd 100644 --- a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/Workspace.kt +++ b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/Workspace.kt @@ -11,7 +11,7 @@ import space.kscience.dataforge.provider.Provider import kotlin.coroutines.CoroutineContext -public fun interface DataSelector { +public fun interface DataSelector { public suspend fun select(workspace: Workspace, meta: Meta): DataTree } diff --git a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/taskBuilders.kt b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/taskBuilders.kt index 1900ff23..35fccc99 100644 --- a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/taskBuilders.kt +++ b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/taskBuilders.kt @@ -3,8 +3,8 @@ package space.kscience.dataforge.workspace import space.kscience.dataforge.actions.Action import space.kscience.dataforge.context.PluginFactory import space.kscience.dataforge.data.DataTree -import space.kscience.dataforge.data.branch import space.kscience.dataforge.data.forEach +import space.kscience.dataforge.data.putAll import space.kscience.dataforge.data.transform import space.kscience.dataforge.meta.* import space.kscience.dataforge.misc.DFExperimental @@ -101,7 +101,7 @@ public suspend inline fun TaskResultBuilder.transformEach( * Set given [dataSet] as a task result. */ public fun TaskResultBuilder.result(dataSet: DataTree) { - branch(dataSet) + this.putAll(dataSet) } /** @@ -113,7 +113,7 @@ public suspend inline fun TaskResultBuilder.actionFrom( action: Action, dependencyMeta: Meta = defaultDependencyMeta, ) { - branch(action.execute(from(selector, dependencyMeta), dependencyMeta)) + this.putAll(action.execute(from(selector, dependencyMeta), dependencyMeta)) } diff --git a/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/FileDataTest.kt b/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/FileDataTest.kt index 10a1c268..a06f24a5 100644 --- a/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/FileDataTest.kt +++ b/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/FileDataTest.kt @@ -23,7 +23,7 @@ import kotlin.test.assertEquals class FileDataTest { val dataNode = DataTree { - branch("dir") { + putAll("dir") { wrap("a", "Some string") { "content" put "Some string" } -- 2.34.1 From aa88f05688199c51a4aa725e7f2123844fe391cb Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Mon, 19 Feb 2024 11:41:27 +0300 Subject: [PATCH 30/62] Replace `branch` data builder with `putAll` --- .../kotlin/space/kscience/dataforge/data/DataSource.kt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSource.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSource.kt index 40086ca2..d379d027 100644 --- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSource.kt +++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSource.kt @@ -98,15 +98,15 @@ public val DataTree<*>.meta: Meta? get() = data?.meta /** * Provide subtree if it exists */ -public tailrec fun > GenericDataTree.putAll(name: Name): TR? = +public tailrec fun > GenericDataTree.branch(name: Name): TR? = when (name.length) { 0 -> self 1 -> items[name.first()] - else -> items[name.first()]?.putAll(name.cutFirst()) + else -> items[name.first()]?.branch(name.cutFirst()) } -public fun > GenericDataTree.putAll(name: String): TR? = - this@branch.putAll(name.parseAsName()) +public fun > GenericDataTree.branch(name: String): TR? = + branch(name.parseAsName()) public fun GenericDataTree<*, *>.isEmpty(): Boolean = data == null && items.isEmpty() -- 2.34.1 From b575801983ff3afc381276c8249f7b628340596f Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Mon, 19 Feb 2024 11:42:04 +0300 Subject: [PATCH 31/62] Replace `branch` data builder with `putAll` --- .../space/kscience/dataforge/data/dataSetBuilderInContext.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/dataSetBuilderInContext.kt b/dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/dataSetBuilderInContext.kt index 6055a5fa..44602c53 100644 --- a/dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/dataSetBuilderInContext.kt +++ b/dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/dataSetBuilderInContext.kt @@ -15,7 +15,7 @@ public infix fun String.put(data: Data): Unit = */ context(DataSink) public infix fun String.put(dataSet: DataTree): Unit = - this.putAll(this, dataSet) + putAll(this, dataSet) /** * Build and append node @@ -23,5 +23,5 @@ public infix fun String.put(dataSet: DataTree): Unit = context(DataSink) public infix fun String.put( block: DataSink.() -> Unit, -): Unit = this.putAll(Name.parse(this), block) +): Unit = putAll(Name.parse(this), block) -- 2.34.1 From db2da2027d05c24a939eb35b68e7949e988c8760 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Mon, 19 Feb 2024 14:26:33 +0300 Subject: [PATCH 32/62] Replace `branch` data builder with `putAll` --- .../kscience/dataforge/data/dataBuilders.kt | 38 ++++++++----------- .../dataforge/data/DataTreeBuilderTest.kt | 4 +- 2 files changed, 18 insertions(+), 24 deletions(-) diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataBuilders.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataBuilders.kt index bdeb2798..63233c07 100644 --- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataBuilders.kt +++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataBuilders.kt @@ -5,7 +5,6 @@ import kotlinx.coroutines.flow.launchIn import kotlinx.coroutines.flow.onEach import space.kscience.dataforge.meta.Meta import space.kscience.dataforge.meta.MutableMeta -import space.kscience.dataforge.misc.DFExperimental import space.kscience.dataforge.names.Name import space.kscience.dataforge.names.asName import space.kscience.dataforge.names.isEmpty @@ -58,14 +57,14 @@ public fun DataSink.putAll(name: Name, tree: DataTree) { } @Deprecated("Use putAll", ReplaceWith("putAll(name, tree)")) -public fun DataSink.branch(name: Name, tree: DataTree): Unit = putAll(name,tree) +public fun DataSink.branch(name: Name, tree: DataTree): Unit = putAll(name, tree) public fun DataSink.putAll(name: String, tree: DataTree) { putAll(Name.parse(name)) { putAll(tree.asSequence()) } } @Deprecated("Use putAll", ReplaceWith("putAll(name, tree)")) -public fun DataSink.branch(name: String, tree: DataTree): Unit = putAll(name,tree) +public fun DataSink.branch(name: String, tree: DataTree): Unit = putAll(name, tree) /** * Produce lazy [Data] and emit it into the [MutableDataTree] @@ -117,30 +116,25 @@ public fun DataSink.putAll(sequence: Sequence>) { } public fun DataSink.putAll(tree: DataTree) { - this.putAll(tree.asSequence()) -} - - -/** - * Update data with given node data and meta with node meta. - */ -@DFExperimental -public fun MutableDataTree.putAll(source: DataTree) { - source.forEach { - put(it.name, it.data) - } + putAll(tree.asSequence()) } /** * Copy given data set and mirror its changes to this [DataSink] in [this@setAndObserve]. Returns an update [Job] */ +public fun DataSink.putAllAndWatch( + branchName: Name = Name.EMPTY, + dataSet: ObservableDataTree, +): Job { + putAll(branchName, dataSet) + return dataSet.updates().onEach { + put(branchName + it.name, it.data) + }.launchIn(dataSet.updatesScope) +} + + +@Deprecated("Use putAllAndWatch", ReplaceWith("putAllAndWatch(name, dataSet)")) public fun DataSink.watchBranch( name: Name, dataSet: ObservableDataTree, -): Job { - putAll(name, dataSet) - return dataSet.updates().onEach { - put(name + it.name, it.data) - }.launchIn(dataSet.updatesScope) - -} \ No newline at end of file +): Job = putAllAndWatch(name, dataSet) \ No newline at end of file diff --git a/dataforge-data/src/jvmTest/kotlin/space/kscience/dataforge/data/DataTreeBuilderTest.kt b/dataforge-data/src/jvmTest/kotlin/space/kscience/dataforge/data/DataTreeBuilderTest.kt index a4e732dc..561e3aa0 100644 --- a/dataforge-data/src/jvmTest/kotlin/space/kscience/dataforge/data/DataTreeBuilderTest.kt +++ b/dataforge-data/src/jvmTest/kotlin/space/kscience/dataforge/data/DataTreeBuilderTest.kt @@ -43,7 +43,7 @@ internal class DataTreeBuilderTest { wrap("b", "b") } wrap("root", "root") - this.putAll(updateData) + putAll(updateData) } assertEquals("a", node["update.a"]?.await()) @@ -55,7 +55,7 @@ internal class DataTreeBuilderTest { val subNode = MutableDataTree() val rootNode = MutableDataTree { - watchBranch("sub".asName(), subNode) + putAllAndWatch("sub".asName(), subNode) } repeat(10) { -- 2.34.1 From facf7c3c7e423ce22039e7ad2612192b3f3d2f1b Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Mon, 19 Feb 2024 20:10:15 +0300 Subject: [PATCH 33/62] Remove some experimental flags --- .../space/kscience/dataforge/actions/MapAction.kt | 3 +-- .../space/kscience/dataforge/actions/ReduceAction.kt | 10 ++++------ .../space/kscience/dataforge/actions/SplitAction.kt | 5 ++--- .../space/kscience/dataforge/data/dataFilterJvm.kt | 2 +- .../kotlin/space/kscience/dataforge/io/ioMisc.kt | 3 --- .../kotlin/space/kscience/dataforge/io/fileIO.kt | 3 --- .../space/kscience/dataforge/workspace/readFileData.kt | 6 ------ 7 files changed, 8 insertions(+), 24 deletions(-) diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/MapAction.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/MapAction.kt index ffa1d93d..0c56f0dd 100644 --- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/MapAction.kt +++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/MapAction.kt @@ -6,7 +6,6 @@ import space.kscience.dataforge.meta.MutableMeta import space.kscience.dataforge.meta.seal import space.kscience.dataforge.meta.toMutableMeta import space.kscience.dataforge.misc.DFBuilder -import space.kscience.dataforge.misc.DFExperimental import space.kscience.dataforge.misc.DFInternal import space.kscience.dataforge.names.Name import kotlin.reflect.KType @@ -95,7 +94,7 @@ internal class MapAction( /** * A one-to-one mapping action */ -@DFExperimental + public inline fun Action.Companion.mapping( noinline builder: MapActionBuilder.() -> Unit, ): Action = MapAction(typeOf(), builder) diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/ReduceAction.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/ReduceAction.kt index 0da9b61c..9df0db26 100644 --- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/ReduceAction.kt +++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/ReduceAction.kt @@ -4,7 +4,6 @@ import space.kscience.dataforge.data.* import space.kscience.dataforge.meta.Meta import space.kscience.dataforge.meta.MutableMeta import space.kscience.dataforge.misc.DFBuilder -import space.kscience.dataforge.misc.DFExperimental import space.kscience.dataforge.misc.DFInternal import space.kscience.dataforge.names.Name import space.kscience.dataforge.names.parseAsName @@ -22,12 +21,12 @@ public class JoinGroup( public lateinit var result: suspend ActionEnv.(Map>) -> R - internal fun result(outputType: KType, f: suspend ActionEnv.(Map>) -> R1) { + internal fun result(outputType: KType, f: suspend ActionEnv.(Map>) -> R1) { this.outputType = outputType this.result = f; } - public inline fun result(noinline f: suspend ActionEnv.(Map>) -> R1) { + public inline fun result(noinline f: suspend ActionEnv.(Map>) -> R1) { outputType = typeOf() this.result = f; } @@ -67,7 +66,7 @@ public class ReduceGroupBuilder( /** * Apply transformation to the whole node */ - public fun result(resultName: String, f: suspend ActionEnv.(Map>) -> R) { + public fun result(resultName: String, f: suspend ActionEnv.(Map>) -> R) { groupRules += { node -> listOf(JoinGroup(resultName, node, outputType).apply { result(outputType, f) }) } @@ -111,7 +110,6 @@ internal class ReduceAction( /** * A one-to-one mapping action */ -@DFExperimental -public inline fun Action.Companion.reducing( +public inline fun Action.Companion.reducing( noinline builder: ReduceGroupBuilder.() -> Unit, ): Action = ReduceAction(typeOf(), builder) diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/SplitAction.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/SplitAction.kt index b2937515..04fef3ab 100644 --- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/SplitAction.kt +++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/SplitAction.kt @@ -5,7 +5,6 @@ import space.kscience.dataforge.meta.Laminate import space.kscience.dataforge.meta.Meta import space.kscience.dataforge.meta.MutableMeta import space.kscience.dataforge.meta.toMutableMeta -import space.kscience.dataforge.misc.DFExperimental import space.kscience.dataforge.names.Name import space.kscience.dataforge.names.parseAsName import kotlin.collections.set @@ -85,7 +84,7 @@ internal class SplitAction( /** * Action that splits each incoming element into a number of fragments defined in builder */ -@DFExperimental -public inline fun Action.Companion.splitting( + +public inline fun Action.Companion.splitting( noinline builder: SplitBuilder.() -> Unit, ): Action = SplitAction(typeOf(), builder) \ No newline at end of file diff --git a/dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/dataFilterJvm.kt b/dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/dataFilterJvm.kt index 48155595..9498c758 100644 --- a/dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/dataFilterJvm.kt +++ b/dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/dataFilterJvm.kt @@ -84,4 +84,4 @@ public fun ObservableDataTree<*>.filterByType( public inline fun ObservableDataTree<*>.filterByType( scope: CoroutineScope, predicate: DataFilter = DataFilter.EMPTY, -): ObservableDataTree = filterByType(typeOf(),scope,predicate) \ No newline at end of file +): ObservableDataTree = filterByType(typeOf(), scope, predicate) \ No newline at end of file diff --git a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/ioMisc.kt b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/ioMisc.kt index 93fd210a..b5e56dfe 100644 --- a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/ioMisc.kt +++ b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/ioMisc.kt @@ -5,7 +5,6 @@ import kotlinx.io.bytestring.ByteString import kotlinx.io.bytestring.decodeToString import kotlinx.io.bytestring.encodeToByteString import space.kscience.dataforge.meta.Meta -import space.kscience.dataforge.misc.DFExperimental import kotlin.math.min /** @@ -52,7 +51,6 @@ public fun IOPlugin.peekBinaryEnvelopeFormat(binary: Binary): EnvelopeFormat? { /** * A zero-copy read from */ -@DFExperimental public fun IOPlugin.readEnvelope( binary: Binary, readNonEnvelopes: Boolean = false, @@ -62,7 +60,6 @@ public fun IOPlugin.readEnvelope( Envelope(Meta.EMPTY, binary) } else error("Can't infer format for $binary") -@DFExperimental public fun IOPlugin.readEnvelope( string: String, readNonEnvelopes: Boolean = false, diff --git a/dataforge-io/src/jvmMain/kotlin/space/kscience/dataforge/io/fileIO.kt b/dataforge-io/src/jvmMain/kotlin/space/kscience/dataforge/io/fileIO.kt index c15280f3..7df23eb5 100644 --- a/dataforge-io/src/jvmMain/kotlin/space/kscience/dataforge/io/fileIO.kt +++ b/dataforge-io/src/jvmMain/kotlin/space/kscience/dataforge/io/fileIO.kt @@ -77,14 +77,12 @@ public fun Path.rewrite(block: Sink.() -> Unit): Unit { stream.asSink().buffered().use(block) } -@DFExperimental public fun EnvelopeFormat.readFile(path: Path): Envelope = readFrom(path.asBinary()) /** * Resolve IOFormat based on type */ @Suppress("UNCHECKED_CAST") -@DFExperimental public inline fun IOPlugin.resolveIOFormat(): IOFormat? = ioFormatFactories.find { it.type.isSupertypeOf(typeOf()) } as IOFormat? @@ -192,7 +190,6 @@ public fun IOPlugin.peekFileEnvelopeFormat(path: Path): EnvelopeFormat? { * * Return null otherwise. */ -@DFExperimental public fun IOPlugin.readEnvelopeFile( path: Path, readNonEnvelopes: Boolean = false, diff --git a/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/readFileData.kt b/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/readFileData.kt index 1815c3e4..e0a4bdef 100644 --- a/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/readFileData.kt +++ b/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/readFileData.kt @@ -7,8 +7,6 @@ import space.kscience.dataforge.data.StaticData import space.kscience.dataforge.io.* import space.kscience.dataforge.meta.Meta import space.kscience.dataforge.meta.copy -import space.kscience.dataforge.misc.DFExperimental -import space.kscience.dataforge.misc.DFInternal import space.kscience.dataforge.names.Name import space.kscience.dataforge.names.NameToken import space.kscience.dataforge.names.asName @@ -39,7 +37,6 @@ public object FileData { * Read data with supported envelope format and binary format. If the envelope format is null, then read binary directly from file. * The operation is blocking since it must read the meta header. The reading of envelope body is lazy */ -@OptIn(DFExperimental::class) public fun IOPlugin.readFileData( path: Path, ): Data { @@ -127,8 +124,6 @@ public fun DataSink.files( private fun Path.toName() = Name(map { NameToken.parse(it.nameWithoutExtension) }) -@DFInternal -@DFExperimental public fun DataSink.monitorFiles( io: IOPlugin, name: Name, @@ -171,7 +166,6 @@ public fun DataSink.monitorFiles( * @param resources The names of the resources to read. * @param classLoader The class loader to use for loading the resources. By default, it uses the current thread's context class loader. */ -@DFExperimental public fun DataSink.resources( io: IOPlugin, vararg resources: String, -- 2.34.1 From b2593d308e7c485ac12cce345e9496eefdbff907 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Sat, 24 Feb 2024 11:26:28 +0300 Subject: [PATCH 34/62] Add depth-first walk for data tree --- .../space/kscience/dataforge/data/DataSource.kt | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSource.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSource.kt index d379d027..90935fa1 100644 --- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSource.kt +++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSource.kt @@ -93,6 +93,18 @@ public fun DataTree.asSequence( } } +/** + * Walk the data tree depth-first + */ +public fun > TR.walk( + namePrefix: Name = Name.EMPTY, +): Sequence = sequence { + yield(this@walk) + items.forEach { (token, tree) -> + yieldAll(tree.walk(namePrefix + token)) + } +} + public val DataTree<*>.meta: Meta? get() = data?.meta /** -- 2.34.1 From 0e72b4b63ceac633c7fa036571a60514f74f6a5f Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Sat, 24 Feb 2024 15:01:27 +0300 Subject: [PATCH 35/62] Add depth-first walk for data tree --- .../kotlin/space/kscience/dataforge/data/DataSource.kt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSource.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSource.kt index 90935fa1..855a249c 100644 --- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSource.kt +++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSource.kt @@ -96,10 +96,10 @@ public fun DataTree.asSequence( /** * Walk the data tree depth-first */ -public fun > TR.walk( +private fun > TR.walk( namePrefix: Name = Name.EMPTY, -): Sequence = sequence { - yield(this@walk) +): Sequence> = sequence { + yield(namePrefix to this@walk) items.forEach { (token, tree) -> yieldAll(tree.walk(namePrefix + token)) } -- 2.34.1 From db938e1ad8b0b2054a79143aef204d4c8755c039 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Thu, 14 Mar 2024 17:07:32 +0300 Subject: [PATCH 36/62] add custom json encoder to serializerable meta converter --- .../kotlin/space/kscience/dataforge/meta/MetaConverter.kt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MetaConverter.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MetaConverter.kt index 8959ae4a..00074525 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MetaConverter.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MetaConverter.kt @@ -135,16 +135,17 @@ public interface MetaConverter: MetaSpec { @DFExperimental public inline fun serializable( descriptor: MetaDescriptor? = null, + jsonEncoder: Json = Json { ignoreUnknownKeys = true }, ): MetaConverter = object : MetaConverter { private val serializer: KSerializer = serializer() override fun readOrNull(source: Meta): T? { val json = source.toJson(descriptor) - return Json.decodeFromJsonElement(serializer, json) + return jsonEncoder.decodeFromJsonElement(serializer, json) } override fun convert(obj: T): Meta { - val json = Json.encodeToJsonElement(obj) + val json = jsonEncoder.encodeToJsonElement(obj) return json.toMeta(descriptor) } -- 2.34.1 From 9fe3deac336e161a12c6a85f7896ecc8f63018a3 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Tue, 2 Apr 2024 10:01:41 +0300 Subject: [PATCH 37/62] Fix scheme set method --- .../space/kscience/dataforge/meta/Scheme.kt | 2 +- .../kscience/dataforge/meta/SchemeTest.kt | 70 ------------------- 2 files changed, 1 insertion(+), 71 deletions(-) delete mode 100644 dataforge-meta/src/commonTest/kotlin/space/kscience/dataforge/meta/SchemeTest.kt diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/Scheme.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/Scheme.kt index 96b6df49..85b5087a 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/Scheme.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/Scheme.kt @@ -122,7 +122,7 @@ public open class Scheme( override fun hashCode(): Int = Meta.hashCode(this) override fun set(name: Name, node: Meta?) { - target[name] = node + target[pathName + name] = node invalidate(name) } diff --git a/dataforge-meta/src/commonTest/kotlin/space/kscience/dataforge/meta/SchemeTest.kt b/dataforge-meta/src/commonTest/kotlin/space/kscience/dataforge/meta/SchemeTest.kt deleted file mode 100644 index eaa766d8..00000000 --- a/dataforge-meta/src/commonTest/kotlin/space/kscience/dataforge/meta/SchemeTest.kt +++ /dev/null @@ -1,70 +0,0 @@ -package space.kscience.dataforge.meta - -import space.kscience.dataforge.misc.DFExperimental -import kotlin.test.Test -import kotlin.test.assertEquals -import kotlin.test.assertNotNull - - -private class SchemeWithInit: Scheme(){ - init { - set("initial", "initialValue") - } - - var initial by string() - companion object: SchemeSpec(::SchemeWithInit) -} - - - -class SchemeTest { - @Test - fun testSchemeWrappingBeforeEdit() { - val config = MutableMeta() - val scheme = TestScheme.write(config) - scheme.a = 29 - assertEquals(29, config["a"].int) - } - - @OptIn(DFExperimental::class) - @Test - fun testSchemeWrappingAfterEdit() { - val scheme = TestScheme.empty() - scheme.a = 29 - val config = MutableMeta() - scheme.retarget(config) - assertEquals(29, scheme.a) - } - - @Test - fun testSchemeSubscription() { - val scheme = TestScheme.empty() - var flag: Int? = null - scheme.useProperty(TestScheme::a) { a -> - flag = a - } - scheme.a = 2 - assertEquals(2, flag) - } - - @Test - fun testListSubscription(){ - val scheme = TestScheme.empty() - var value: Value? = null - scheme.v = ListValue(0.0,0.0,0.0) - scheme.useProperty(TestScheme::v){ - value = it - } - scheme.v = ListValue(1.0, 2.0, 3.0) - assertNotNull(value) - } - - - @Test - fun testSchemeWithInit(){ - val scheme = SchemeWithInit() - assertEquals("initialValue", scheme.initial) - scheme.initial = "none" - assertEquals("none", scheme.initial) - } -} \ No newline at end of file -- 2.34.1 From 104111f62dce898d45f0682254f8197293fb5e5e Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Tue, 2 Apr 2024 10:03:18 +0300 Subject: [PATCH 38/62] Add byte array value. Refactor exotic values --- .../kscience/dataforge/meta/MetaConverter.kt | 2 +- .../dataforge/meta/MutableMetaDelegate.kt | 11 --- .../space/kscience/dataforge/meta/Value.kt | 2 - .../kscience/dataforge/meta/exoticValues.kt | 80 ++++++++++++++++++ .../dataforge/meta/valueExtensions.kt | 7 -- .../kscience/dataforge/misc/annotations.kt | 9 +- .../dataforge/meta/SpecificationTest.kt | 84 +++++++++++++++++++ .../dataforge/values/DoubleArrayValue.kt | 4 + 8 files changed, 177 insertions(+), 22 deletions(-) create mode 100644 dataforge-meta/src/commonTest/kotlin/space/kscience/dataforge/values/DoubleArrayValue.kt diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MetaConverter.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MetaConverter.kt index 00074525..124f49f5 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MetaConverter.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MetaConverter.kt @@ -135,7 +135,7 @@ public interface MetaConverter: MetaSpec { @DFExperimental public inline fun serializable( descriptor: MetaDescriptor? = null, - jsonEncoder: Json = Json { ignoreUnknownKeys = true }, + jsonEncoder: Json = Json, ): MetaConverter = object : MetaConverter { private val serializer: KSerializer = serializer() diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MutableMetaDelegate.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MutableMetaDelegate.kt index 70c4aceb..e308a63d 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MutableMetaDelegate.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MutableMetaDelegate.kt @@ -197,17 +197,6 @@ public fun MutableMetaProvider.numberList( reader = { it?.list?.map { value -> value.numberOrNull ?: Double.NaN } ?: listOf(*default) }, ) -/* A special delegate for double arrays */ - - -public fun MutableMetaProvider.doubleArray( - vararg default: Double, - key: Name? = null, -): ReadWriteProperty = value( - key, - writer = { DoubleArrayValue(it) }, - reader = { it?.doubleArray ?: doubleArrayOf(*default) }, -) public fun MutableMetaProvider.listValue( key: Name? = null, diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/Value.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/Value.kt index 66e14c86..2ab7b9ee 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/Value.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/Value.kt @@ -256,8 +256,6 @@ public fun ShortArray.asValue(): Value = if (isEmpty()) Null else ListValue(map public fun FloatArray.asValue(): Value = if (isEmpty()) Null else ListValue(map { NumberValue(it) }) -public fun ByteArray.asValue(): Value = if (isEmpty()) Null else ListValue(map { NumberValue(it) }) - public fun > E.asValue(): Value = EnumValue(this) diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/exoticValues.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/exoticValues.kt index 74952053..eb39e985 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/exoticValues.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/exoticValues.kt @@ -1,5 +1,9 @@ package space.kscience.dataforge.meta +import space.kscience.dataforge.names.Name +import kotlin.properties.ReadOnlyProperty +import kotlin.properties.ReadWriteProperty + /** * A value built from string which content and type are parsed on-demand @@ -44,3 +48,79 @@ public class DoubleArrayValue(override val value: DoubleArray) : Value, Iterable } public fun DoubleArray.asValue(): Value = if (isEmpty()) Null else DoubleArrayValue(this) + +public val Value.doubleArray: DoubleArray + get() = if (this is DoubleArrayValue) { + value + } else { + DoubleArray(list.size) { list[it].double } + } + +public val Meta?.doubleArray: DoubleArray? get() = this?.value?.doubleArray + +public fun MetaProvider.doubleArray( + vararg default: Double, + key: Name? = null, +): ReadOnlyProperty = value( + key, + reader = { it?.doubleArray ?: doubleArrayOf(*default) }, +) + +public fun MutableMetaProvider.doubleArray( + vararg default: Double, + key: Name? = null, +): ReadWriteProperty = value( + key, + writer = { DoubleArrayValue(it) }, + reader = { it?.doubleArray ?: doubleArrayOf(*default) }, +) + + +public class ByteArrayValue(override val value: ByteArray) : Value, Iterable { + override val type: ValueType get() = ValueType.LIST + override val list: List get() = value.map { NumberValue(it) } + + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (other !is Value) return false + + return when (other) { + is ByteArrayValue -> value.contentEquals(other.value) + else -> list == other.list + } + } + + override fun hashCode(): Int = value.contentHashCode() + + override fun toString(): String = list.joinToString(prefix = "[", postfix = "]") + + override fun iterator(): Iterator = value.iterator() +} + +public fun ByteArray.asValue(): Value = ByteArrayValue(this) + +public val Value.byteArray: ByteArray + get() = if (this is ByteArrayValue) { + value + } else { + ByteArray(list.size) { list[it].number.toByte() } + } + +public val Meta?.byteArray: ByteArray? get() = this?.value?.byteArray + +public fun MetaProvider.byteArray( + vararg default: Byte, + key: Name? = null, +): ReadOnlyProperty = value( + key, + reader = { it?.byteArray ?: byteArrayOf(*default) }, +) + +public fun MutableMetaProvider.byteArray( + vararg default: Byte, + key: Name? = null, +): ReadWriteProperty = value( + key, + writer = { ByteArrayValue(it) }, + reader = { it?.byteArray ?: byteArrayOf(*default) }, +) \ No newline at end of file diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/valueExtensions.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/valueExtensions.kt index e6b622ff..73464305 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/valueExtensions.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/valueExtensions.kt @@ -31,12 +31,5 @@ public inline fun > Value.enum(): E = if (this is EnumValue< public val Value.stringList: List get() = list.map { it.string } -public val Value.doubleArray: DoubleArray - get() = if (this is DoubleArrayValue) { - value - } else { - DoubleArray(list.size) { list[it].double } - } - public fun Value.toMeta(): Meta = Meta(this) \ No newline at end of file diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/misc/annotations.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/misc/annotations.kt index 3c9d6ac3..29568403 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/misc/annotations.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/misc/annotations.kt @@ -18,4 +18,11 @@ public annotation class DFExperimental */ @RequiresOptIn(level = RequiresOptIn.Level.WARNING) @Retention(AnnotationRetention.BINARY) -public annotation class DFInternal \ No newline at end of file +public annotation class DFInternal + +/** + * Annotation marks methods that explicitly use KType without checking that it corresponds to the type parameter + */ +@RequiresOptIn(level = RequiresOptIn.Level.WARNING) +@Retention(AnnotationRetention.BINARY) +public annotation class UnsafeKType \ No newline at end of file diff --git a/dataforge-meta/src/commonTest/kotlin/space/kscience/dataforge/meta/SpecificationTest.kt b/dataforge-meta/src/commonTest/kotlin/space/kscience/dataforge/meta/SpecificationTest.kt index 8d4d3537..dc9b9d64 100644 --- a/dataforge-meta/src/commonTest/kotlin/space/kscience/dataforge/meta/SpecificationTest.kt +++ b/dataforge-meta/src/commonTest/kotlin/space/kscience/dataforge/meta/SpecificationTest.kt @@ -1,7 +1,17 @@ package space.kscience.dataforge.meta +import space.kscience.dataforge.misc.DFExperimental import kotlin.test.Test import kotlin.test.assertEquals +import kotlin.test.assertNotNull + + +internal class SubScheme : Scheme() { + + var subValue by string() + + companion object : SchemeSpec(::SubScheme) +} internal class TestScheme : Scheme() { var list by numberList(1, 2, 3) @@ -11,9 +21,23 @@ internal class TestScheme : Scheme() { var v by value() + var sub by scheme(SubScheme) + companion object : SchemeSpec(::TestScheme) } +private class SchemeWithInit: Scheme(){ + init { + set("initial", "initialValue") + } + + var initial by string() + + companion object: SchemeSpec(::SchemeWithInit) +} + + + class SpecificationTest { // @Test @@ -71,4 +95,64 @@ class SpecificationTest { assertEquals(22, config["child.a"].int) assertEquals("test", config["child.b"].string) } + + @Test + fun testSchemeWrappingBeforeEdit() { + val config = MutableMeta() + val scheme = TestScheme.write(config) + scheme.a = 29 + assertEquals(29, config["a"].int) + } + + @OptIn(DFExperimental::class) + @Test + fun testSchemeWrappingAfterEdit() { + val scheme = TestScheme.empty() + scheme.a = 29 + val config = MutableMeta() + scheme.retarget(config) + assertEquals(29, scheme.a) + } + + @Test + fun testSchemeSubscription() { + val scheme = TestScheme.empty() + var flag: Int? = null + scheme.useProperty(TestScheme::a) { a -> + flag = a + } + scheme.a = 2 + assertEquals(2, flag) + } + + @Test + fun testListSubscription(){ + val scheme = TestScheme.empty() + var value: Value? = null + scheme.v = ListValue(0.0,0.0,0.0) + scheme.useProperty(TestScheme::v){ + value = it + } + scheme.v = ListValue(1.0, 2.0, 3.0) + assertNotNull(value) + } + + @Test + fun testSubScheme(){ + val scheme = TestScheme.empty() + + scheme.sub.subValue = "aaa" + + assertEquals("aaa",scheme.sub.subValue) + } + + + @Test + fun testSchemeWithInit(){ + val scheme = SchemeWithInit() + assertEquals("initialValue", scheme.initial) + scheme.initial = "none" + assertEquals("none", scheme.initial) + } + } \ No newline at end of file diff --git a/dataforge-meta/src/commonTest/kotlin/space/kscience/dataforge/values/DoubleArrayValue.kt b/dataforge-meta/src/commonTest/kotlin/space/kscience/dataforge/values/DoubleArrayValue.kt new file mode 100644 index 00000000..59ccecae --- /dev/null +++ b/dataforge-meta/src/commonTest/kotlin/space/kscience/dataforge/values/DoubleArrayValue.kt @@ -0,0 +1,4 @@ +package space.kscience.dataforge.values + +class DoubleArrayValue { +} \ No newline at end of file -- 2.34.1 From e7db1cc763210de45306b61642e2d75cf0d56307 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Tue, 2 Apr 2024 10:04:46 +0300 Subject: [PATCH 39/62] Refactor of data trees filtering --- CHANGELOG.md | 4 +- build.gradle.kts | 4 +- .../dataforge/actions/AbstractAction.kt | 46 ++- .../kscience/dataforge/actions/Action.kt | 17 +- .../kscience/dataforge/actions/MapAction.kt | 25 +- .../dataforge/actions/ReduceAction.kt | 8 +- .../kscience/dataforge/actions/SplitAction.kt | 34 ++- .../space/kscience/dataforge/data/Data.kt | 16 +- .../kscience/dataforge/data/DataFilter.kt | 45 ++- .../space/kscience/dataforge/data/DataSink.kt | 157 +++++++++++ .../kscience/dataforge/data/DataSource.kt | 263 ++---------------- .../kscience/dataforge/data/GroupRule.kt | 10 +- .../kscience/dataforge/data/MetaMaskData.kt | 2 +- .../kscience/dataforge/data/NamedData.kt | 28 +- .../kscience/dataforge/data/dataBuilders.kt | 71 ++--- .../kscience/dataforge/data/dataTransform.kt | 17 +- .../dataforge/data/dataTreeBuilder.kt | 112 ++++++++ .../kscience/dataforge/data/ActionsTest.kt | 22 +- .../dataforge/data/DataTreeBuilderTest.kt | 71 +++++ .../kscience/dataforge/data/dataFilterJvm.kt | 45 +-- .../dataforge/data/dataSetBuilderInContext.kt | 4 +- .../dataforge/data/DataTreeBuilderTest.kt | 68 ----- .../kscience/dataforge/workspace/Task.kt | 9 +- .../dataforge/workspace/TaskResult.kt | 8 +- .../kscience/dataforge/workspace/Workspace.kt | 7 +- .../dataforge/workspace/WorkspaceBuilder.kt | 10 +- .../dataforge/workspace/WorkspaceImpl.kt | 4 +- .../dataforge/workspace/envelopeData.kt | 4 +- .../dataforge/workspace/taskBuilders.kt | 2 +- .../dataforge/workspace/FileWorkspaceCache.kt | 3 +- .../workspace/InMemoryWorkspaceCache.kt | 3 +- .../dataforge/workspace/readFileData.kt | 26 +- .../workspace/CachingWorkspaceTest.kt | 4 +- .../workspace/DataPropagationTest.kt | 2 +- .../dataforge/workspace/FileDataTest.kt | 4 +- .../workspace/FileWorkspaceCacheTest.kt | 6 +- .../workspace/SimpleWorkspaceTest.kt | 14 +- 37 files changed, 644 insertions(+), 531 deletions(-) create mode 100644 dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSink.kt create mode 100644 dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataTreeBuilder.kt rename dataforge-data/src/{jvmTest => commonTest}/kotlin/space/kscience/dataforge/data/ActionsTest.kt (65%) create mode 100644 dataforge-data/src/commonTest/kotlin/space/kscience/dataforge/data/DataTreeBuilderTest.kt delete mode 100644 dataforge-data/src/jvmTest/kotlin/space/kscience/dataforge/data/DataTreeBuilderTest.kt diff --git a/CHANGELOG.md b/CHANGELOG.md index 2849fce6..a7c134e6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,7 +13,9 @@ ### Removed ### Fixed -- `listOfScheme` and `listOfConvertable` delegates provides correct items order +- `listOfScheme` and `listOfConvertable` delegates provides correct items order. +- Scheme meta setter works with proper sub-branch. +- ### Security diff --git a/build.gradle.kts b/build.gradle.kts index d41918ac..d4e22686 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -8,7 +8,7 @@ plugins { allprojects { group = "space.kscience" - version = "0.8.1-dev-1" + version = "0.8.1" } subprojects { @@ -30,7 +30,7 @@ ksciencePublish { useApache2Licence() useSPCTeam() } - repository("spc","https://maven.sciprog.center/kscience") + repository("spc", "https://maven.sciprog.center/kscience") sonatype("https://oss.sonatype.org") } diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/AbstractAction.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/AbstractAction.kt index 00201a90..4ed5f8df 100644 --- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/AbstractAction.kt +++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/AbstractAction.kt @@ -1,11 +1,14 @@ package space.kscience.dataforge.actions -import kotlinx.coroutines.flow.launchIn +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.flow.collect import kotlinx.coroutines.flow.onEach -import kotlinx.coroutines.launch -import space.kscience.dataforge.data.* +import space.kscience.dataforge.data.DataSink +import space.kscience.dataforge.data.DataTree +import space.kscience.dataforge.data.DataUpdate +import space.kscience.dataforge.data.launchUpdate import space.kscience.dataforge.meta.Meta -import space.kscience.dataforge.misc.DFInternal +import space.kscience.dataforge.misc.UnsafeKType import space.kscience.dataforge.names.Name import space.kscience.dataforge.names.startsWith import kotlin.reflect.KType @@ -29,7 +32,7 @@ public abstract class AbstractAction( * Generate initial content of the output */ protected abstract fun DataSink.generate( - data: DataTree, + source: DataTree, meta: Meta, ) @@ -40,35 +43,28 @@ public abstract class AbstractAction( * @param meta the metadata used for the whole data tree * @param updatedData an updated item */ - protected open fun DataSink.update( + protected open suspend fun DataSink.update( source: DataTree, meta: Meta, - updatedData: NamedData, + updatedData: DataUpdate, ) { //by default regenerate the whole data set generate(source, meta) } - @OptIn(DFInternal::class) + @OptIn(UnsafeKType::class) override fun execute( - dataSet: DataTree, + source: DataTree, meta: Meta, - ): DataTree = if (dataSet.isObservable()) { - MutableDataTree(outputType, dataSet.updatesScope).apply { - generate(dataSet, meta) - dataSet.updates().onEach { - update(dataSet, meta, it) - }.launchIn(updatesScope) - - //close updates when the source is closed - updatesScope.launch { - dataSet.awaitClose() - close() - } - } - } else { - DataTree(outputType) { - generate(dataSet, meta) + updatesScope: CoroutineScope + ): DataTree = DataTree(outputType) { + generate(source, meta) + //propagate updates + launchUpdate(updatesScope) { + source.updates.onEach { update -> + update(source, meta, update) + }.collect() } } } + diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/Action.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/Action.kt index 5ed60db9..80898aa8 100644 --- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/Action.kt +++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/Action.kt @@ -1,5 +1,8 @@ package space.kscience.dataforge.actions +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.DelicateCoroutinesApi +import kotlinx.coroutines.GlobalScope import space.kscience.dataforge.data.DataTree import space.kscience.dataforge.meta.Meta import space.kscience.dataforge.misc.DFExperimental @@ -13,7 +16,7 @@ public fun interface Action { * Transform the data in the node, producing a new node. By default, it is assumed that all calculations are lazy * so not actual computation is started at this moment. */ - public fun execute(dataSet: DataTree, meta: Meta): DataTree + public fun execute(source: DataTree, meta: Meta, updatesScope: CoroutineScope): DataTree public companion object } @@ -21,23 +24,27 @@ public fun interface Action { /** * A convenience method to transform data using given [action] */ +@OptIn(DelicateCoroutinesApi::class) public fun DataTree.transform( action: Action, meta: Meta = Meta.EMPTY, -): DataTree = action.execute(this, meta) + updateScope: CoroutineScope = GlobalScope, +): DataTree = action.execute(this, meta, updateScope) /** * Action composition. The result is terminal if one of its parts is terminal */ -public infix fun Action.then(action: Action): Action = Action { dataSet, meta -> - action.execute(this@then.execute(dataSet, meta), meta) +public infix fun Action.then(action: Action): Action = Action { dataSet, meta, scope -> + action.execute(this@then.execute(dataSet, meta, scope), meta, scope) } @DFExperimental +@OptIn(DelicateCoroutinesApi::class) public operator fun Action.invoke( dataSet: DataTree, meta: Meta = Meta.EMPTY, -): DataTree = execute(dataSet, meta) + updateScope: CoroutineScope = GlobalScope, +): DataTree = execute(dataSet, meta, updateScope) diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/MapAction.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/MapAction.kt index 0c56f0dd..a1ca8e59 100644 --- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/MapAction.kt +++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/MapAction.kt @@ -6,7 +6,7 @@ import space.kscience.dataforge.meta.MutableMeta import space.kscience.dataforge.meta.seal import space.kscience.dataforge.meta.toMutableMeta import space.kscience.dataforge.misc.DFBuilder -import space.kscience.dataforge.misc.DFInternal +import space.kscience.dataforge.misc.UnsafeKType import space.kscience.dataforge.names.Name import kotlin.reflect.KType import kotlin.reflect.typeOf @@ -54,7 +54,12 @@ internal class MapAction( private val block: MapActionBuilder.() -> Unit, ) : AbstractAction(outputType) { - private fun DataSink.mapOne(name: Name, data: Data, meta: Meta) { + private fun DataSink.mapOne(name: Name, data: Data?, meta: Meta) { + //fast return for null data + if (data == null) { + put(name, null) + return + } // Creating a new environment for action using **old** name, old meta and task meta val env = ActionEnv(name, data.meta, meta) @@ -73,7 +78,7 @@ internal class MapAction( //getting new meta val newMeta = builder.meta.seal() - @OptIn(DFInternal::class) + @OptIn(UnsafeKType::class) val newData = Data(builder.outputType, newMeta, dependencies = listOf(data)) { builder.result(env, data.await()) } @@ -81,12 +86,18 @@ internal class MapAction( put(newName, newData) } - override fun DataSink.generate(data: DataTree, meta: Meta) { - data.forEach { mapOne(it.name, it.data, meta) } + override fun DataSink.generate(source: DataTree, meta: Meta) { + source.forEach { mapOne(it.name, it.data, meta) } } - override fun DataSink.update(source: DataTree, meta: Meta, updatedData: NamedData) { - mapOne(updatedData.name, updatedData.data, updatedData.meta) + + + override suspend fun DataSink.update( + source: DataTree, + meta: Meta, + updatedData: DataUpdate, + ) { + mapOne(updatedData.name, updatedData.data, meta) } } diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/ReduceAction.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/ReduceAction.kt index 9df0db26..93278442 100644 --- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/ReduceAction.kt +++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/ReduceAction.kt @@ -4,7 +4,7 @@ import space.kscience.dataforge.data.* import space.kscience.dataforge.meta.Meta import space.kscience.dataforge.meta.MutableMeta import space.kscience.dataforge.misc.DFBuilder -import space.kscience.dataforge.misc.DFInternal +import space.kscience.dataforge.misc.UnsafeKType import space.kscience.dataforge.names.Name import space.kscience.dataforge.names.parseAsName import kotlin.reflect.KType @@ -84,8 +84,8 @@ internal class ReduceAction( ) : AbstractAction(outputType) { //TODO optimize reduction. Currently, the whole action recalculates on push - override fun DataSink.generate(data: DataTree, meta: Meta) { - ReduceGroupBuilder(meta, outputType).apply(action).buildGroups(data).forEach { group -> + override fun DataSink.generate(source: DataTree, meta: Meta) { + ReduceGroupBuilder(meta, outputType).apply(action).buildGroups(source).forEach { group -> val dataFlow: Map> = group.set.asSequence().fold(HashMap()) { acc, value -> acc.apply { acc[value.name] = value.data @@ -97,7 +97,7 @@ internal class ReduceAction( val groupMeta = group.meta val env = ActionEnv(groupName.parseAsName(), groupMeta, meta) - @OptIn(DFInternal::class) val res: Data = dataFlow.reduceToData( + @OptIn(UnsafeKType::class) val res: Data = dataFlow.reduceToData( group.outputType, meta = groupMeta ) { group.result.invoke(env, it) } diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/SplitAction.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/SplitAction.kt index 04fef3ab..2268b0fa 100644 --- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/SplitAction.kt +++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/SplitAction.kt @@ -48,10 +48,10 @@ internal class SplitAction( private val action: SplitBuilder.() -> Unit, ) : AbstractAction(outputType) { - private fun DataSink.splitOne(name: Name, data: Data, meta: Meta) { - val laminate = Laminate(data.meta, meta) + private fun DataSink.splitOne(name: Name, data: Data?, meta: Meta) { + val laminate = Laminate(data?.meta, meta) - val split = SplitBuilder(name, data.meta).apply(action) + val split = SplitBuilder(name, data?.meta ?: Meta.EMPTY).apply(action) // apply individual fragment rules to result @@ -63,21 +63,29 @@ internal class SplitAction( ).apply(rule) //data.map(outputType, meta = env.meta) { env.result(it) }.named(fragmentName) - put( - fragmentName, - @Suppress("OPT_IN_USAGE") Data(outputType, meta = env.meta, dependencies = listOf(data)) { - env.result(data.await()) - } - ) + if (data == null) { + put(fragmentName, null) + } else { + put( + fragmentName, + @Suppress("OPT_IN_USAGE") Data(outputType, meta = env.meta, dependencies = listOf(data)) { + env.result(data.await()) + } + ) + } } } - override fun DataSink.generate(data: DataTree, meta: Meta) { - data.forEach { splitOne(it.name, it.data, meta) } + override fun DataSink.generate(source: DataTree, meta: Meta) { + source.forEach { splitOne(it.name, it.data, meta) } } - override fun DataSink.update(source: DataTree, meta: Meta, updatedData: NamedData) { - splitOne(updatedData.name, updatedData.data, updatedData.meta) + override suspend fun DataSink.update( + source: DataTree, + meta: Meta, + updatedData: DataUpdate, + ) { + splitOne(updatedData.name, updatedData.data, meta) } } diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/Data.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/Data.kt index a0bb58ea..b9946a48 100644 --- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/Data.kt +++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/Data.kt @@ -4,8 +4,8 @@ import kotlinx.coroutines.* import space.kscience.dataforge.meta.Meta import space.kscience.dataforge.meta.MetaRepr import space.kscience.dataforge.meta.isEmpty -import space.kscience.dataforge.misc.DFInternal import space.kscience.dataforge.misc.DfType +import space.kscience.dataforge.misc.UnsafeKType import kotlin.coroutines.CoroutineContext import kotlin.coroutines.EmptyCoroutineContext import kotlin.reflect.KType @@ -41,7 +41,7 @@ public interface Data : Goal, MetaRepr { */ internal val TYPE_OF_NOTHING: KType = typeOf() - public inline fun static( + public inline fun wrapValue( value: T, meta: Meta = Meta.EMPTY, ): Data = StaticData(typeOf(), value, meta) @@ -50,10 +50,10 @@ public interface Data : Goal, MetaRepr { * An empty data containing only meta */ @OptIn(DelicateCoroutinesApi::class) - public fun empty(meta: Meta): Data = object : Data { - override val type: KType = TYPE_OF_NOTHING + public fun buildEmpty(meta: Meta): Data = object : Data { + override val type: KType get() = TYPE_OF_NOTHING override val meta: Meta = meta - override val dependencies: Collection> = emptyList() + override val dependencies: Collection> get() = emptyList() override val deferred: Deferred get() = GlobalScope.async(start = CoroutineStart.LAZY) { error("The Data is empty and could not be computed") @@ -62,6 +62,8 @@ public interface Data : Goal, MetaRepr { override fun async(coroutineScope: CoroutineScope): Deferred = deferred override fun reset() {} } + + public val EMPTY: Data = buildEmpty(Meta.EMPTY) } } @@ -87,7 +89,7 @@ public class StaticData( public inline fun Data(value: T, meta: Meta = Meta.EMPTY): StaticData = StaticData(typeOf(), value, meta) -@DFInternal +@UnsafeKType public fun Data( type: KType, meta: Meta = Meta.EMPTY, @@ -96,7 +98,7 @@ public fun Data( block: suspend () -> T, ): Data = LazyData(type, meta, context, dependencies, block) -@OptIn(DFInternal::class) +@OptIn(UnsafeKType::class) public inline fun Data( meta: Meta = Meta.EMPTY, context: CoroutineContext = EmptyCoroutineContext, diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataFilter.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataFilter.kt index f45570ad..bc66e910 100644 --- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataFilter.kt +++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataFilter.kt @@ -1,28 +1,30 @@ package space.kscience.dataforge.data -import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.filter import space.kscience.dataforge.meta.Meta import space.kscience.dataforge.names.Name +import space.kscience.dataforge.names.NameToken +import space.kscience.dataforge.names.plus import kotlin.reflect.KType public fun interface DataFilter { - public fun accepts(name: Name, meta: Meta, type: KType): Boolean + public fun accepts(name: Name, meta: Meta?, type: KType): Boolean public companion object { public val EMPTY: DataFilter = DataFilter { _, _, _ -> true } } } -public fun DataFilter.accepts(data: NamedData<*>): Boolean = accepts(data.name, data.meta, data.type) -public fun Sequence>.filterData(predicate: DataFilter): Sequence> = filter { data -> +public fun DataFilter.accepts(update: DataUpdate<*>): Boolean = accepts(update.name, update.data?.meta, update.type) + +public fun > Sequence.filterData(predicate: DataFilter): Sequence = filter { data -> predicate.accepts(data) } -public fun Flow>.filterData(predicate: DataFilter): Flow> = filter { data -> +public fun > Flow.filterData(predicate: DataFilter): Flow = filter { data -> predicate.accepts(data) } @@ -41,7 +43,8 @@ public fun DataSource.filterData( public fun ObservableDataSource.filterData( predicate: DataFilter, ): ObservableDataSource = object : ObservableDataSource { - override fun updates(): Flow> = this@filterData.updates().filter { predicate.accepts(it) } + override val updates: Flow> + get() = this@filterData.updates.filter { predicate.accepts(it) } override val dataType: KType get() = this@filterData.dataType @@ -49,14 +52,32 @@ public fun ObservableDataSource.filterData( this@filterData.read(name)?.takeIf { predicate.accepts(name, it.meta, it.type) } } -public fun GenericDataTree.filterData( - predicate: DataFilter, -): DataTree = asSequence().filterData(predicate).toTree(dataType) +internal class FilteredDataTree( + val source: DataTree, + val filter: DataFilter, + val branch: Name, + override val dataType: KType = source.dataType, +) : DataTree { -public fun GenericObservableDataTree.filterData( - scope: CoroutineScope, + override val data: Data? + get() = source[branch].takeIf { + filter.accepts(Name.EMPTY, data?.meta, data?.type ?: dataType) + } + + override val items: Map> + get() = source.branch(branch)?.items + ?.mapValues { FilteredDataTree(source, filter, branch + it.key) } + ?.filter { !it.value.isEmpty() } + ?: emptyMap() + + override val updates: Flow> + get() = source.updates.filter { filter.accepts(it) } +} + + +public fun DataTree.filterData( predicate: DataFilter, -): ObservableDataTree = asSequence().filterData(predicate).toObservableTree(dataType, scope, updates().filterData(predicate)) +): DataTree = FilteredDataTree(this, predicate, Name.EMPTY) ///** diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSink.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSink.kt new file mode 100644 index 00000000..9d860310 --- /dev/null +++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSink.kt @@ -0,0 +1,157 @@ +package space.kscience.dataforge.data + +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Job +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.MutableSharedFlow +import kotlinx.coroutines.flow.mapNotNull +import kotlinx.coroutines.launch +import space.kscience.dataforge.misc.UnsafeKType +import space.kscience.dataforge.names.* +import kotlin.reflect.KType +import kotlin.reflect.typeOf + +public interface DataSink { + /** + * Put data without notification + */ + public fun put(name: Name, data: Data?) + + /** + * Put data and propagate changes downstream + */ + public suspend fun update(name: Name, data: Data?) +} + +/** + * Launch continuous update using + */ +public fun DataSink.launchUpdate( + scope: CoroutineScope, + updater: suspend DataSink.() -> Unit, +): Job = scope.launch { + object : DataSink { + override fun put(name: Name, data: Data?) { + launch { + this@launchUpdate.update(name, data) + } + } + + override suspend fun update(name: Name, data: Data?) { + this@launchUpdate.update(name, data) + } + }.updater() +} + +/** + * A mutable version of [DataTree] + */ +public interface MutableDataTree : DataTree, DataSink { + override var data: Data? + + override val items: Map> + + public fun getOrCreateItem(token: NameToken): MutableDataTree + + public operator fun set(token: NameToken, data: Data?) + + override fun put(name: Name, data: Data?): Unit = set(name, data) +} + +public tailrec operator fun MutableDataTree.set(name: Name, data: Data?): Unit { + when (name.length) { + 0 -> this.data = data + 1 -> set(name.first(), data) + else -> getOrCreateItem(name.first())[name.cutFirst()] = data + } +} + +/** + * Provide a mutable subtree if it exists + */ +public tailrec fun MutableDataTree.branch(name: Name): MutableDataTree? = + when (name.length) { + 0 -> this + 1 -> items[name.first()] + else -> items[name.first()]?.branch(name.cutFirst()) + } + +private class MutableDataTreeRoot( + override val dataType: KType, +) : MutableDataTree { + + override val updates = MutableSharedFlow>() + + inner class MutableDataTreeBranch(val branchName: Name) : MutableDataTree { + + override var data: Data? = null + + override val items = HashMap>() + + override val updates: Flow> = this@MutableDataTreeRoot.updates.mapNotNull { update -> + update.name.removeFirstOrNull(branchName)?.let { + DataUpdate(update.data?.type ?: dataType, it, update.data) + } + } + override val dataType: KType get() = this@MutableDataTreeRoot.dataType + + + override fun getOrCreateItem(token: NameToken): MutableDataTree = + items.getOrPut(token) { MutableDataTreeBranch(branchName + token) } + + + override fun set(token: NameToken, data: Data?) { + val subTree = getOrCreateItem(token) + subTree.data = data + } + + override suspend fun update(name: Name, data: Data?) { + if (name.isEmpty()) { + this.data = data + } else { + getOrCreateItem(name.first()).update(name.cutFirst(), data) + } + this@MutableDataTreeRoot.updates.emit(DataUpdate(data?.type ?: dataType, branchName + name, data)) + } + + } + + + override var data: Data? = null + + override val items = HashMap>() + + override fun getOrCreateItem(token: NameToken): MutableDataTree = items.getOrPut(token) { + MutableDataTreeRoot(dataType) + } + + override fun set(token: NameToken, data: Data?) { + val subTree = getOrCreateItem(token) + subTree.data = data + } + + override suspend fun update(name: Name, data: Data?) { + if (name.isEmpty()) { + this.data = data + } else { + getOrCreateItem(name.first()).update(name.cutFirst(), data) + } + updates.emit(DataUpdate(data?.type ?: dataType, name, data)) + } +} + +/** + * Create a new [MutableDataTree] + */ +@UnsafeKType +public fun MutableDataTree( + type: KType, +): MutableDataTree = MutableDataTreeRoot(type) + +/** + * Create and initialize a observable mutable data tree. + */ +@OptIn(UnsafeKType::class) +public inline fun MutableDataTree( + generator: MutableDataTree.() -> Unit = {}, +): MutableDataTree = MutableDataTree(typeOf()).apply { generator() } \ No newline at end of file diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSource.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSource.kt index 855a249c..a9756be7 100644 --- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSource.kt +++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSource.kt @@ -1,9 +1,8 @@ package space.kscience.dataforge.data -import kotlinx.coroutines.* -import kotlinx.coroutines.flow.* +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.emptyFlow import space.kscience.dataforge.meta.Meta -import space.kscience.dataforge.misc.DFInternal import space.kscience.dataforge.names.* import kotlin.contracts.contract import kotlin.reflect.KType @@ -33,46 +32,42 @@ public interface ObservableDataSource : DataSource { /** * Flow updates made to the data */ - public fun updates(): Flow> + public val updates: Flow> } /** * A tree like structure for data holding */ -public interface GenericDataTree> : DataSource { - public val self: TR +public interface DataTree : ObservableDataSource { public val data: Data? - public val items: Map - + public val items: Map> override fun read(name: Name): Data? = when (name.length) { 0 -> data else -> items[name.first()]?.read(name.cutFirst()) } + /** + * Flow updates made to the data + */ + override val updates: Flow> + public companion object { - private object EmptyDataTree : GenericDataTree { - override val self: EmptyDataTree get() = this + private object EmptyDataTree : + DataTree { override val data: Data? = null override val items: Map = emptyMap() override val dataType: KType = typeOf() override fun read(name: Name): Data? = null - + override val updates: Flow> get() = emptyFlow() } - public val EMPTY: GenericDataTree = EmptyDataTree + public val EMPTY: DataTree = EmptyDataTree } } -public typealias DataTree = GenericDataTree> - -/** - * Return a single data in this tree. Throw error if it is not single. - */ -public fun DataTree.single(): NamedData = asSequence().single() - /** * An alias for easier access to tree values */ @@ -94,11 +89,13 @@ public fun DataTree.asSequence( } /** - * Walk the data tree depth-first + * Walk the data tree depth-first. + * + * @return a [Sequence] of pairs [Name]-[DataTree] for all nodes including the root one. */ -private fun > TR.walk( +public fun DataTree.walk( namePrefix: Name = Name.EMPTY, -): Sequence> = sequence { +): Sequence>> = sequence { yield(namePrefix to this@walk) items.forEach { (token, tree) -> yieldAll(tree.walk(namePrefix + token)) @@ -110,233 +107,25 @@ public val DataTree<*>.meta: Meta? get() = data?.meta /** * Provide subtree if it exists */ -public tailrec fun > GenericDataTree.branch(name: Name): TR? = +public tailrec fun DataTree.branch(name: Name): DataTree? = when (name.length) { - 0 -> self + 0 -> this 1 -> items[name.first()] else -> items[name.first()]?.branch(name.cutFirst()) } -public fun > GenericDataTree.branch(name: String): TR? = +public fun DataTree.branch(name: String): DataTree? = branch(name.parseAsName()) -public fun GenericDataTree<*, *>.isEmpty(): Boolean = data == null && items.isEmpty() - -@PublishedApi -internal class FlatDataTree( - override val dataType: KType, - private val dataSet: Map>, - private val prefix: Name, -) : GenericDataTree> { - override val self: FlatDataTree get() = this - override val data: Data? get() = dataSet[prefix] - override val items: Map> - get() = dataSet.keys - .filter { it.startsWith(prefix) && it.length > prefix.length } - .map { it.tokens[prefix.length] } - .associateWith { FlatDataTree(dataType, dataSet, prefix + it) } - - override fun read(name: Name): Data? = dataSet[prefix + name] -} - -/** - * Represent this flat data map as a [DataTree] without copying it - */ -public inline fun Map>.asTree(): DataTree = FlatDataTree(typeOf(), this, Name.EMPTY) - -internal fun Sequence>.toTree(type: KType): DataTree = - FlatDataTree(type, associate { it.name to it.data }, Name.EMPTY) - -/** - * Collect a sequence of [NamedData] to a [DataTree] - */ -public inline fun Sequence>.toTree(): DataTree = - FlatDataTree(typeOf(), associate { it.name to it.data }, Name.EMPTY) - -public interface GenericObservableDataTree> : - GenericDataTree, ObservableDataSource, AutoCloseable { - - /** - * A scope that is used to propagate updates. When this scope is closed, no new updates could arrive. - */ - public val updatesScope: CoroutineScope - - /** - * Close this data tree updates channel - */ - override fun close() { - updatesScope.cancel() - } - -} - -public typealias ObservableDataTree = GenericObservableDataTree> +public fun DataTree<*>.isEmpty(): Boolean = data == null && items.isEmpty() /** * Check if the [DataTree] is observable */ -public fun DataTree.isObservable(): Boolean { +public fun DataSource.isObservable(): Boolean { contract { - returns(true) implies (this@isObservable is GenericObservableDataTree) + returns(true) implies (this@isObservable is ObservableDataSource) } - return this is GenericObservableDataTree + return this is ObservableDataSource } -/** - * Wait for this data tree to stop spawning updates (updatesScope is closed). - * If this [DataTree] is not observable, return immediately. - */ -public suspend fun DataTree.awaitClose() { - if (isObservable()) { - updatesScope.coroutineContext[Job]?.join() - } -} - -public fun DataTree.updates(): Flow> = - if (this is GenericObservableDataTree) updates() else emptyFlow() - -public fun interface DataSink { - public fun put(name: Name, data: Data?) -} - -@DFInternal -public class DataTreeBuilder(private val type: KType) : DataSink { - private val map = HashMap>() - override fun put(name: Name, data: Data?) { - if (data == null) { - map.remove(name) - } else { - map[name] = data - } - } - - public fun build(): DataTree = FlatDataTree(type, map, Name.EMPTY) -} - -@DFInternal -public inline fun DataTree( - dataType: KType, - generator: DataSink.() -> Unit, -): DataTree = DataTreeBuilder(dataType).apply(generator).build() - -/** - * Create and a data tree. - */ -@OptIn(DFInternal::class) -public inline fun DataTree( - generator: DataSink.() -> Unit, -): DataTree = DataTreeBuilder(typeOf()).apply(generator).build() - -/** - * A mutable version of [GenericDataTree] - */ -public interface MutableDataTree : GenericObservableDataTree>, DataSink { - override var data: Data? - - override val items: Map> - - public fun getOrCreateItem(token: NameToken): MutableDataTree - - public operator fun set(token: NameToken, data: Data?) - - override fun put(name: Name, data: Data?): Unit = set(name, data) -} - -public tailrec operator fun MutableDataTree.set(name: Name, data: Data?): Unit { - when (name.length) { - 0 -> this.data = data - 1 -> set(name.first(), data) - else -> getOrCreateItem(name.first())[name.cutFirst()] = data - } -} - -private class MutableDataTreeImpl( - override val dataType: KType, - override val updatesScope: CoroutineScope, -) : MutableDataTree { - - - private val updates = MutableSharedFlow>() - - private val children = HashMap>() - - override var data: Data? = null - set(value) { - if (!updatesScope.isActive) error("Can't send updates to closed MutableDataTree") - field = value - if (value != null) { - updatesScope.launch { - updates.emit(value.named(Name.EMPTY)) - } - } - } - - override val items: Map> get() = children - - override fun getOrCreateItem(token: NameToken): MutableDataTree = children.getOrPut(token){ - MutableDataTreeImpl(dataType, updatesScope) - } - - override val self: MutableDataTree get() = this - - override fun set(token: NameToken, data: Data?) { - if (!updatesScope.isActive) error("Can't send updates to closed MutableDataTree") - val subTree = getOrCreateItem(token) - subTree.updates().onEach { - updates.emit(it.named(token + it.name)) - }.launchIn(updatesScope) - subTree.data = data - } - - override fun updates(): Flow> = updates -} - -/** - * Create a new [MutableDataTree] - * - * @param parentScope a [CoroutineScope] to control data propagation. By default uses [GlobalScope] - */ -@OptIn(DelicateCoroutinesApi::class) -public fun MutableDataTree( - type: KType, - parentScope: CoroutineScope = GlobalScope, -): MutableDataTree = MutableDataTreeImpl( - type, - CoroutineScope(parentScope.coroutineContext + Job(parentScope.coroutineContext[Job])) -) - -/** - * Create and initialize a observable mutable data tree. - */ -@OptIn(DelicateCoroutinesApi::class) -public inline fun MutableDataTree( - parentScope: CoroutineScope = GlobalScope, - generator: MutableDataTree.() -> Unit = {}, -): MutableDataTree = MutableDataTree(typeOf(), parentScope).apply { generator() } - -//@DFInternal -//public fun ObservableDataTree( -// type: KType, -// scope: CoroutineScope, -// generator: suspend MutableDataTree.() -> Unit = {}, -//): ObservableDataTree = MutableDataTree(type, scope.coroutineContext).apply(generator) - -public inline fun ObservableDataTree( - parentScope: CoroutineScope, - generator: MutableDataTree.() -> Unit = {}, -): ObservableDataTree = MutableDataTree(typeOf(), parentScope).apply(generator) - - -/** - * Collect a [Sequence] into an observable tree with additional [updates] - */ -public fun Sequence>.toObservableTree( - dataType: KType, - parentScope: CoroutineScope, - updates: Flow>, -): ObservableDataTree = MutableDataTree(dataType, parentScope).apply { - this.putAll(this@toObservableTree) - updates.onEach { - put(it.name, it.data) - }.launchIn(updatesScope) -} diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/GroupRule.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/GroupRule.kt index ff7c94a8..6dd3caa9 100644 --- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/GroupRule.kt +++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/GroupRule.kt @@ -17,7 +17,7 @@ package space.kscience.dataforge.data import space.kscience.dataforge.meta.get import space.kscience.dataforge.meta.string -import space.kscience.dataforge.misc.DFInternal +import space.kscience.dataforge.misc.UnsafeKType public interface GroupRule { public fun gather(set: DataTree): Map> @@ -31,7 +31,7 @@ public interface GroupRule { * @param defaultTagValue * @return */ - @OptIn(DFInternal::class) + @OptIn(UnsafeKType::class) public fun byMetaValue( key: String, defaultTagValue: String, @@ -40,15 +40,15 @@ public interface GroupRule { override fun gather( set: DataTree, ): Map> { - val map = HashMap>() + val map = HashMap>() set.forEach { data -> val tagValue: String = data.meta[key]?.string ?: defaultTagValue - map.getOrPut(tagValue) { DataTreeBuilder(set.dataType) }.put(data.name, data.data) + map.getOrPut(tagValue) { MutableDataTree(set.dataType) }.put(data.name, data.data) } - return map.mapValues { it.value.build() } + return map } } } diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/MetaMaskData.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/MetaMaskData.kt index acf2410d..85f0b2f9 100644 --- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/MetaMaskData.kt +++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/MetaMaskData.kt @@ -20,4 +20,4 @@ public fun Data.withMeta(newMeta: Meta): Data = if (this is MetaMaskDa * Create a new [Data] with the same computation, but different meta. The meta is created by applying [block] to * the existing data meta. */ -public inline fun Data.mapMeta(block: MutableMeta.() -> Unit): Data = withMeta(meta.copy(block)) \ No newline at end of file +public inline fun Data.withMeta(block: MutableMeta.() -> Unit): Data = withMeta(meta.copy(block)) \ No newline at end of file diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/NamedData.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/NamedData.kt index bf65292c..9cb4c2d9 100644 --- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/NamedData.kt +++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/NamedData.kt @@ -3,10 +3,30 @@ package space.kscience.dataforge.data import space.kscience.dataforge.meta.isEmpty import space.kscience.dataforge.misc.Named import space.kscience.dataforge.names.Name +import kotlin.reflect.KType -public interface NamedData : Named, Data { +/** + * An interface implementing a data update event. + * + * If [data] is null, then corresponding element should be removed. + */ +public interface DataUpdate : Named { + public val type: KType override val name: Name - public val data: Data + public val data: Data? +} + +public fun DataUpdate(type: KType, name: Name, data: Data?): DataUpdate = object : DataUpdate { + override val type: KType = type + override val name: Name = name + override val data: Data? = data +} + +/** + * A data coupled to a name. + */ +public interface NamedData : DataUpdate, Data { + override val data: Data } public operator fun NamedData<*>.component1(): Name = name @@ -32,4 +52,6 @@ public fun Data.named(name: Name): NamedData = if (this is NamedData) NamedDataImpl(name, this.data) } else { NamedDataImpl(name, this) -} \ No newline at end of file +} + +public fun NamedData(name: Name, data: Data): NamedData = data.named(name) \ No newline at end of file diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataBuilders.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataBuilders.kt index 63233c07..8ec3e361 100644 --- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataBuilders.kt +++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataBuilders.kt @@ -1,5 +1,6 @@ package space.kscience.dataforge.data +import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Job import kotlinx.coroutines.flow.launchIn import kotlinx.coroutines.flow.onEach @@ -22,31 +23,27 @@ public inline fun DataSink.putAll( if (prefix.isEmpty()) { apply(block) } else { - val proxyDataSink = DataSink { nameWithoutPrefix, data -> - this.put(prefix + nameWithoutPrefix, data) + val proxyDataSink = object :DataSink{ + override fun put(name: Name, data: Data?) { + this@putAll.put(prefix + name, data) + } + + override suspend fun update(name: Name, data: Data?) { + this@putAll.update(prefix + name, data) + } + } proxyDataSink.apply(block) } } -@Deprecated("Use putAll", ReplaceWith("putAll(prefix, block)")) -public inline fun DataSink.branch( - prefix: Name, - block: DataSink.() -> Unit, -): Unit = putAll(prefix, block) - public inline fun DataSink.putAll( prefix: String, block: DataSink.() -> Unit, ): Unit = putAll(prefix.asName(), block) -@Deprecated("Use putAll", ReplaceWith("putAll(prefix, block)")) -public inline fun DataSink.branch( - prefix: String, - block: DataSink.() -> Unit, -): Unit = putAll(prefix, block) public fun DataSink.put(name: String, value: Data) { put(Name.parse(name), value) @@ -56,20 +53,15 @@ public fun DataSink.putAll(name: Name, tree: DataTree) { putAll(name) { putAll(tree.asSequence()) } } -@Deprecated("Use putAll", ReplaceWith("putAll(name, tree)")) -public fun DataSink.branch(name: Name, tree: DataTree): Unit = putAll(name, tree) public fun DataSink.putAll(name: String, tree: DataTree) { putAll(Name.parse(name)) { putAll(tree.asSequence()) } } -@Deprecated("Use putAll", ReplaceWith("putAll(name, tree)")) -public fun DataSink.branch(name: String, tree: DataTree): Unit = putAll(name, tree) - /** * Produce lazy [Data] and emit it into the [MutableDataTree] */ -public inline fun DataSink.put( +public inline fun DataSink.putValue( name: String, meta: Meta = Meta.EMPTY, noinline producer: suspend () -> T, @@ -78,7 +70,7 @@ public inline fun DataSink.put( put(name, data) } -public inline fun DataSink.put( +public inline fun DataSink.putValue( name: Name, meta: Meta = Meta.EMPTY, noinline producer: suspend () -> T, @@ -90,24 +82,23 @@ public inline fun DataSink.put( /** * Emit static data with the fixed value */ -public inline fun DataSink.wrap( +public inline fun DataSink.putValue( name: String, - data: T, + value: T, meta: Meta = Meta.EMPTY, -): Unit = put(name, Data.static(data, meta)) +): Unit = put(name, Data.wrapValue(value, meta)) -public inline fun DataSink.wrap( +public inline fun DataSink.putValue( name: Name, - data: T, + value: T, meta: Meta = Meta.EMPTY, -): Unit = put(name, Data.static(data, meta)) +): Unit = put(name, Data.wrapValue(value, meta)) -public inline fun DataSink.wrap( +public inline fun DataSink.putValue( name: String, - data: T, - mutableMeta: MutableMeta.() -> Unit, -): Unit = put(Name.parse(name), Data.static(data, Meta(mutableMeta))) - + value: T, + metaBuilder: MutableMeta.() -> Unit, +): Unit = put(Name.parse(name), Data.wrapValue(value, Meta(metaBuilder))) public fun DataSink.putAll(sequence: Sequence>) { sequence.forEach { @@ -123,18 +114,12 @@ public fun DataSink.putAll(tree: DataTree) { * Copy given data set and mirror its changes to this [DataSink] in [this@setAndObserve]. Returns an update [Job] */ public fun DataSink.putAllAndWatch( + scope: CoroutineScope, branchName: Name = Name.EMPTY, - dataSet: ObservableDataTree, + source: DataTree, ): Job { - putAll(branchName, dataSet) - return dataSet.updates().onEach { + putAll(branchName, source) + return source.updates.onEach { put(branchName + it.name, it.data) - }.launchIn(dataSet.updatesScope) -} - - -@Deprecated("Use putAllAndWatch", ReplaceWith("putAllAndWatch(name, dataSet)")) -public fun DataSink.watchBranch( - name: Name, - dataSet: ObservableDataTree, -): Job = putAllAndWatch(name, dataSet) \ No newline at end of file + }.launchIn(scope) +} \ No newline at end of file diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataTransform.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataTransform.kt index 8c7ce70e..ab54eb3e 100644 --- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataTransform.kt +++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataTransform.kt @@ -1,7 +1,7 @@ package space.kscience.dataforge.data import space.kscience.dataforge.meta.* -import space.kscience.dataforge.misc.DFInternal +import space.kscience.dataforge.misc.UnsafeKType import space.kscience.dataforge.names.Name import space.kscience.dataforge.names.NameToken import kotlin.coroutines.CoroutineContext @@ -68,7 +68,7 @@ internal fun Map<*, Data<*>>.joinMeta(): Meta = Meta { } } -@DFInternal +@UnsafeKType public fun Map>.reduceToData( outputType: KType, meta: Meta = joinMeta(), @@ -103,7 +103,7 @@ public inline fun Map>.reduceToData( //Iterable operations -@DFInternal +@UnsafeKType public inline fun Iterable>.reduceToData( outputType: KType, meta: Meta = joinMeta(), @@ -118,7 +118,7 @@ public inline fun Iterable>.reduceToData( transformation(map { it.awaitWithMeta() }) } -@OptIn(DFInternal::class) +@OptIn(UnsafeKType::class) public inline fun Iterable>.reduceToData( meta: Meta = joinMeta(), coroutineContext: CoroutineContext = EmptyCoroutineContext, @@ -141,7 +141,7 @@ public inline fun Iterable>.foldToData( /** * Transform an [Iterable] of [NamedData] to a single [Data]. */ -@DFInternal +@UnsafeKType public inline fun Iterable>.reduceNamedToData( outputType: KType, meta: Meta = joinMeta(), @@ -156,7 +156,7 @@ public inline fun Iterable>.reduceNamedToData( transformation(map { it.awaitWithMeta() }) } -@OptIn(DFInternal::class) +@OptIn(UnsafeKType::class) public inline fun Iterable>.reduceNamedToData( meta: Meta = joinMeta(), coroutineContext: CoroutineContext = EmptyCoroutineContext, @@ -181,7 +181,8 @@ public inline fun Iterable>.foldNamedToData( //DataSet operations -@DFInternal + +@UnsafeKType public suspend fun DataTree.transform( outputType: KType, metaTransform: MutableMeta.() -> Unit = {}, @@ -198,7 +199,7 @@ public suspend fun DataTree.transform( } } -@OptIn(DFInternal::class) +@OptIn(UnsafeKType::class) public suspend inline fun DataTree.transform( noinline metaTransform: MutableMeta.() -> Unit = {}, coroutineContext: CoroutineContext = EmptyCoroutineContext, diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataTreeBuilder.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataTreeBuilder.kt new file mode 100644 index 00000000..ac0492f0 --- /dev/null +++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataTreeBuilder.kt @@ -0,0 +1,112 @@ +package space.kscience.dataforge.data + +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.MutableSharedFlow +import kotlinx.coroutines.flow.mapNotNull +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import space.kscience.dataforge.misc.UnsafeKType +import space.kscience.dataforge.names.* +import kotlin.reflect.KType +import kotlin.reflect.typeOf + + +private class FlatDataTree( + override val dataType: KType, + private val dataSet: Map>, + private val sourceUpdates: Flow>, + private val prefix: Name, +) : DataTree { + override val data: Data? get() = dataSet[prefix] + override val items: Map> + get() = dataSet.keys + .filter { it.startsWith(prefix) && it.length > prefix.length } + .map { it.tokens[prefix.length] } + .associateWith { FlatDataTree(dataType, dataSet, sourceUpdates, prefix + it) } + + override fun read(name: Name): Data? = dataSet[prefix + name] + + override val updates: Flow> = + sourceUpdates.mapNotNull { update -> + update.name.removeFirstOrNull(prefix)?.let { DataUpdate(dataType, it, update.data) } + } +} + +/** + * A builder for static [DataTree]. + */ +private class DataTreeBuilder( + private val type: KType, + initialData: Map> = emptyMap(), +) : DataSink { + + private val map = HashMap>(initialData) + + private val mutex = Mutex() + + private val updatesFlow = MutableSharedFlow>() + + override fun put(name: Name, data: Data?) { + if (data == null) { + map.remove(name) + } else { + map[name] = data + } + } + + override suspend fun update(name: Name, data: Data?) { + mutex.withLock { + if (data == null) { + map.remove(name) + } else { + map.put(name, data) + } + } + updatesFlow.emit(DataUpdate(data?.type ?: type, name, data)) + } + + public fun build(): DataTree = FlatDataTree(type, map, updatesFlow, Name.EMPTY) +} + +/** + * Create a static [DataTree] + */ +@UnsafeKType +public fun DataTree( + dataType: KType, + generator: DataSink.() -> Unit, +): DataTree = DataTreeBuilder(dataType).apply(generator).build() + +/** + * Create and a data tree. + */ +@OptIn(UnsafeKType::class) +public inline fun DataTree( + noinline generator: DataSink.() -> Unit, +): DataTree = DataTree(typeOf(), generator) + + +/** + * Represent this flat data map as a [DataTree] without copying it + */ +@UnsafeKType +public fun Map>.asTree(type: KType): DataTree = + DataTreeBuilder(type, this).build() + +/** + * Represent this flat data map as a [DataTree] without copying it + */ +@OptIn(UnsafeKType::class) +public inline fun Map>.asTree(): DataTree = asTree(typeOf()) + + +@UnsafeKType +public fun Sequence>.toTree(type: KType): DataTree = + DataTreeBuilder(type, associate { it.name to it.data }).build() + + +/** + * Collect a sequence of [NamedData] to a [DataTree] + */ +@OptIn(UnsafeKType::class) +public inline fun Sequence>.toTree(): DataTree = toTree(typeOf()) diff --git a/dataforge-data/src/jvmTest/kotlin/space/kscience/dataforge/data/ActionsTest.kt b/dataforge-data/src/commonTest/kotlin/space/kscience/dataforge/data/ActionsTest.kt similarity index 65% rename from dataforge-data/src/jvmTest/kotlin/space/kscience/dataforge/data/ActionsTest.kt rename to dataforge-data/src/commonTest/kotlin/space/kscience/dataforge/data/ActionsTest.kt index f6eae12c..ef9b14ab 100644 --- a/dataforge-data/src/jvmTest/kotlin/space/kscience/dataforge/data/ActionsTest.kt +++ b/dataforge-data/src/commonTest/kotlin/space/kscience/dataforge/data/ActionsTest.kt @@ -1,22 +1,24 @@ package space.kscience.dataforge.data -import kotlinx.coroutines.delay -import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.flow.collect +import kotlinx.coroutines.flow.onEach +import kotlinx.coroutines.flow.take import kotlinx.coroutines.test.runTest -import org.junit.jupiter.api.Test import space.kscience.dataforge.actions.Action import space.kscience.dataforge.actions.invoke import space.kscience.dataforge.actions.mapping import space.kscience.dataforge.misc.DFExperimental +import kotlin.test.Test import kotlin.test.assertEquals +import kotlin.time.Duration.Companion.milliseconds @OptIn(DFExperimental::class) internal class ActionsTest { @Test - fun testStaticMapAction() = runTest { + fun testStaticMapAction() = runTest(timeout = 500.milliseconds) { val data: DataTree = DataTree { repeat(10) { - wrap(it.toString(), it) + putValue(it.toString(), it) } } @@ -28,7 +30,7 @@ internal class ActionsTest { } @Test - fun testDynamicMapAction() = runBlocking { + fun testDynamicMapAction() = runTest(timeout = 500.milliseconds) { val source: MutableDataTree = MutableDataTree() val plusOne = Action.mapping { @@ -39,13 +41,9 @@ internal class ActionsTest { repeat(10) { - source.wrap(it.toString(), it) + source.putValue(it.toString(), it) } - - delay(20) - - source.close() - result.awaitClose() + result.updates.take(10).onEach { println(it.name) }.collect() assertEquals(2, result["1"]?.await()) } diff --git a/dataforge-data/src/commonTest/kotlin/space/kscience/dataforge/data/DataTreeBuilderTest.kt b/dataforge-data/src/commonTest/kotlin/space/kscience/dataforge/data/DataTreeBuilderTest.kt new file mode 100644 index 00000000..b5738820 --- /dev/null +++ b/dataforge-data/src/commonTest/kotlin/space/kscience/dataforge/data/DataTreeBuilderTest.kt @@ -0,0 +1,71 @@ +package space.kscience.dataforge.data + +import kotlinx.coroutines.cancel +import kotlinx.coroutines.flow.collect +import kotlinx.coroutines.flow.take +import kotlinx.coroutines.launch +import kotlinx.coroutines.test.runTest +import space.kscience.dataforge.names.asName +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.time.Duration.Companion.milliseconds + + +internal class DataTreeBuilderTest { + @Test + fun testTreeBuild() = runTest(timeout = 500.milliseconds) { + val node = DataTree { + putAll("primary"){ + putValue("a", "a") + putValue("b", "b") + } + putValue("c.d", "c.d") + putValue("c.f", "c.f") + } + assertEquals("a", node["primary.a"]?.await()) + assertEquals("b", node["primary.b"]?.await()) + assertEquals("c.d", node["c.d"]?.await()) + assertEquals("c.f", node["c.f"]?.await()) + + } + + @Test + fun testDataUpdate() = runTest(timeout = 500.milliseconds) { + val updateData = DataTree { + putAll("update") { + put("a", Data.wrapValue("a")) + put("b", Data.wrapValue("b")) + } + } + + val node = DataTree { + putAll("primary") { + putValue("a", "a") + putValue("b", "b") + } + putValue("root", "root") + putAll(updateData) + } + + assertEquals("a", node["update.a"]?.await()) + assertEquals("a", node["primary.a"]?.await()) + } + + @Test + fun testDynamicUpdates() = runTest(timeout = 500.milliseconds) { + launch { + val subNode = MutableDataTree() + + val rootNode = MutableDataTree() { + putAllAndWatch(this@launch, "sub".asName(), subNode) + } + + repeat(10) { + subNode.putValue("value[$it]", it) + } + subNode.updates.take(10).collect() + assertEquals(9, rootNode["sub.value[9]"]?.await()) + cancel() + }.join() + } +} \ No newline at end of file diff --git a/dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/dataFilterJvm.kt b/dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/dataFilterJvm.kt index 9498c758..0cc81f7a 100644 --- a/dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/dataFilterJvm.kt +++ b/dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/dataFilterJvm.kt @@ -1,6 +1,5 @@ package space.kscience.dataforge.data -import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.filter import space.kscience.dataforge.misc.DFInternal @@ -25,32 +24,40 @@ private fun Data<*>.castOrNull(type: KType): Data? = @Suppress("UNCHECKED_CAST") @DFInternal -public fun Sequence>.filterByDataType(type: KType): Sequence> = +public fun Sequence>.filterByDataType(type: KType): Sequence> = filter { it.type.isSubtypeOf(type) } as Sequence> @Suppress("UNCHECKED_CAST") @DFInternal -public fun Flow>.filterByDataType(type: KType): Flow> = +public fun Flow>.filterByDataType(type: KType): Flow> = filter { it.type.isSubtypeOf(type) } as Flow> /** * Select all data matching given type and filters. Does not modify paths * - * @param predicate additional filtering condition based on item name and meta. By default, accepts all + * @param filter additional filtering condition based on item name and meta. By default, accepts all */ +@Suppress("UNCHECKED_CAST") @DFInternal public fun DataTree<*>.filterByType( type: KType, - predicate: DataFilter = DataFilter.EMPTY, -): DataTree = asSequence().filterByDataType(type).filterData(predicate).toTree(type) + branch: Name = Name.EMPTY, + filter: DataFilter = DataFilter.EMPTY, +): DataTree { + val filterWithType = DataFilter { name, meta, dataType -> + filter.accepts(name, meta, dataType) && dataType.isSubtypeOf(type) + } + return FilteredDataTree(this, filterWithType, branch, type) as DataTree +} /** * Select a single datum of the appropriate type */ @OptIn(DFInternal::class) public inline fun DataTree<*>.filterByType( - predicate: DataFilter = DataFilter.EMPTY, -): DataTree = filterByType(typeOf(), predicate) + branch: Name = Name.EMPTY, + filter: DataFilter = DataFilter.EMPTY, +): DataTree = filterByType(typeOf(), branch, filter = filter) /** * Select a single datum if it is present and of given [type] @@ -63,25 +70,3 @@ public inline fun DataTree<*>.getByType(name: Name): NamedData public inline fun DataTree<*>.getByType(name: String): NamedData? = this@getByType.getByType(typeOf(), Name.parse(name)) - -/** - * Select all data matching given type and filters. Does not modify paths - * - * @param predicate additional filtering condition based on item name and meta. By default, accepts all - */ -@DFInternal -public fun ObservableDataTree<*>.filterByType( - type: KType, - scope: CoroutineScope, - predicate: DataFilter = DataFilter.EMPTY, -): ObservableDataTree = asSequence() - .filterByDataType(type) - .filterData(predicate) - .toObservableTree(type, scope, updates().filterByDataType(type).filterData(predicate)) - - -@OptIn(DFInternal::class) -public inline fun ObservableDataTree<*>.filterByType( - scope: CoroutineScope, - predicate: DataFilter = DataFilter.EMPTY, -): ObservableDataTree = filterByType(typeOf(), scope, predicate) \ No newline at end of file diff --git a/dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/dataSetBuilderInContext.kt b/dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/dataSetBuilderInContext.kt index 44602c53..8ce50a22 100644 --- a/dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/dataSetBuilderInContext.kt +++ b/dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/dataSetBuilderInContext.kt @@ -14,14 +14,14 @@ public infix fun String.put(data: Data): Unit = * Append node */ context(DataSink) -public infix fun String.put(dataSet: DataTree): Unit = +public infix fun String.putAll(dataSet: DataTree): Unit = putAll(this, dataSet) /** * Build and append node */ context(DataSink) -public infix fun String.put( +public infix fun String.putAll( block: DataSink.() -> Unit, ): Unit = putAll(Name.parse(this), block) diff --git a/dataforge-data/src/jvmTest/kotlin/space/kscience/dataforge/data/DataTreeBuilderTest.kt b/dataforge-data/src/jvmTest/kotlin/space/kscience/dataforge/data/DataTreeBuilderTest.kt deleted file mode 100644 index 561e3aa0..00000000 --- a/dataforge-data/src/jvmTest/kotlin/space/kscience/dataforge/data/DataTreeBuilderTest.kt +++ /dev/null @@ -1,68 +0,0 @@ -package space.kscience.dataforge.data - -import kotlinx.coroutines.delay -import kotlinx.coroutines.runBlocking -import kotlinx.coroutines.test.runTest -import space.kscience.dataforge.misc.DFExperimental -import space.kscience.dataforge.names.asName -import kotlin.test.Test -import kotlin.test.assertEquals - - -internal class DataTreeBuilderTest { - @Test - fun testTreeBuild() = runTest { - val node = DataTree { - "primary" put { - wrap("a", "a") - wrap("b", "b") - } - wrap("c.d", "c.d") - wrap("c.f", "c.f") - } - assertEquals("a", node["primary.a"]?.await()) - assertEquals("b", node["primary.b"]?.await()) - assertEquals("c.d", node["c.d"]?.await()) - assertEquals("c.f", node["c.f"]?.await()) - - } - - @OptIn(DFExperimental::class) - @Test - fun testDataUpdate() = runTest { - val updateData = DataTree { - "update" put { - "a" put Data.static("a") - "b" put Data.static("b") - } - } - - val node = DataTree { - "primary" put { - wrap("a", "a") - wrap("b", "b") - } - wrap("root", "root") - putAll(updateData) - } - - assertEquals("a", node["update.a"]?.await()) - assertEquals("a", node["primary.a"]?.await()) - } - - @Test - fun testDynamicUpdates() = runBlocking { - val subNode = MutableDataTree() - - val rootNode = MutableDataTree { - putAllAndWatch("sub".asName(), subNode) - } - - repeat(10) { - subNode.wrap("value[$it]", it) - } - - delay(20) - assertEquals(9, rootNode["sub.value[9]"]?.await()) - } -} \ No newline at end of file diff --git a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/Task.kt b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/Task.kt index e636de49..372b119f 100644 --- a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/Task.kt +++ b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/Task.kt @@ -10,6 +10,7 @@ import space.kscience.dataforge.meta.MetaSpec import space.kscience.dataforge.meta.descriptors.Described import space.kscience.dataforge.meta.descriptors.MetaDescriptor import space.kscience.dataforge.misc.DfType +import space.kscience.dataforge.misc.UnsafeKType import space.kscience.dataforge.names.Name import space.kscience.dataforge.workspace.Task.Companion.TYPE import kotlin.reflect.KType @@ -90,7 +91,8 @@ public fun Task( taskMeta: Meta, ): TaskResult { //TODO use safe builder and check for external data on add and detects cycles - val dataset = MutableDataTree(resultType, workspace.context).apply { + @OptIn(UnsafeKType::class) + val dataset = MutableDataTree(resultType).apply { TaskResultBuilder(workspace, taskName, taskMeta, this).apply { withContext(GoalExecutionRestriction() + workspace.goalLogger) { builder() @@ -98,7 +100,6 @@ public fun Task( } } return workspace.wrapResult(dataset, taskName, taskMeta) - } } @@ -117,6 +118,7 @@ public inline fun Task( * @param builder for resulting data set */ + @Suppress("FunctionName") public fun Task( resultType: KType, @@ -132,7 +134,8 @@ public fun Task( ): TaskResult = withContext(GoalExecutionRestriction() + workspace.goalLogger) { //TODO use safe builder and check for external data on add and detects cycles val taskMeta = configuration.toMeta() - val dataset = MutableDataTree(resultType, this).apply { + @OptIn(UnsafeKType::class) + val dataset = MutableDataTree(resultType).apply { TaskResultBuilder(workspace, taskName, taskMeta, this).apply { builder(configuration) } } workspace.wrapResult(dataset, taskName, taskMeta) diff --git a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/TaskResult.kt b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/TaskResult.kt index 7aa94101..d4d4291a 100644 --- a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/TaskResult.kt +++ b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/TaskResult.kt @@ -4,7 +4,7 @@ import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Job import kotlinx.coroutines.joinAll import kotlinx.coroutines.launch -import space.kscience.dataforge.data.ObservableDataTree +import space.kscience.dataforge.data.DataTree import space.kscience.dataforge.data.asSequence import space.kscience.dataforge.data.launch import space.kscience.dataforge.meta.Meta @@ -17,16 +17,16 @@ import space.kscience.dataforge.names.Name * @param taskMeta The configuration of the task that produced the result */ public data class TaskResult( - public val content: ObservableDataTree, + public val content: DataTree, public val workspace: Workspace, public val taskName: Name, public val taskMeta: Meta, -) : ObservableDataTree by content +) : DataTree by content /** * Wrap data into [TaskResult] */ -public fun Workspace.wrapResult(data: ObservableDataTree, taskName: Name, taskMeta: Meta): TaskResult = +public fun Workspace.wrapResult(data: DataTree, taskName: Name, taskMeta: Meta): TaskResult = TaskResult(data, this, taskName, taskMeta) /** diff --git a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/Workspace.kt b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/Workspace.kt index e4e315fd..7247240b 100644 --- a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/Workspace.kt +++ b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/Workspace.kt @@ -2,7 +2,10 @@ package space.kscience.dataforge.workspace import kotlinx.coroutines.CoroutineScope import space.kscience.dataforge.context.ContextAware -import space.kscience.dataforge.data.* +import space.kscience.dataforge.data.Data +import space.kscience.dataforge.data.DataTree +import space.kscience.dataforge.data.asSequence +import space.kscience.dataforge.data.get import space.kscience.dataforge.meta.Meta import space.kscience.dataforge.meta.MutableMeta import space.kscience.dataforge.misc.DfType @@ -26,7 +29,7 @@ public interface Workspace : ContextAware, Provider, CoroutineScope { /** * The whole data node for current workspace */ - public val data: ObservableDataTree<*> + public val data: DataTree<*> /** * All targets associated with the workspace diff --git a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/WorkspaceBuilder.kt b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/WorkspaceBuilder.kt index 4705c3b0..cf263a46 100644 --- a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/WorkspaceBuilder.kt +++ b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/WorkspaceBuilder.kt @@ -1,6 +1,5 @@ package space.kscience.dataforge.workspace -import kotlinx.coroutines.CoroutineScope import space.kscience.dataforge.actions.Action import space.kscience.dataforge.context.Context import space.kscience.dataforge.context.ContextBuilder @@ -12,6 +11,7 @@ import space.kscience.dataforge.meta.* import space.kscience.dataforge.meta.descriptors.MetaDescriptor import space.kscience.dataforge.meta.descriptors.MetaDescriptorBuilder import space.kscience.dataforge.misc.DFBuilder +import space.kscience.dataforge.misc.UnsafeKType import space.kscience.dataforge.names.Name import space.kscience.dataforge.names.asName import kotlin.collections.set @@ -98,19 +98,19 @@ public inline fun TaskContainer.task( public inline fun TaskContainer.action( selector: DataSelector, action: Action, - noinline metaTransform: MutableMeta.()-> Unit = {}, + noinline metaTransform: MutableMeta.() -> Unit = {}, noinline descriptorBuilder: MetaDescriptorBuilder.() -> Unit = {}, ): PropertyDelegateProvider>> = task(MetaDescriptor(descriptorBuilder)) { - result(action.execute(from(selector), taskMeta.copy(metaTransform))) + result(action.execute(from(selector), taskMeta.copy(metaTransform), workspace)) } public class WorkspaceBuilder( private val parentContext: Context = Global, - private val coroutineScope: CoroutineScope = parentContext, ) : TaskContainer { private var context: Context? = null - private val data = MutableDataTree(typeOf(), coroutineScope) + @OptIn(UnsafeKType::class) + private val data = MutableDataTree(typeOf()) private val targets: HashMap = HashMap() private val tasks = HashMap>() private var cache: WorkspaceCache? = null diff --git a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/WorkspaceImpl.kt b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/WorkspaceImpl.kt index 21c5e8c2..94839d62 100644 --- a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/WorkspaceImpl.kt +++ b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/WorkspaceImpl.kt @@ -2,14 +2,14 @@ package space.kscience.dataforge.workspace import space.kscience.dataforge.context.Context import space.kscience.dataforge.context.gather -import space.kscience.dataforge.data.ObservableDataTree +import space.kscience.dataforge.data.DataTree import space.kscience.dataforge.meta.Meta import space.kscience.dataforge.names.Name internal class WorkspaceImpl internal constructor( override val context: Context, - override val data: ObservableDataTree<*>, + override val data: DataTree<*>, override val targets: Map, tasks: Map>, private val postProcess: suspend (TaskResult<*>) -> TaskResult<*>, diff --git a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/envelopeData.kt b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/envelopeData.kt index d54ff510..a74f8f05 100644 --- a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/envelopeData.kt +++ b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/envelopeData.kt @@ -3,14 +3,14 @@ package space.kscience.dataforge.workspace import space.kscience.dataforge.data.Data import space.kscience.dataforge.data.await import space.kscience.dataforge.io.* -import space.kscience.dataforge.misc.DFInternal +import space.kscience.dataforge.misc.UnsafeKType import kotlin.reflect.typeOf /** * Convert an [Envelope] to a data via given format. The actual parsing is done lazily. */ -@OptIn(DFInternal::class) +@OptIn(UnsafeKType::class) public inline fun Envelope.toData(format: IOReader): Data = Data(typeOf(), meta) { data?.readWith(format) ?: error("Can't convert envelope without data to Data") } diff --git a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/taskBuilders.kt b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/taskBuilders.kt index 35fccc99..15565995 100644 --- a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/taskBuilders.kt +++ b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/taskBuilders.kt @@ -113,7 +113,7 @@ public suspend inline fun TaskResultBuilder.actionFrom( action: Action, dependencyMeta: Meta = defaultDependencyMeta, ) { - this.putAll(action.execute(from(selector, dependencyMeta), dependencyMeta)) + putAll(action.execute(from(selector, dependencyMeta), dependencyMeta, workspace)) } diff --git a/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/FileWorkspaceCache.kt b/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/FileWorkspaceCache.kt index a43657f8..da09c095 100644 --- a/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/FileWorkspaceCache.kt +++ b/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/FileWorkspaceCache.kt @@ -1,5 +1,6 @@ package space.kscience.dataforge.workspace +import kotlinx.coroutines.flow.filterIsInstance import kotlinx.coroutines.flow.map import kotlinx.io.* import kotlinx.serialization.ExperimentalSerializationApi @@ -96,7 +97,7 @@ public class FileWorkspaceCache(public val cacheDirectory: Path) : WorkspaceCach val cachedTree = result.asSequence().map { cacheOne(it) } - .toObservableTree(result.dataType, result.workspace, result.updates().map { cacheOne(it) }) + .toTree(result.dataType, result.updates.filterIsInstance>().map { cacheOne(it) }) return result.workspace.wrapResult(cachedTree, result.taskName, result.taskMeta) } diff --git a/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/InMemoryWorkspaceCache.kt b/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/InMemoryWorkspaceCache.kt index a3792231..f3b2ee15 100644 --- a/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/InMemoryWorkspaceCache.kt +++ b/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/InMemoryWorkspaceCache.kt @@ -1,5 +1,6 @@ package space.kscience.dataforge.workspace +import kotlinx.coroutines.flow.filterIsInstance import kotlinx.coroutines.flow.map import space.kscience.dataforge.data.* import space.kscience.dataforge.meta.Meta @@ -31,7 +32,7 @@ public class InMemoryWorkspaceCache : WorkspaceCache { val cachedTree = result.asSequence().map { cacheOne(it) } - .toObservableTree(result.dataType, result.workspace, result.updates().map { cacheOne(it) }) + .toTree(result.dataType, result.updates.filterIsInstance>().map { cacheOne(it) }) return result.workspace.wrapResult(cachedTree, result.taskName, result.taskMeta) } diff --git a/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/readFileData.kt b/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/readFileData.kt index e0a4bdef..37dafab9 100644 --- a/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/readFileData.kt +++ b/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/readFileData.kt @@ -7,14 +7,12 @@ import space.kscience.dataforge.data.StaticData import space.kscience.dataforge.io.* import space.kscience.dataforge.meta.Meta import space.kscience.dataforge.meta.copy +import space.kscience.dataforge.misc.DFExperimental import space.kscience.dataforge.names.Name import space.kscience.dataforge.names.NameToken import space.kscience.dataforge.names.asName import space.kscience.dataforge.names.plus -import java.nio.file.Files -import java.nio.file.Path -import java.nio.file.StandardWatchEventKinds -import java.nio.file.WatchEvent +import java.nio.file.* import java.nio.file.attribute.BasicFileAttributes import java.nio.file.spi.FileSystemProvider import kotlin.io.path.* @@ -166,15 +164,25 @@ public fun DataSink.monitorFiles( * @param resources The names of the resources to read. * @param classLoader The class loader to use for loading the resources. By default, it uses the current thread's context class loader. */ +@DFExperimental public fun DataSink.resources( io: IOPlugin, - vararg resources: String, + resource: String, + vararg otherResources: String, classLoader: ClassLoader = Thread.currentThread().contextClassLoader, ) { - resources.forEach { resource -> - val path = classLoader.getResource(resource)?.toURI()?.toPath() ?: error( - "Resource with name $resource is not resolved" + //create a file system if necessary + val uri = Thread.currentThread().contextClassLoader.getResource("common")!!.toURI() + try { + uri.toPath() + } catch (e: FileSystemNotFoundException) { + FileSystems.newFileSystem(uri, mapOf("create" to "true")) + } + + listOf(resource,*otherResources).forEach { r -> + val path = classLoader.getResource(r)?.toURI()?.toPath() ?: error( + "Resource with name $r is not resolved" ) - files(io, resource.asName(), path) + files(io, r.asName(), path) } } diff --git a/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/CachingWorkspaceTest.kt b/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/CachingWorkspaceTest.kt index e5c2c230..7a6a8202 100644 --- a/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/CachingWorkspaceTest.kt +++ b/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/CachingWorkspaceTest.kt @@ -3,7 +3,7 @@ package space.kscience.dataforge.workspace import kotlinx.coroutines.coroutineScope import kotlinx.coroutines.test.runTest import org.junit.jupiter.api.Test -import space.kscience.dataforge.data.wrap +import space.kscience.dataforge.data.putValue import space.kscience.dataforge.meta.Meta import space.kscience.dataforge.meta.boolean import space.kscience.dataforge.meta.get @@ -22,7 +22,7 @@ internal class CachingWorkspaceTest { data { //statically initialize data repeat(5) { - wrap("myData[$it]", it) + putValue("myData[$it]", it) } } diff --git a/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/DataPropagationTest.kt b/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/DataPropagationTest.kt index d611b1c8..cd38f809 100644 --- a/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/DataPropagationTest.kt +++ b/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/DataPropagationTest.kt @@ -47,7 +47,7 @@ class DataPropagationTest { } data { repeat(100) { - wrap("myData[$it]", it) + putValue("myData[$it]", it) } } } diff --git a/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/FileDataTest.kt b/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/FileDataTest.kt index a06f24a5..5466da76 100644 --- a/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/FileDataTest.kt +++ b/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/FileDataTest.kt @@ -24,11 +24,11 @@ import kotlin.test.assertEquals class FileDataTest { val dataNode = DataTree { putAll("dir") { - wrap("a", "Some string") { + putValue("a", "Some string") { "content" put "Some string" } } - wrap("b", "root data") + putValue("b", "root data") // meta { // "content" put "This is root meta node" // } diff --git a/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/FileWorkspaceCacheTest.kt b/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/FileWorkspaceCacheTest.kt index 0f16b1c8..0cf4f401 100644 --- a/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/FileWorkspaceCacheTest.kt +++ b/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/FileWorkspaceCacheTest.kt @@ -3,11 +3,11 @@ package space.kscience.dataforge.workspace import kotlinx.coroutines.ExperimentalCoroutinesApi import kotlinx.coroutines.test.runTest import org.junit.jupiter.api.Test -import space.kscience.dataforge.data.wrap +import space.kscience.dataforge.data.putValue import space.kscience.dataforge.misc.DFExperimental import java.nio.file.Files -@OptIn(ExperimentalCoroutinesApi::class,DFExperimental::class) +@OptIn(ExperimentalCoroutinesApi::class, DFExperimental::class) class FileWorkspaceCacheTest { @Test @@ -16,7 +16,7 @@ class FileWorkspaceCacheTest { data { //statically initialize data repeat(5) { - wrap("myData[$it]", it) + putValue("myData[$it]", it) } } fileCache(Files.createTempDirectory("dataforge-temporary-cache")) diff --git a/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/SimpleWorkspaceTest.kt b/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/SimpleWorkspaceTest.kt index b49b9d54..837b61bc 100644 --- a/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/SimpleWorkspaceTest.kt +++ b/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/SimpleWorkspaceTest.kt @@ -6,7 +6,6 @@ package space.kscience.dataforge.workspace import kotlinx.coroutines.ExperimentalCoroutinesApi import kotlinx.coroutines.runBlocking import kotlinx.coroutines.test.runTest -import org.junit.jupiter.api.Timeout import space.kscience.dataforge.context.* import space.kscience.dataforge.data.* import space.kscience.dataforge.meta.* @@ -16,6 +15,7 @@ import space.kscience.dataforge.names.plus import kotlin.test.Test import kotlin.test.assertEquals import kotlin.test.assertTrue +import kotlin.time.Duration.Companion.milliseconds /** @@ -62,7 +62,7 @@ internal class SimpleWorkspaceTest { data { //statically initialize data repeat(100) { - wrap("myData[$it]", it) + putValue("myData[$it]", it) } } @@ -148,18 +148,16 @@ internal class SimpleWorkspaceTest { } @Test - @Timeout(1) - fun testWorkspace() = runTest { + fun testWorkspace() = runTest(timeout = 10.milliseconds) { val node = workspace.produce("sum") val res = node.asSequence().single() assertEquals(328350, res.await()) } @Test - @Timeout(1) - fun testMetaPropagation() = runTest { + fun testMetaPropagation() = runTest(timeout = 10.milliseconds) { val node = workspace.produce("sum") { "testFlag" put true } - val res = node.single().await() + val res = node.data!!.await() } @Test @@ -188,7 +186,7 @@ internal class SimpleWorkspaceTest { val node = workspace.produce("filterOne") { "name" put "myData[12]" } - assertEquals(12, node.single().await()) + assertEquals(12, node.data!!.await()) } } \ No newline at end of file -- 2.34.1 From 1c6045324423463e4df2852c2160d80efb3a2b10 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Tue, 16 Apr 2024 15:35:21 +0300 Subject: [PATCH 40/62] Fix NameToken.parse --- CHANGELOG.md | 4 +++- .../kscience/dataforge/names/NameToken.kt | 2 +- .../kscience/dataforge/names/NameTest.kt | 20 +++++++++++++++---- 3 files changed, 20 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a7c134e6..f709b8be 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ ### Added - Name index comparator +- Specialized ByteArrayValue ### Changed - DataSink `branch` is replaced with `putAll` to avoid confusion with DataTree methods @@ -15,7 +16,8 @@ ### Fixed - `listOfScheme` and `listOfConvertable` delegates provides correct items order. - Scheme meta setter works with proper sub-branch. -- +- NameToken.parse improper work with indices + ### Security diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/names/NameToken.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/names/NameToken.kt index 0dc83c57..83752b9a 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/names/NameToken.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/names/NameToken.kt @@ -69,7 +69,7 @@ public class NameToken(public val body: String, public val index: String? = null public fun parse(string: String): NameToken { val body = string.substringBefore('[') val index = string.substringAfter('[', "") - if (index.isNotEmpty() && index.endsWith(']')) error("NameToken with index must end with ']'") + if (index.isNotEmpty() && !index.endsWith(']')) error("NameToken with index must end with ']'") return NameToken(body, index.removeSuffix("]")) } } diff --git a/dataforge-meta/src/commonTest/kotlin/space/kscience/dataforge/names/NameTest.kt b/dataforge-meta/src/commonTest/kotlin/space/kscience/dataforge/names/NameTest.kt index a5bdf3fc..db630487 100644 --- a/dataforge-meta/src/commonTest/kotlin/space/kscience/dataforge/names/NameTest.kt +++ b/dataforge-meta/src/commonTest/kotlin/space/kscience/dataforge/names/NameTest.kt @@ -1,9 +1,6 @@ package space.kscience.dataforge.names -import kotlin.test.Test -import kotlin.test.assertEquals -import kotlin.test.assertFalse -import kotlin.test.assertTrue +import kotlin.test.* class NameTest { @Test @@ -50,4 +47,19 @@ class NameTest { val name = Name.parse("a.b.c") assertEquals("a.b".parseAsName(), name.cutLast()) } + + @Test + fun tokenParseTest(){ + val token1 = NameToken.parse("token[index]") + assertEquals("token", token1.body) + assertEquals("index", token1.index) + + val token2 = NameToken.parse("token-body") + assertEquals("token-body", token2.body) + assertEquals("", token2.index) + + assertFails { + NameToken.parse("token[22") + } + } } \ No newline at end of file -- 2.34.1 From a720b63d70bbe0cc32cfc21c057cce4d62f444c3 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Tue, 16 Apr 2024 15:35:59 +0300 Subject: [PATCH 41/62] Add ByteArrayValue --- build.gradle.kts | 2 +- .../kscience/dataforge/meta/MutableMeta.kt | 4 ++++ .../dataforge/values/DoubleArrayValue.kt | 20 +++++++++++++++++++ 3 files changed, 25 insertions(+), 1 deletion(-) diff --git a/build.gradle.kts b/build.gradle.kts index d4e22686..b2f7e5e9 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -8,7 +8,7 @@ plugins { allprojects { group = "space.kscience" - version = "0.8.1" + version = "0.8.2" } subprojects { diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MutableMeta.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MutableMeta.kt index 6e4a5daa..69e221f5 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MutableMeta.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MutableMeta.kt @@ -128,6 +128,10 @@ public interface MutableMeta : Meta, MutableMetaProvider { setValue(Name.parse(this), array.asValue()) } + public infix fun String.put(array: ByteArray) { + setValue(Name.parse(this), array.asValue()) + } + public infix fun String.put(repr: MetaRepr) { set(Name.parse(this), repr.toMeta()) } diff --git a/dataforge-meta/src/commonTest/kotlin/space/kscience/dataforge/values/DoubleArrayValue.kt b/dataforge-meta/src/commonTest/kotlin/space/kscience/dataforge/values/DoubleArrayValue.kt index 59ccecae..30bc2adc 100644 --- a/dataforge-meta/src/commonTest/kotlin/space/kscience/dataforge/values/DoubleArrayValue.kt +++ b/dataforge-meta/src/commonTest/kotlin/space/kscience/dataforge/values/DoubleArrayValue.kt @@ -1,4 +1,24 @@ package space.kscience.dataforge.values +import space.kscience.dataforge.meta.DoubleArrayValue +import space.kscience.dataforge.meta.Meta +import space.kscience.dataforge.meta.doubleArray +import space.kscience.dataforge.meta.get +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertTrue + class DoubleArrayValue { + @Test + fun doubleArrayWriteRead(){ + val meta = Meta{ + "doubleArray" put doubleArrayOf(1.0,2.0,3.0) + } + + assertTrue { + meta["doubleArray"]?.value is DoubleArrayValue + } + + assertEquals(2.0, meta["doubleArray"].doubleArray?.get(1)) + } } \ No newline at end of file -- 2.34.1 From 28f4beb3485cf07df493347cb7a317e1217dfcc4 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Tue, 23 Apr 2024 18:59:12 +0300 Subject: [PATCH 42/62] Fix all tests --- CHANGELOG.md | 4 +- build.gradle.kts | 2 +- dataforge-context/api/dataforge-context.api | 2 - .../descriptors/reflectiveDescriptors.kt | 22 +- .../descriptors/TestAutoDescriptors.kt | 13 +- .../kscience/dataforge/actions/MapAction.kt | 6 +- .../kscience/dataforge/data/DataFilter.kt | 2 +- .../space/kscience/dataforge/data/DataSink.kt | 10 +- .../kscience/dataforge/data/dataBuilders.kt | 23 +- .../kscience/dataforge/data/ActionsTest.kt | 2 +- .../dataforge/data/DataTreeBuilderTest.kt | 32 +-- .../space/kscience/dataforge/io/IOPlugin.kt | 6 +- dataforge-meta/api/dataforge-meta.api | 246 ++++++++++-------- .../kscience/dataforge/meta/MetaDelegate.kt | 116 +++++---- .../dataforge/meta/MutableMetaDelegate.kt | 129 +++++---- .../dataforge/workspace/CachingAction.kt | 19 ++ .../dataforge/workspace/FileWorkspaceCache.kt | 23 +- .../workspace/InMemoryWorkspaceCache.kt | 17 +- .../workspace/SimpleWorkspaceTest.kt | 8 +- 19 files changed, 409 insertions(+), 273 deletions(-) create mode 100644 dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/CachingAction.kt diff --git a/CHANGELOG.md b/CHANGELOG.md index f709b8be..a56e56e6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ ### Changed - DataSink `branch` is replaced with `putAll` to avoid confusion with DataTree methods +- Meta delegate now uses a specific class that has a descriptor ### Deprecated @@ -16,7 +17,8 @@ ### Fixed - `listOfScheme` and `listOfConvertable` delegates provides correct items order. - Scheme meta setter works with proper sub-branch. -- NameToken.parse improper work with indices +- NameToken.parse improper work with indices. +- Proper data handling for cache. ### Security diff --git a/build.gradle.kts b/build.gradle.kts index b2f7e5e9..78dc6d35 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -8,7 +8,7 @@ plugins { allprojects { group = "space.kscience" - version = "0.8.2" + version = "0.8.2-dev-1" } subprojects { diff --git a/dataforge-context/api/dataforge-context.api b/dataforge-context/api/dataforge-context.api index 12bca176..eb94f89d 100644 --- a/dataforge-context/api/dataforge-context.api +++ b/dataforge-context/api/dataforge-context.api @@ -265,8 +265,6 @@ public abstract interface annotation class space/kscience/dataforge/descriptors/ } public final class space/kscience/dataforge/descriptors/ReflectiveDescriptorsKt { - public static final fun forClass (Lspace/kscience/dataforge/meta/descriptors/MetaDescriptor$Companion;Lkotlin/reflect/KClass;Lkotlin/jvm/functions/Function1;)Lspace/kscience/dataforge/meta/descriptors/MetaDescriptor; - public static synthetic fun forClass$default (Lspace/kscience/dataforge/meta/descriptors/MetaDescriptor$Companion;Lkotlin/reflect/KClass;Lkotlin/jvm/functions/Function1;ILjava/lang/Object;)Lspace/kscience/dataforge/meta/descriptors/MetaDescriptor; } public final class space/kscience/dataforge/properties/MetaAsFlowKt { diff --git a/dataforge-context/src/jvmMain/kotlin/space/kscience/dataforge/descriptors/reflectiveDescriptors.kt b/dataforge-context/src/jvmMain/kotlin/space/kscience/dataforge/descriptors/reflectiveDescriptors.kt index 1f2db7fc..9c36410a 100644 --- a/dataforge-context/src/jvmMain/kotlin/space/kscience/dataforge/descriptors/reflectiveDescriptors.kt +++ b/dataforge-context/src/jvmMain/kotlin/space/kscience/dataforge/descriptors/reflectiveDescriptors.kt @@ -10,11 +10,11 @@ import space.kscience.dataforge.meta.ValueType import space.kscience.dataforge.meta.descriptors.MetaDescriptor import space.kscience.dataforge.meta.descriptors.MetaDescriptorBuilder import space.kscience.dataforge.meta.descriptors.node +import space.kscience.dataforge.misc.DFExperimental import java.net.URL import kotlin.reflect.KClass import kotlin.reflect.full.isSubclassOf import kotlin.reflect.full.memberProperties -import kotlin.reflect.typeOf /** @@ -58,9 +58,9 @@ private fun MetaDescriptorBuilder.loadDescriptorFromResource(resource: Descripto } } - -public fun MetaDescriptor.Companion.forClass( - kClass: KClass, +@DFExperimental +public fun MetaDescriptor.Companion.forClass( + kClass: KClass, mod: MetaDescriptorBuilder.() -> Unit = {}, ): MetaDescriptor = MetaDescriptor { when { @@ -79,7 +79,7 @@ public fun MetaDescriptor.Companion.forClass( is DescriptorUrl -> loadDescriptorFromUrl(URL(it.url)) } } - kClass.memberProperties.forEach { property -> + kClass.memberProperties.forEach { property-> var flag = false @@ -88,6 +88,12 @@ public fun MetaDescriptor.Companion.forClass( (property.returnType.classifier as? KClass<*>)?.let { from(forClass(it)) } +// +// (property.getDelegate(Unit) as? MetaDelegate<*>)?.descriptor?.let { +// from(it) +// flag = true +// } + property.annotations.forEach { when (it) { is Description -> { @@ -119,6 +125,6 @@ public fun MetaDescriptor.Companion.forClass( mod() } -@Suppress("UNCHECKED_CAST") -public inline fun SchemeSpec.autoDescriptor( noinline mod: MetaDescriptorBuilder.() -> Unit = {}): MetaDescriptor = - MetaDescriptor.forClass(typeOf().classifier as KClass, mod) \ No newline at end of file +@DFExperimental +public inline fun SchemeSpec.autoDescriptor(noinline mod: MetaDescriptorBuilder.() -> Unit = {}): MetaDescriptor = + MetaDescriptor.forClass(T::class, mod) \ No newline at end of file diff --git a/dataforge-context/src/jvmTest/kotlin/space/kscience/dataforge/descriptors/TestAutoDescriptors.kt b/dataforge-context/src/jvmTest/kotlin/space/kscience/dataforge/descriptors/TestAutoDescriptors.kt index 617e85cc..df849ce5 100644 --- a/dataforge-context/src/jvmTest/kotlin/space/kscience/dataforge/descriptors/TestAutoDescriptors.kt +++ b/dataforge-context/src/jvmTest/kotlin/space/kscience/dataforge/descriptors/TestAutoDescriptors.kt @@ -1,3 +1,5 @@ +@file:OptIn(DFExperimental::class) + package space.kscience.dataforge.descriptors import kotlinx.serialization.encodeToString @@ -8,8 +10,9 @@ import space.kscience.dataforge.meta.SchemeSpec import space.kscience.dataforge.meta.descriptors.MetaDescriptor import space.kscience.dataforge.meta.int import space.kscience.dataforge.meta.string +import space.kscience.dataforge.misc.DFExperimental -private class TestScheme: Scheme(){ +private class TestScheme : Scheme() { @Description("A") val a by string() @@ -17,15 +20,17 @@ private class TestScheme: Scheme(){ @Description("B") val b by int() - companion object: SchemeSpec(::TestScheme){ + val c by int() + + companion object : SchemeSpec(::TestScheme) { override val descriptor: MetaDescriptor = autoDescriptor() } } class TestAutoDescriptors { @Test - fun autoDescriptor(){ + fun autoDescriptor() { val autoDescriptor = MetaDescriptor.forClass(TestScheme::class) - println(Json{prettyPrint = true}.encodeToString(autoDescriptor)) + println(Json { prettyPrint = true }.encodeToString(autoDescriptor)) } } \ No newline at end of file diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/MapAction.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/MapAction.kt index a1ca8e59..08bf08e9 100644 --- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/MapAction.kt +++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/MapAction.kt @@ -48,8 +48,8 @@ public class MapActionBuilder( public inline fun result(noinline f: suspend ActionEnv.(T) -> R1): Unit = result(typeOf(), f) } -@PublishedApi -internal class MapAction( +@UnsafeKType +public class MapAction( outputType: KType, private val block: MapActionBuilder.() -> Unit, ) : AbstractAction(outputType) { @@ -78,7 +78,6 @@ internal class MapAction( //getting new meta val newMeta = builder.meta.seal() - @OptIn(UnsafeKType::class) val newData = Data(builder.outputType, newMeta, dependencies = listOf(data)) { builder.result(env, data.await()) } @@ -106,6 +105,7 @@ internal class MapAction( * A one-to-one mapping action */ +@OptIn(UnsafeKType::class) public inline fun Action.Companion.mapping( noinline builder: MapActionBuilder.() -> Unit, ): Action = MapAction(typeOf(), builder) diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataFilter.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataFilter.kt index bc66e910..38174e50 100644 --- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataFilter.kt +++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataFilter.kt @@ -61,7 +61,7 @@ internal class FilteredDataTree( override val data: Data? get() = source[branch].takeIf { - filter.accepts(Name.EMPTY, data?.meta, data?.type ?: dataType) + filter.accepts(Name.EMPTY, it?.meta, it?.type ?: dataType) } override val items: Map> diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSink.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSink.kt index 9d860310..6daeae98 100644 --- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSink.kt +++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSink.kt @@ -2,6 +2,7 @@ package space.kscience.dataforge.data import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Job +import kotlinx.coroutines.channels.BufferOverflow import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.MutableSharedFlow import kotlinx.coroutines.flow.mapNotNull @@ -80,7 +81,8 @@ private class MutableDataTreeRoot( override val dataType: KType, ) : MutableDataTree { - override val updates = MutableSharedFlow>() + override val updates = MutableSharedFlow>(100, onBufferOverflow = BufferOverflow.DROP_LATEST) + inner class MutableDataTreeBranch(val branchName: Name) : MutableDataTree { @@ -108,10 +110,10 @@ private class MutableDataTreeRoot( override suspend fun update(name: Name, data: Data?) { if (name.isEmpty()) { this.data = data + this@MutableDataTreeRoot.updates.emit(DataUpdate(data?.type ?: dataType, branchName + name, data)) } else { getOrCreateItem(name.first()).update(name.cutFirst(), data) } - this@MutableDataTreeRoot.updates.emit(DataUpdate(data?.type ?: dataType, branchName + name, data)) } } @@ -122,7 +124,7 @@ private class MutableDataTreeRoot( override val items = HashMap>() override fun getOrCreateItem(token: NameToken): MutableDataTree = items.getOrPut(token) { - MutableDataTreeRoot(dataType) + MutableDataTreeBranch(token.asName()) } override fun set(token: NameToken, data: Data?) { @@ -133,10 +135,10 @@ private class MutableDataTreeRoot( override suspend fun update(name: Name, data: Data?) { if (name.isEmpty()) { this.data = data + updates.emit(DataUpdate(data?.type ?: dataType, name, data)) } else { getOrCreateItem(name.first()).update(name.cutFirst(), data) } - updates.emit(DataUpdate(data?.type ?: dataType, name, data)) } } diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataBuilders.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataBuilders.kt index 8ec3e361..649cfd19 100644 --- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataBuilders.kt +++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataBuilders.kt @@ -6,10 +6,7 @@ import kotlinx.coroutines.flow.launchIn import kotlinx.coroutines.flow.onEach import space.kscience.dataforge.meta.Meta import space.kscience.dataforge.meta.MutableMeta -import space.kscience.dataforge.names.Name -import space.kscience.dataforge.names.asName -import space.kscience.dataforge.names.isEmpty -import space.kscience.dataforge.names.plus +import space.kscience.dataforge.names.* public fun DataSink.put(value: NamedData) { @@ -83,13 +80,13 @@ public inline fun DataSink.putValue( * Emit static data with the fixed value */ public inline fun DataSink.putValue( - name: String, + name: Name, value: T, meta: Meta = Meta.EMPTY, ): Unit = put(name, Data.wrapValue(value, meta)) public inline fun DataSink.putValue( - name: Name, + name: String, value: T, meta: Meta = Meta.EMPTY, ): Unit = put(name, Data.wrapValue(value, meta)) @@ -100,6 +97,18 @@ public inline fun DataSink.putValue( metaBuilder: MutableMeta.() -> Unit, ): Unit = put(Name.parse(name), Data.wrapValue(value, Meta(metaBuilder))) +public suspend inline fun DataSink.updateValue( + name: Name, + value: T, + meta: Meta = Meta.EMPTY, +): Unit = update(name, Data.wrapValue(value, meta)) + +public suspend inline fun DataSink.updateValue( + name: String, + value: T, + meta: Meta = Meta.EMPTY, +): Unit = update(name.parseAsName(), Data.wrapValue(value, meta)) + public fun DataSink.putAll(sequence: Sequence>) { sequence.forEach { put(it.name, it.data) @@ -120,6 +129,6 @@ public fun DataSink.putAllAndWatch( ): Job { putAll(branchName, source) return source.updates.onEach { - put(branchName + it.name, it.data) + update(branchName + it.name, it.data) }.launchIn(scope) } \ No newline at end of file diff --git a/dataforge-data/src/commonTest/kotlin/space/kscience/dataforge/data/ActionsTest.kt b/dataforge-data/src/commonTest/kotlin/space/kscience/dataforge/data/ActionsTest.kt index ef9b14ab..477ca592 100644 --- a/dataforge-data/src/commonTest/kotlin/space/kscience/dataforge/data/ActionsTest.kt +++ b/dataforge-data/src/commonTest/kotlin/space/kscience/dataforge/data/ActionsTest.kt @@ -41,7 +41,7 @@ internal class ActionsTest { repeat(10) { - source.putValue(it.toString(), it) + source.updateValue(it.toString(), it) } result.updates.take(10).onEach { println(it.name) }.collect() diff --git a/dataforge-data/src/commonTest/kotlin/space/kscience/dataforge/data/DataTreeBuilderTest.kt b/dataforge-data/src/commonTest/kotlin/space/kscience/dataforge/data/DataTreeBuilderTest.kt index b5738820..760aeec2 100644 --- a/dataforge-data/src/commonTest/kotlin/space/kscience/dataforge/data/DataTreeBuilderTest.kt +++ b/dataforge-data/src/commonTest/kotlin/space/kscience/dataforge/data/DataTreeBuilderTest.kt @@ -1,9 +1,8 @@ package space.kscience.dataforge.data -import kotlinx.coroutines.cancel +import kotlinx.coroutines.Job import kotlinx.coroutines.flow.collect import kotlinx.coroutines.flow.take -import kotlinx.coroutines.launch import kotlinx.coroutines.test.runTest import space.kscience.dataforge.names.asName import kotlin.test.Test @@ -15,7 +14,7 @@ internal class DataTreeBuilderTest { @Test fun testTreeBuild() = runTest(timeout = 500.milliseconds) { val node = DataTree { - putAll("primary"){ + putAll("primary") { putValue("a", "a") putValue("b", "b") } @@ -53,19 +52,22 @@ internal class DataTreeBuilderTest { @Test fun testDynamicUpdates() = runTest(timeout = 500.milliseconds) { - launch { - val subNode = MutableDataTree() + var job: Job? = null - val rootNode = MutableDataTree() { - putAllAndWatch(this@launch, "sub".asName(), subNode) - } + val subNode = MutableDataTree() - repeat(10) { - subNode.putValue("value[$it]", it) - } - subNode.updates.take(10).collect() - assertEquals(9, rootNode["sub.value[9]"]?.await()) - cancel() - }.join() + val rootNode = MutableDataTree() { + job = putAllAndWatch(this@runTest, "sub".asName(), subNode) + } + + repeat(10) { + subNode.updateValue("value[$it]", it) + } + + rootNode.updates.take(10).collect() + assertEquals(9, rootNode["sub.value[9]"]?.await()) + assertEquals(8, rootNode["sub.value[8]"]?.await()) + + job?.cancel() } } \ No newline at end of file diff --git a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/IOPlugin.kt b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/IOPlugin.kt index 0d79da4d..f431a731 100644 --- a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/IOPlugin.kt +++ b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/IOPlugin.kt @@ -6,7 +6,7 @@ import space.kscience.dataforge.io.IOFormatFactory.Companion.IO_FORMAT_TYPE import space.kscience.dataforge.io.MetaFormatFactory.Companion.META_FORMAT_TYPE import space.kscience.dataforge.meta.Meta import space.kscience.dataforge.meta.string -import space.kscience.dataforge.misc.DFInternal +import space.kscience.dataforge.misc.UnsafeKType import space.kscience.dataforge.names.Name import kotlin.reflect.KType import kotlin.reflect.typeOf @@ -19,11 +19,11 @@ public class IOPlugin(meta: Meta) : AbstractPlugin(meta) { } @Suppress("UNCHECKED_CAST") - @DFInternal + @UnsafeKType public fun resolveIOFormat(type: KType, meta: Meta): IOFormat? = ioFormatFactories.singleOrNull { it.type == type }?.build(context, meta) as? IOFormat - @OptIn(DFInternal::class) + @OptIn(UnsafeKType::class) public inline fun resolveIOFormat(meta: Meta = Meta.EMPTY): IOFormat? = resolveIOFormat(typeOf(), meta) diff --git a/dataforge-meta/api/dataforge-meta.api b/dataforge-meta/api/dataforge-meta.api index f9834c54..620e64d4 100644 --- a/dataforge-meta/api/dataforge-meta.api +++ b/dataforge-meta/api/dataforge-meta.api @@ -1,3 +1,15 @@ +public final class space/kscience/dataforge/meta/ByteArrayValue : java/lang/Iterable, kotlin/jvm/internal/markers/KMappedMarker, space/kscience/dataforge/meta/Value { + public fun ([B)V + public fun equals (Ljava/lang/Object;)Z + public fun getList ()Ljava/util/List; + public fun getType ()Lspace/kscience/dataforge/meta/ValueType; + public synthetic fun getValue ()Ljava/lang/Object; + public fun getValue ()[B + public fun hashCode ()I + public fun iterator ()Ljava/util/Iterator; + public fun toString ()Ljava/lang/String; +} + public abstract interface class space/kscience/dataforge/meta/Configurable { public abstract fun getMeta ()Lspace/kscience/dataforge/meta/MutableMeta; } @@ -30,7 +42,20 @@ public final class space/kscience/dataforge/meta/EnumValue : space/kscience/data } public final class space/kscience/dataforge/meta/ExoticValuesKt { + public static final fun asValue ([B)Lspace/kscience/dataforge/meta/Value; public static final fun asValue ([D)Lspace/kscience/dataforge/meta/Value; + public static final fun byteArray (Lspace/kscience/dataforge/meta/MetaProvider;[BLspace/kscience/dataforge/names/Name;)Lkotlin/properties/ReadOnlyProperty; + public static final fun byteArray (Lspace/kscience/dataforge/meta/MutableMetaProvider;[BLspace/kscience/dataforge/names/Name;)Lkotlin/properties/ReadWriteProperty; + public static synthetic fun byteArray$default (Lspace/kscience/dataforge/meta/MetaProvider;[BLspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lkotlin/properties/ReadOnlyProperty; + public static synthetic fun byteArray$default (Lspace/kscience/dataforge/meta/MutableMetaProvider;[BLspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lkotlin/properties/ReadWriteProperty; + public static final fun doubleArray (Lspace/kscience/dataforge/meta/MetaProvider;[DLspace/kscience/dataforge/names/Name;)Lkotlin/properties/ReadOnlyProperty; + public static final fun doubleArray (Lspace/kscience/dataforge/meta/MutableMetaProvider;[DLspace/kscience/dataforge/names/Name;)Lkotlin/properties/ReadWriteProperty; + public static synthetic fun doubleArray$default (Lspace/kscience/dataforge/meta/MetaProvider;[DLspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lkotlin/properties/ReadOnlyProperty; + public static synthetic fun doubleArray$default (Lspace/kscience/dataforge/meta/MutableMetaProvider;[DLspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lkotlin/properties/ReadWriteProperty; + public static final fun getByteArray (Lspace/kscience/dataforge/meta/Meta;)[B + public static final fun getByteArray (Lspace/kscience/dataforge/meta/Value;)[B + public static final fun getDoubleArray (Lspace/kscience/dataforge/meta/Meta;)[D + public static final fun getDoubleArray (Lspace/kscience/dataforge/meta/Value;)[D public static final fun lazyParseValue (Ljava/lang/String;)Lspace/kscience/dataforge/meta/LazyParsedValue; } @@ -199,53 +224,56 @@ public final class space/kscience/dataforge/meta/MetaConverterKt { public static final fun convertNullable (Lspace/kscience/dataforge/meta/MetaConverter;Ljava/lang/Object;)Lspace/kscience/dataforge/meta/Meta; } +public abstract interface class space/kscience/dataforge/meta/MetaDelegate : kotlin/properties/ReadOnlyProperty, space/kscience/dataforge/meta/descriptors/Described { +} + public final class space/kscience/dataforge/meta/MetaDelegateKt { - public static final fun boolean (Lspace/kscience/dataforge/meta/MetaProvider;Lspace/kscience/dataforge/names/Name;)Lkotlin/properties/ReadOnlyProperty; - public static final fun boolean (Lspace/kscience/dataforge/meta/MetaProvider;Lspace/kscience/dataforge/names/Name;Lkotlin/jvm/functions/Function0;)Lkotlin/properties/ReadOnlyProperty; - public static final fun boolean (Lspace/kscience/dataforge/meta/MetaProvider;ZLspace/kscience/dataforge/names/Name;)Lkotlin/properties/ReadOnlyProperty; - public static synthetic fun boolean$default (Lspace/kscience/dataforge/meta/MetaProvider;Lspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lkotlin/properties/ReadOnlyProperty; - public static synthetic fun boolean$default (Lspace/kscience/dataforge/meta/MetaProvider;Lspace/kscience/dataforge/names/Name;Lkotlin/jvm/functions/Function0;ILjava/lang/Object;)Lkotlin/properties/ReadOnlyProperty; - public static synthetic fun boolean$default (Lspace/kscience/dataforge/meta/MetaProvider;ZLspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lkotlin/properties/ReadOnlyProperty; - public static final fun double (Lspace/kscience/dataforge/meta/MetaProvider;DLspace/kscience/dataforge/names/Name;)Lkotlin/properties/ReadOnlyProperty; - public static final fun double (Lspace/kscience/dataforge/meta/MetaProvider;Lspace/kscience/dataforge/names/Name;)Lkotlin/properties/ReadOnlyProperty; - public static synthetic fun double$default (Lspace/kscience/dataforge/meta/MetaProvider;DLspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lkotlin/properties/ReadOnlyProperty; - public static synthetic fun double$default (Lspace/kscience/dataforge/meta/MetaProvider;Lspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lkotlin/properties/ReadOnlyProperty; - public static final fun float (Lspace/kscience/dataforge/meta/MetaProvider;FLspace/kscience/dataforge/names/Name;)Lkotlin/properties/ReadOnlyProperty; - public static final fun float (Lspace/kscience/dataforge/meta/MetaProvider;Lspace/kscience/dataforge/names/Name;)Lkotlin/properties/ReadOnlyProperty; - public static synthetic fun float$default (Lspace/kscience/dataforge/meta/MetaProvider;FLspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lkotlin/properties/ReadOnlyProperty; - public static synthetic fun float$default (Lspace/kscience/dataforge/meta/MetaProvider;Lspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lkotlin/properties/ReadOnlyProperty; - public static final fun int (Lspace/kscience/dataforge/meta/MetaProvider;ILspace/kscience/dataforge/names/Name;)Lkotlin/properties/ReadOnlyProperty; - public static final fun int (Lspace/kscience/dataforge/meta/MetaProvider;Lspace/kscience/dataforge/names/Name;)Lkotlin/properties/ReadOnlyProperty; - public static synthetic fun int$default (Lspace/kscience/dataforge/meta/MetaProvider;ILspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lkotlin/properties/ReadOnlyProperty; - public static synthetic fun int$default (Lspace/kscience/dataforge/meta/MetaProvider;Lspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lkotlin/properties/ReadOnlyProperty; - public static final fun listOfSpec (Lspace/kscience/dataforge/meta/Meta;Lspace/kscience/dataforge/meta/MetaSpec;Lspace/kscience/dataforge/names/Name;)Lkotlin/properties/ReadOnlyProperty; - public static synthetic fun listOfSpec$default (Lspace/kscience/dataforge/meta/Meta;Lspace/kscience/dataforge/meta/MetaSpec;Lspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lkotlin/properties/ReadOnlyProperty; - public static final fun long (Lspace/kscience/dataforge/meta/MetaProvider;JLspace/kscience/dataforge/names/Name;)Lkotlin/properties/ReadOnlyProperty; - public static final fun long (Lspace/kscience/dataforge/meta/MetaProvider;Lspace/kscience/dataforge/names/Name;)Lkotlin/properties/ReadOnlyProperty; - public static synthetic fun long$default (Lspace/kscience/dataforge/meta/MetaProvider;JLspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lkotlin/properties/ReadOnlyProperty; - public static synthetic fun long$default (Lspace/kscience/dataforge/meta/MetaProvider;Lspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lkotlin/properties/ReadOnlyProperty; - public static final fun node (Lspace/kscience/dataforge/meta/MetaProvider;Lspace/kscience/dataforge/names/Name;)Lkotlin/properties/ReadOnlyProperty; + public static final fun boolean (Lspace/kscience/dataforge/meta/MetaProvider;Lspace/kscience/dataforge/names/Name;)Lspace/kscience/dataforge/meta/MetaDelegate; + public static final fun boolean (Lspace/kscience/dataforge/meta/MetaProvider;Lspace/kscience/dataforge/names/Name;Lkotlin/jvm/functions/Function0;)Lspace/kscience/dataforge/meta/MetaDelegate; + public static final fun boolean (Lspace/kscience/dataforge/meta/MetaProvider;ZLspace/kscience/dataforge/names/Name;)Lspace/kscience/dataforge/meta/MetaDelegate; + public static synthetic fun boolean$default (Lspace/kscience/dataforge/meta/MetaProvider;Lspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lspace/kscience/dataforge/meta/MetaDelegate; + public static synthetic fun boolean$default (Lspace/kscience/dataforge/meta/MetaProvider;Lspace/kscience/dataforge/names/Name;Lkotlin/jvm/functions/Function0;ILjava/lang/Object;)Lspace/kscience/dataforge/meta/MetaDelegate; + public static synthetic fun boolean$default (Lspace/kscience/dataforge/meta/MetaProvider;ZLspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lspace/kscience/dataforge/meta/MetaDelegate; + public static final fun double (Lspace/kscience/dataforge/meta/MetaProvider;DLspace/kscience/dataforge/names/Name;)Lspace/kscience/dataforge/meta/MetaDelegate; + public static final fun double (Lspace/kscience/dataforge/meta/MetaProvider;Lspace/kscience/dataforge/names/Name;)Lspace/kscience/dataforge/meta/MetaDelegate; + public static synthetic fun double$default (Lspace/kscience/dataforge/meta/MetaProvider;DLspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lspace/kscience/dataforge/meta/MetaDelegate; + public static synthetic fun double$default (Lspace/kscience/dataforge/meta/MetaProvider;Lspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lspace/kscience/dataforge/meta/MetaDelegate; + public static final fun float (Lspace/kscience/dataforge/meta/MetaProvider;FLspace/kscience/dataforge/names/Name;)Lspace/kscience/dataforge/meta/MetaDelegate; + public static final fun float (Lspace/kscience/dataforge/meta/MetaProvider;Lspace/kscience/dataforge/names/Name;)Lspace/kscience/dataforge/meta/MetaDelegate; + public static synthetic fun float$default (Lspace/kscience/dataforge/meta/MetaProvider;FLspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lspace/kscience/dataforge/meta/MetaDelegate; + public static synthetic fun float$default (Lspace/kscience/dataforge/meta/MetaProvider;Lspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lspace/kscience/dataforge/meta/MetaDelegate; + public static final fun int (Lspace/kscience/dataforge/meta/MetaProvider;ILspace/kscience/dataforge/names/Name;)Lspace/kscience/dataforge/meta/MetaDelegate; + public static final fun int (Lspace/kscience/dataforge/meta/MetaProvider;Lspace/kscience/dataforge/names/Name;)Lspace/kscience/dataforge/meta/MetaDelegate; + public static synthetic fun int$default (Lspace/kscience/dataforge/meta/MetaProvider;ILspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lspace/kscience/dataforge/meta/MetaDelegate; + public static synthetic fun int$default (Lspace/kscience/dataforge/meta/MetaProvider;Lspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lspace/kscience/dataforge/meta/MetaDelegate; + public static final fun listOfSpec (Lspace/kscience/dataforge/meta/Meta;Lspace/kscience/dataforge/meta/MetaSpec;Lspace/kscience/dataforge/names/Name;)Lspace/kscience/dataforge/meta/MetaDelegate; + public static synthetic fun listOfSpec$default (Lspace/kscience/dataforge/meta/Meta;Lspace/kscience/dataforge/meta/MetaSpec;Lspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lspace/kscience/dataforge/meta/MetaDelegate; + public static final fun long (Lspace/kscience/dataforge/meta/MetaProvider;JLspace/kscience/dataforge/names/Name;)Lspace/kscience/dataforge/meta/MetaDelegate; + public static final fun long (Lspace/kscience/dataforge/meta/MetaProvider;Lspace/kscience/dataforge/names/Name;)Lspace/kscience/dataforge/meta/MetaDelegate; + public static synthetic fun long$default (Lspace/kscience/dataforge/meta/MetaProvider;JLspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lspace/kscience/dataforge/meta/MetaDelegate; + public static synthetic fun long$default (Lspace/kscience/dataforge/meta/MetaProvider;Lspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lspace/kscience/dataforge/meta/MetaDelegate; public static final fun node (Lspace/kscience/dataforge/meta/MetaProvider;Lspace/kscience/dataforge/names/Name;Lspace/kscience/dataforge/meta/MetaSpec;)Lkotlin/properties/ReadOnlyProperty; - public static synthetic fun node$default (Lspace/kscience/dataforge/meta/MetaProvider;Lspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lkotlin/properties/ReadOnlyProperty; + public static final fun node (Lspace/kscience/dataforge/meta/MetaProvider;Lspace/kscience/dataforge/names/Name;Lspace/kscience/dataforge/meta/descriptors/MetaDescriptor;)Lspace/kscience/dataforge/meta/MetaDelegate; public static synthetic fun node$default (Lspace/kscience/dataforge/meta/MetaProvider;Lspace/kscience/dataforge/names/Name;Lspace/kscience/dataforge/meta/MetaSpec;ILjava/lang/Object;)Lkotlin/properties/ReadOnlyProperty; - public static final fun number (Lspace/kscience/dataforge/meta/MetaProvider;Ljava/lang/Number;Lspace/kscience/dataforge/names/Name;)Lkotlin/properties/ReadOnlyProperty; - public static final fun number (Lspace/kscience/dataforge/meta/MetaProvider;Lspace/kscience/dataforge/names/Name;)Lkotlin/properties/ReadOnlyProperty; - public static final fun number (Lspace/kscience/dataforge/meta/MetaProvider;Lspace/kscience/dataforge/names/Name;Lkotlin/jvm/functions/Function0;)Lkotlin/properties/ReadOnlyProperty; - public static synthetic fun number$default (Lspace/kscience/dataforge/meta/MetaProvider;Ljava/lang/Number;Lspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lkotlin/properties/ReadOnlyProperty; - public static synthetic fun number$default (Lspace/kscience/dataforge/meta/MetaProvider;Lspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lkotlin/properties/ReadOnlyProperty; - public static synthetic fun number$default (Lspace/kscience/dataforge/meta/MetaProvider;Lspace/kscience/dataforge/names/Name;Lkotlin/jvm/functions/Function0;ILjava/lang/Object;)Lkotlin/properties/ReadOnlyProperty; - public static final fun spec (Lspace/kscience/dataforge/meta/MetaProvider;Lspace/kscience/dataforge/meta/MetaSpec;Lspace/kscience/dataforge/names/Name;)Lkotlin/properties/ReadOnlyProperty; - public static synthetic fun spec$default (Lspace/kscience/dataforge/meta/MetaProvider;Lspace/kscience/dataforge/meta/MetaSpec;Lspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lkotlin/properties/ReadOnlyProperty; - public static final fun string (Lspace/kscience/dataforge/meta/MetaProvider;Ljava/lang/String;Lspace/kscience/dataforge/names/Name;)Lkotlin/properties/ReadOnlyProperty; - public static final fun string (Lspace/kscience/dataforge/meta/MetaProvider;Lspace/kscience/dataforge/names/Name;)Lkotlin/properties/ReadOnlyProperty; - public static final fun string (Lspace/kscience/dataforge/meta/MetaProvider;Lspace/kscience/dataforge/names/Name;Lkotlin/jvm/functions/Function0;)Lkotlin/properties/ReadOnlyProperty; - public static synthetic fun string$default (Lspace/kscience/dataforge/meta/MetaProvider;Ljava/lang/String;Lspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lkotlin/properties/ReadOnlyProperty; - public static synthetic fun string$default (Lspace/kscience/dataforge/meta/MetaProvider;Lspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lkotlin/properties/ReadOnlyProperty; - public static synthetic fun string$default (Lspace/kscience/dataforge/meta/MetaProvider;Lspace/kscience/dataforge/names/Name;Lkotlin/jvm/functions/Function0;ILjava/lang/Object;)Lkotlin/properties/ReadOnlyProperty; - public static final fun value (Lspace/kscience/dataforge/meta/MetaProvider;Lspace/kscience/dataforge/names/Name;)Lkotlin/properties/ReadOnlyProperty; - public static final fun value (Lspace/kscience/dataforge/meta/MetaProvider;Lspace/kscience/dataforge/names/Name;Lkotlin/jvm/functions/Function1;)Lkotlin/properties/ReadOnlyProperty; - public static synthetic fun value$default (Lspace/kscience/dataforge/meta/MetaProvider;Lspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lkotlin/properties/ReadOnlyProperty; - public static synthetic fun value$default (Lspace/kscience/dataforge/meta/MetaProvider;Lspace/kscience/dataforge/names/Name;Lkotlin/jvm/functions/Function1;ILjava/lang/Object;)Lkotlin/properties/ReadOnlyProperty; + public static synthetic fun node$default (Lspace/kscience/dataforge/meta/MetaProvider;Lspace/kscience/dataforge/names/Name;Lspace/kscience/dataforge/meta/descriptors/MetaDescriptor;ILjava/lang/Object;)Lspace/kscience/dataforge/meta/MetaDelegate; + public static final fun number (Lspace/kscience/dataforge/meta/MetaProvider;Ljava/lang/Number;Lspace/kscience/dataforge/names/Name;)Lspace/kscience/dataforge/meta/MetaDelegate; + public static final fun number (Lspace/kscience/dataforge/meta/MetaProvider;Lspace/kscience/dataforge/names/Name;)Lspace/kscience/dataforge/meta/MetaDelegate; + public static final fun number (Lspace/kscience/dataforge/meta/MetaProvider;Lspace/kscience/dataforge/names/Name;Lkotlin/jvm/functions/Function0;)Lspace/kscience/dataforge/meta/MetaDelegate; + public static synthetic fun number$default (Lspace/kscience/dataforge/meta/MetaProvider;Ljava/lang/Number;Lspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lspace/kscience/dataforge/meta/MetaDelegate; + public static synthetic fun number$default (Lspace/kscience/dataforge/meta/MetaProvider;Lspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lspace/kscience/dataforge/meta/MetaDelegate; + public static synthetic fun number$default (Lspace/kscience/dataforge/meta/MetaProvider;Lspace/kscience/dataforge/names/Name;Lkotlin/jvm/functions/Function0;ILjava/lang/Object;)Lspace/kscience/dataforge/meta/MetaDelegate; + public static final fun spec (Lspace/kscience/dataforge/meta/MetaProvider;Lspace/kscience/dataforge/meta/MetaSpec;Lspace/kscience/dataforge/names/Name;)Lspace/kscience/dataforge/meta/MetaDelegate; + public static synthetic fun spec$default (Lspace/kscience/dataforge/meta/MetaProvider;Lspace/kscience/dataforge/meta/MetaSpec;Lspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lspace/kscience/dataforge/meta/MetaDelegate; + public static final fun string (Lspace/kscience/dataforge/meta/MetaProvider;Ljava/lang/String;Lspace/kscience/dataforge/names/Name;)Lspace/kscience/dataforge/meta/MetaDelegate; + public static final fun string (Lspace/kscience/dataforge/meta/MetaProvider;Lspace/kscience/dataforge/names/Name;)Lspace/kscience/dataforge/meta/MetaDelegate; + public static final fun string (Lspace/kscience/dataforge/meta/MetaProvider;Lspace/kscience/dataforge/names/Name;Lkotlin/jvm/functions/Function0;)Lspace/kscience/dataforge/meta/MetaDelegate; + public static synthetic fun string$default (Lspace/kscience/dataforge/meta/MetaProvider;Ljava/lang/String;Lspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lspace/kscience/dataforge/meta/MetaDelegate; + public static synthetic fun string$default (Lspace/kscience/dataforge/meta/MetaProvider;Lspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lspace/kscience/dataforge/meta/MetaDelegate; + public static synthetic fun string$default (Lspace/kscience/dataforge/meta/MetaProvider;Lspace/kscience/dataforge/names/Name;Lkotlin/jvm/functions/Function0;ILjava/lang/Object;)Lspace/kscience/dataforge/meta/MetaDelegate; + public static final fun value (Lspace/kscience/dataforge/meta/MetaProvider;Lspace/kscience/dataforge/names/Name;Lspace/kscience/dataforge/meta/descriptors/MetaDescriptor;)Lspace/kscience/dataforge/meta/MetaDelegate; + public static final fun value (Lspace/kscience/dataforge/meta/MetaProvider;Lspace/kscience/dataforge/names/Name;Lspace/kscience/dataforge/meta/descriptors/MetaDescriptor;Lkotlin/jvm/functions/Function1;)Lspace/kscience/dataforge/meta/MetaDelegate; + public static synthetic fun value$default (Lspace/kscience/dataforge/meta/MetaProvider;Lspace/kscience/dataforge/names/Name;Lspace/kscience/dataforge/meta/descriptors/MetaDescriptor;ILjava/lang/Object;)Lspace/kscience/dataforge/meta/MetaDelegate; + public static synthetic fun value$default (Lspace/kscience/dataforge/meta/MetaProvider;Lspace/kscience/dataforge/names/Name;Lspace/kscience/dataforge/meta/descriptors/MetaDescriptor;Lkotlin/jvm/functions/Function1;ILjava/lang/Object;)Lspace/kscience/dataforge/meta/MetaDelegate; } public final class space/kscience/dataforge/meta/MetaKt { @@ -352,6 +380,7 @@ public abstract interface class space/kscience/dataforge/meta/MutableMeta : spac public fun put (Ljava/lang/String;Lspace/kscience/dataforge/meta/MetaRepr;)V public fun put (Ljava/lang/String;Lspace/kscience/dataforge/meta/Value;)V public fun put (Ljava/lang/String;Z)V + public fun put (Ljava/lang/String;[B)V public fun put (Ljava/lang/String;[D)V public fun put (Lspace/kscience/dataforge/names/Name;Ljava/lang/Enum;)V public fun put (Lspace/kscience/dataforge/names/Name;Ljava/lang/Number;)V @@ -371,63 +400,64 @@ public final class space/kscience/dataforge/meta/MutableMeta$Companion { public final fun serializer ()Lkotlinx/serialization/KSerializer; } +public abstract interface class space/kscience/dataforge/meta/MutableMetaDelegate : kotlin/properties/ReadWriteProperty, space/kscience/dataforge/meta/descriptors/Described { +} + public final class space/kscience/dataforge/meta/MutableMetaDelegateKt { - public static final fun boolean (Lspace/kscience/dataforge/meta/MutableMetaProvider;Lspace/kscience/dataforge/names/Name;)Lkotlin/properties/ReadWriteProperty; - public static final fun boolean (Lspace/kscience/dataforge/meta/MutableMetaProvider;Lspace/kscience/dataforge/names/Name;Lkotlin/jvm/functions/Function0;)Lkotlin/properties/ReadWriteProperty; - public static final fun boolean (Lspace/kscience/dataforge/meta/MutableMetaProvider;ZLspace/kscience/dataforge/names/Name;)Lkotlin/properties/ReadWriteProperty; - public static synthetic fun boolean$default (Lspace/kscience/dataforge/meta/MutableMetaProvider;Lspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lkotlin/properties/ReadWriteProperty; - public static synthetic fun boolean$default (Lspace/kscience/dataforge/meta/MutableMetaProvider;Lspace/kscience/dataforge/names/Name;Lkotlin/jvm/functions/Function0;ILjava/lang/Object;)Lkotlin/properties/ReadWriteProperty; - public static synthetic fun boolean$default (Lspace/kscience/dataforge/meta/MutableMetaProvider;ZLspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lkotlin/properties/ReadWriteProperty; - public static final fun convertable (Lspace/kscience/dataforge/meta/MutableMetaProvider;Lspace/kscience/dataforge/meta/MetaConverter;Lspace/kscience/dataforge/names/Name;)Lkotlin/properties/ReadWriteProperty; - public static synthetic fun convertable$default (Lspace/kscience/dataforge/meta/MutableMetaProvider;Lspace/kscience/dataforge/meta/MetaConverter;Lspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lkotlin/properties/ReadWriteProperty; - public static final fun double (Lspace/kscience/dataforge/meta/MutableMetaProvider;DLspace/kscience/dataforge/names/Name;)Lkotlin/properties/ReadWriteProperty; - public static final fun double (Lspace/kscience/dataforge/meta/MutableMetaProvider;Lspace/kscience/dataforge/names/Name;)Lkotlin/properties/ReadWriteProperty; - public static synthetic fun double$default (Lspace/kscience/dataforge/meta/MutableMetaProvider;DLspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lkotlin/properties/ReadWriteProperty; - public static synthetic fun double$default (Lspace/kscience/dataforge/meta/MutableMetaProvider;Lspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lkotlin/properties/ReadWriteProperty; - public static final fun doubleArray (Lspace/kscience/dataforge/meta/MutableMetaProvider;[DLspace/kscience/dataforge/names/Name;)Lkotlin/properties/ReadWriteProperty; - public static synthetic fun doubleArray$default (Lspace/kscience/dataforge/meta/MutableMetaProvider;[DLspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lkotlin/properties/ReadWriteProperty; - public static final fun float (Lspace/kscience/dataforge/meta/MutableMetaProvider;FLspace/kscience/dataforge/names/Name;)Lkotlin/properties/ReadWriteProperty; - public static final fun float (Lspace/kscience/dataforge/meta/MutableMetaProvider;Lspace/kscience/dataforge/names/Name;)Lkotlin/properties/ReadWriteProperty; - public static synthetic fun float$default (Lspace/kscience/dataforge/meta/MutableMetaProvider;FLspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lkotlin/properties/ReadWriteProperty; - public static synthetic fun float$default (Lspace/kscience/dataforge/meta/MutableMetaProvider;Lspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lkotlin/properties/ReadWriteProperty; - public static final fun int (Lspace/kscience/dataforge/meta/MutableMetaProvider;ILspace/kscience/dataforge/names/Name;)Lkotlin/properties/ReadWriteProperty; - public static final fun int (Lspace/kscience/dataforge/meta/MutableMetaProvider;Lspace/kscience/dataforge/names/Name;)Lkotlin/properties/ReadWriteProperty; - public static synthetic fun int$default (Lspace/kscience/dataforge/meta/MutableMetaProvider;ILspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lkotlin/properties/ReadWriteProperty; - public static synthetic fun int$default (Lspace/kscience/dataforge/meta/MutableMetaProvider;Lspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lkotlin/properties/ReadWriteProperty; - public static final fun listOfConvertable (Lspace/kscience/dataforge/meta/MutableMeta;Lspace/kscience/dataforge/meta/MetaConverter;Lspace/kscience/dataforge/names/Name;)Lkotlin/properties/ReadWriteProperty; - public static synthetic fun listOfConvertable$default (Lspace/kscience/dataforge/meta/MutableMeta;Lspace/kscience/dataforge/meta/MetaConverter;Lspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lkotlin/properties/ReadWriteProperty; - public static final fun listValue (Lspace/kscience/dataforge/meta/MutableMetaProvider;Lspace/kscience/dataforge/names/Name;Lkotlin/jvm/functions/Function1;Lkotlin/jvm/functions/Function1;)Lkotlin/properties/ReadWriteProperty; - public static synthetic fun listValue$default (Lspace/kscience/dataforge/meta/MutableMetaProvider;Lspace/kscience/dataforge/names/Name;Lkotlin/jvm/functions/Function1;Lkotlin/jvm/functions/Function1;ILjava/lang/Object;)Lkotlin/properties/ReadWriteProperty; - public static final fun long (Lspace/kscience/dataforge/meta/MutableMetaProvider;JLspace/kscience/dataforge/names/Name;)Lkotlin/properties/ReadWriteProperty; - public static final fun long (Lspace/kscience/dataforge/meta/MutableMetaProvider;Lspace/kscience/dataforge/names/Name;)Lkotlin/properties/ReadWriteProperty; - public static synthetic fun long$default (Lspace/kscience/dataforge/meta/MutableMetaProvider;JLspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lkotlin/properties/ReadWriteProperty; - public static synthetic fun long$default (Lspace/kscience/dataforge/meta/MutableMetaProvider;Lspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lkotlin/properties/ReadWriteProperty; - public static final fun node (Lspace/kscience/dataforge/meta/MutableMetaProvider;Lspace/kscience/dataforge/names/Name;)Lkotlin/properties/ReadWriteProperty; - public static final fun node (Lspace/kscience/dataforge/meta/MutableMetaProvider;Lspace/kscience/dataforge/names/Name;Lspace/kscience/dataforge/meta/MetaConverter;)Lkotlin/properties/ReadWriteProperty; - public static synthetic fun node$default (Lspace/kscience/dataforge/meta/MutableMetaProvider;Lspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lkotlin/properties/ReadWriteProperty; - public static synthetic fun node$default (Lspace/kscience/dataforge/meta/MutableMetaProvider;Lspace/kscience/dataforge/names/Name;Lspace/kscience/dataforge/meta/MetaConverter;ILjava/lang/Object;)Lkotlin/properties/ReadWriteProperty; - public static final fun number (Lspace/kscience/dataforge/meta/MutableMetaProvider;Ljava/lang/Number;Lspace/kscience/dataforge/names/Name;)Lkotlin/properties/ReadWriteProperty; - public static final fun number (Lspace/kscience/dataforge/meta/MutableMetaProvider;Lspace/kscience/dataforge/names/Name;)Lkotlin/properties/ReadWriteProperty; - public static final fun number (Lspace/kscience/dataforge/meta/MutableMetaProvider;Lspace/kscience/dataforge/names/Name;Lkotlin/jvm/functions/Function0;)Lkotlin/properties/ReadWriteProperty; - public static synthetic fun number$default (Lspace/kscience/dataforge/meta/MutableMetaProvider;Ljava/lang/Number;Lspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lkotlin/properties/ReadWriteProperty; - public static synthetic fun number$default (Lspace/kscience/dataforge/meta/MutableMetaProvider;Lspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lkotlin/properties/ReadWriteProperty; - public static synthetic fun number$default (Lspace/kscience/dataforge/meta/MutableMetaProvider;Lspace/kscience/dataforge/names/Name;Lkotlin/jvm/functions/Function0;ILjava/lang/Object;)Lkotlin/properties/ReadWriteProperty; - public static final fun numberList (Lspace/kscience/dataforge/meta/MutableMetaProvider;[Ljava/lang/Number;Lspace/kscience/dataforge/names/Name;)Lkotlin/properties/ReadWriteProperty; - public static synthetic fun numberList$default (Lspace/kscience/dataforge/meta/MutableMetaProvider;[Ljava/lang/Number;Lspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lkotlin/properties/ReadWriteProperty; - public static final fun string (Lspace/kscience/dataforge/meta/MutableMetaProvider;Ljava/lang/String;Lspace/kscience/dataforge/names/Name;)Lkotlin/properties/ReadWriteProperty; - public static final fun string (Lspace/kscience/dataforge/meta/MutableMetaProvider;Lspace/kscience/dataforge/names/Name;)Lkotlin/properties/ReadWriteProperty; - public static final fun string (Lspace/kscience/dataforge/meta/MutableMetaProvider;Lspace/kscience/dataforge/names/Name;Lkotlin/jvm/functions/Function0;)Lkotlin/properties/ReadWriteProperty; - public static synthetic fun string$default (Lspace/kscience/dataforge/meta/MutableMetaProvider;Ljava/lang/String;Lspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lkotlin/properties/ReadWriteProperty; - public static synthetic fun string$default (Lspace/kscience/dataforge/meta/MutableMetaProvider;Lspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lkotlin/properties/ReadWriteProperty; - public static synthetic fun string$default (Lspace/kscience/dataforge/meta/MutableMetaProvider;Lspace/kscience/dataforge/names/Name;Lkotlin/jvm/functions/Function0;ILjava/lang/Object;)Lkotlin/properties/ReadWriteProperty; - public static final fun stringList (Lspace/kscience/dataforge/meta/MutableMetaProvider;Lspace/kscience/dataforge/names/Name;)Lkotlin/properties/ReadWriteProperty; - public static final fun stringList (Lspace/kscience/dataforge/meta/MutableMetaProvider;[Ljava/lang/String;Lspace/kscience/dataforge/names/Name;)Lkotlin/properties/ReadWriteProperty; - public static synthetic fun stringList$default (Lspace/kscience/dataforge/meta/MutableMetaProvider;Lspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lkotlin/properties/ReadWriteProperty; - public static synthetic fun stringList$default (Lspace/kscience/dataforge/meta/MutableMetaProvider;[Ljava/lang/String;Lspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lkotlin/properties/ReadWriteProperty; - public static final fun value (Lspace/kscience/dataforge/meta/MutableMetaProvider;Lspace/kscience/dataforge/names/Name;)Lkotlin/properties/ReadWriteProperty; - public static final fun value (Lspace/kscience/dataforge/meta/MutableMetaProvider;Lspace/kscience/dataforge/names/Name;Lkotlin/jvm/functions/Function1;Lkotlin/jvm/functions/Function1;)Lkotlin/properties/ReadWriteProperty; - public static synthetic fun value$default (Lspace/kscience/dataforge/meta/MutableMetaProvider;Lspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lkotlin/properties/ReadWriteProperty; - public static synthetic fun value$default (Lspace/kscience/dataforge/meta/MutableMetaProvider;Lspace/kscience/dataforge/names/Name;Lkotlin/jvm/functions/Function1;Lkotlin/jvm/functions/Function1;ILjava/lang/Object;)Lkotlin/properties/ReadWriteProperty; + public static final fun boolean (Lspace/kscience/dataforge/meta/MutableMetaProvider;Lspace/kscience/dataforge/names/Name;)Lspace/kscience/dataforge/meta/MutableMetaDelegate; + public static final fun boolean (Lspace/kscience/dataforge/meta/MutableMetaProvider;Lspace/kscience/dataforge/names/Name;Lkotlin/jvm/functions/Function0;)Lspace/kscience/dataforge/meta/MutableMetaDelegate; + public static final fun boolean (Lspace/kscience/dataforge/meta/MutableMetaProvider;ZLspace/kscience/dataforge/names/Name;)Lspace/kscience/dataforge/meta/MutableMetaDelegate; + public static synthetic fun boolean$default (Lspace/kscience/dataforge/meta/MutableMetaProvider;Lspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lspace/kscience/dataforge/meta/MutableMetaDelegate; + public static synthetic fun boolean$default (Lspace/kscience/dataforge/meta/MutableMetaProvider;Lspace/kscience/dataforge/names/Name;Lkotlin/jvm/functions/Function0;ILjava/lang/Object;)Lspace/kscience/dataforge/meta/MutableMetaDelegate; + public static synthetic fun boolean$default (Lspace/kscience/dataforge/meta/MutableMetaProvider;ZLspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lspace/kscience/dataforge/meta/MutableMetaDelegate; + public static final fun convertable (Lspace/kscience/dataforge/meta/MutableMetaProvider;Lspace/kscience/dataforge/meta/MetaConverter;Lspace/kscience/dataforge/names/Name;)Lspace/kscience/dataforge/meta/MutableMetaDelegate; + public static synthetic fun convertable$default (Lspace/kscience/dataforge/meta/MutableMetaProvider;Lspace/kscience/dataforge/meta/MetaConverter;Lspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lspace/kscience/dataforge/meta/MutableMetaDelegate; + public static final fun double (Lspace/kscience/dataforge/meta/MutableMetaProvider;DLspace/kscience/dataforge/names/Name;)Lspace/kscience/dataforge/meta/MutableMetaDelegate; + public static final fun double (Lspace/kscience/dataforge/meta/MutableMetaProvider;Lspace/kscience/dataforge/names/Name;)Lspace/kscience/dataforge/meta/MutableMetaDelegate; + public static synthetic fun double$default (Lspace/kscience/dataforge/meta/MutableMetaProvider;DLspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lspace/kscience/dataforge/meta/MutableMetaDelegate; + public static synthetic fun double$default (Lspace/kscience/dataforge/meta/MutableMetaProvider;Lspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lspace/kscience/dataforge/meta/MutableMetaDelegate; + public static final fun float (Lspace/kscience/dataforge/meta/MutableMetaProvider;FLspace/kscience/dataforge/names/Name;)Lspace/kscience/dataforge/meta/MutableMetaDelegate; + public static final fun float (Lspace/kscience/dataforge/meta/MutableMetaProvider;Lspace/kscience/dataforge/names/Name;)Lspace/kscience/dataforge/meta/MutableMetaDelegate; + public static synthetic fun float$default (Lspace/kscience/dataforge/meta/MutableMetaProvider;FLspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lspace/kscience/dataforge/meta/MutableMetaDelegate; + public static synthetic fun float$default (Lspace/kscience/dataforge/meta/MutableMetaProvider;Lspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lspace/kscience/dataforge/meta/MutableMetaDelegate; + public static final fun int (Lspace/kscience/dataforge/meta/MutableMetaProvider;ILspace/kscience/dataforge/names/Name;)Lspace/kscience/dataforge/meta/MutableMetaDelegate; + public static final fun int (Lspace/kscience/dataforge/meta/MutableMetaProvider;Lspace/kscience/dataforge/names/Name;)Lspace/kscience/dataforge/meta/MutableMetaDelegate; + public static synthetic fun int$default (Lspace/kscience/dataforge/meta/MutableMetaProvider;ILspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lspace/kscience/dataforge/meta/MutableMetaDelegate; + public static synthetic fun int$default (Lspace/kscience/dataforge/meta/MutableMetaProvider;Lspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lspace/kscience/dataforge/meta/MutableMetaDelegate; + public static final fun listOfConvertable (Lspace/kscience/dataforge/meta/MutableMeta;Lspace/kscience/dataforge/meta/MetaConverter;Lspace/kscience/dataforge/names/Name;)Lspace/kscience/dataforge/meta/MutableMetaDelegate; + public static synthetic fun listOfConvertable$default (Lspace/kscience/dataforge/meta/MutableMeta;Lspace/kscience/dataforge/meta/MetaConverter;Lspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lspace/kscience/dataforge/meta/MutableMetaDelegate; + public static final fun listValue (Lspace/kscience/dataforge/meta/MutableMetaProvider;Lspace/kscience/dataforge/names/Name;Lkotlin/jvm/functions/Function1;Lkotlin/jvm/functions/Function1;)Lspace/kscience/dataforge/meta/MutableMetaDelegate; + public static synthetic fun listValue$default (Lspace/kscience/dataforge/meta/MutableMetaProvider;Lspace/kscience/dataforge/names/Name;Lkotlin/jvm/functions/Function1;Lkotlin/jvm/functions/Function1;ILjava/lang/Object;)Lspace/kscience/dataforge/meta/MutableMetaDelegate; + public static final fun long (Lspace/kscience/dataforge/meta/MutableMetaProvider;JLspace/kscience/dataforge/names/Name;)Lspace/kscience/dataforge/meta/MutableMetaDelegate; + public static final fun long (Lspace/kscience/dataforge/meta/MutableMetaProvider;Lspace/kscience/dataforge/names/Name;)Lspace/kscience/dataforge/meta/MutableMetaDelegate; + public static synthetic fun long$default (Lspace/kscience/dataforge/meta/MutableMetaProvider;JLspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lspace/kscience/dataforge/meta/MutableMetaDelegate; + public static synthetic fun long$default (Lspace/kscience/dataforge/meta/MutableMetaProvider;Lspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lspace/kscience/dataforge/meta/MutableMetaDelegate; + public static final fun node (Lspace/kscience/dataforge/meta/MutableMetaProvider;Lspace/kscience/dataforge/names/Name;Lspace/kscience/dataforge/meta/MetaConverter;)Lspace/kscience/dataforge/meta/MutableMetaDelegate; + public static final fun node (Lspace/kscience/dataforge/meta/MutableMetaProvider;Lspace/kscience/dataforge/names/Name;Lspace/kscience/dataforge/meta/descriptors/MetaDescriptor;)Lspace/kscience/dataforge/meta/MutableMetaDelegate; + public static synthetic fun node$default (Lspace/kscience/dataforge/meta/MutableMetaProvider;Lspace/kscience/dataforge/names/Name;Lspace/kscience/dataforge/meta/MetaConverter;ILjava/lang/Object;)Lspace/kscience/dataforge/meta/MutableMetaDelegate; + public static synthetic fun node$default (Lspace/kscience/dataforge/meta/MutableMetaProvider;Lspace/kscience/dataforge/names/Name;Lspace/kscience/dataforge/meta/descriptors/MetaDescriptor;ILjava/lang/Object;)Lspace/kscience/dataforge/meta/MutableMetaDelegate; + public static final fun number (Lspace/kscience/dataforge/meta/MutableMetaProvider;Ljava/lang/Number;Lspace/kscience/dataforge/names/Name;)Lspace/kscience/dataforge/meta/MutableMetaDelegate; + public static final fun number (Lspace/kscience/dataforge/meta/MutableMetaProvider;Lspace/kscience/dataforge/names/Name;)Lspace/kscience/dataforge/meta/MutableMetaDelegate; + public static final fun number (Lspace/kscience/dataforge/meta/MutableMetaProvider;Lspace/kscience/dataforge/names/Name;Lkotlin/jvm/functions/Function0;)Lspace/kscience/dataforge/meta/MutableMetaDelegate; + public static synthetic fun number$default (Lspace/kscience/dataforge/meta/MutableMetaProvider;Ljava/lang/Number;Lspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lspace/kscience/dataforge/meta/MutableMetaDelegate; + public static synthetic fun number$default (Lspace/kscience/dataforge/meta/MutableMetaProvider;Lspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lspace/kscience/dataforge/meta/MutableMetaDelegate; + public static synthetic fun number$default (Lspace/kscience/dataforge/meta/MutableMetaProvider;Lspace/kscience/dataforge/names/Name;Lkotlin/jvm/functions/Function0;ILjava/lang/Object;)Lspace/kscience/dataforge/meta/MutableMetaDelegate; + public static final fun numberList (Lspace/kscience/dataforge/meta/MutableMetaProvider;[Ljava/lang/Number;Lspace/kscience/dataforge/names/Name;)Lspace/kscience/dataforge/meta/MutableMetaDelegate; + public static synthetic fun numberList$default (Lspace/kscience/dataforge/meta/MutableMetaProvider;[Ljava/lang/Number;Lspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lspace/kscience/dataforge/meta/MutableMetaDelegate; + public static final fun string (Lspace/kscience/dataforge/meta/MutableMetaProvider;Ljava/lang/String;Lspace/kscience/dataforge/names/Name;)Lspace/kscience/dataforge/meta/MutableMetaDelegate; + public static final fun string (Lspace/kscience/dataforge/meta/MutableMetaProvider;Lspace/kscience/dataforge/names/Name;)Lspace/kscience/dataforge/meta/MutableMetaDelegate; + public static final fun string (Lspace/kscience/dataforge/meta/MutableMetaProvider;Lspace/kscience/dataforge/names/Name;Lkotlin/jvm/functions/Function0;)Lspace/kscience/dataforge/meta/MutableMetaDelegate; + public static synthetic fun string$default (Lspace/kscience/dataforge/meta/MutableMetaProvider;Ljava/lang/String;Lspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lspace/kscience/dataforge/meta/MutableMetaDelegate; + public static synthetic fun string$default (Lspace/kscience/dataforge/meta/MutableMetaProvider;Lspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lspace/kscience/dataforge/meta/MutableMetaDelegate; + public static synthetic fun string$default (Lspace/kscience/dataforge/meta/MutableMetaProvider;Lspace/kscience/dataforge/names/Name;Lkotlin/jvm/functions/Function0;ILjava/lang/Object;)Lspace/kscience/dataforge/meta/MutableMetaDelegate; + public static final fun stringList (Lspace/kscience/dataforge/meta/MutableMetaProvider;Lspace/kscience/dataforge/names/Name;)Lspace/kscience/dataforge/meta/MutableMetaDelegate; + public static final fun stringList (Lspace/kscience/dataforge/meta/MutableMetaProvider;[Ljava/lang/String;Lspace/kscience/dataforge/names/Name;)Lspace/kscience/dataforge/meta/MutableMetaDelegate; + public static synthetic fun stringList$default (Lspace/kscience/dataforge/meta/MutableMetaProvider;Lspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lspace/kscience/dataforge/meta/MutableMetaDelegate; + public static synthetic fun stringList$default (Lspace/kscience/dataforge/meta/MutableMetaProvider;[Ljava/lang/String;Lspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lspace/kscience/dataforge/meta/MutableMetaDelegate; + public static final fun value (Lspace/kscience/dataforge/meta/MutableMetaProvider;Lspace/kscience/dataforge/names/Name;Lkotlin/jvm/functions/Function1;Lspace/kscience/dataforge/meta/descriptors/MetaDescriptor;Lkotlin/jvm/functions/Function1;)Lspace/kscience/dataforge/meta/MutableMetaDelegate; + public static final fun value (Lspace/kscience/dataforge/meta/MutableMetaProvider;Lspace/kscience/dataforge/names/Name;Lspace/kscience/dataforge/meta/descriptors/MetaDescriptor;)Lspace/kscience/dataforge/meta/MutableMetaDelegate; + public static synthetic fun value$default (Lspace/kscience/dataforge/meta/MutableMetaProvider;Lspace/kscience/dataforge/names/Name;Lkotlin/jvm/functions/Function1;Lspace/kscience/dataforge/meta/descriptors/MetaDescriptor;Lkotlin/jvm/functions/Function1;ILjava/lang/Object;)Lspace/kscience/dataforge/meta/MutableMetaDelegate; + public static synthetic fun value$default (Lspace/kscience/dataforge/meta/MutableMetaProvider;Lspace/kscience/dataforge/names/Name;Lspace/kscience/dataforge/meta/descriptors/MetaDescriptor;ILjava/lang/Object;)Lspace/kscience/dataforge/meta/MutableMetaDelegate; } public final class space/kscience/dataforge/meta/MutableMetaKt { @@ -447,6 +477,7 @@ public final class space/kscience/dataforge/meta/MutableMetaKt { public static final fun getOrCreate (Lspace/kscience/dataforge/meta/MutableTypedMeta;Ljava/lang/String;)Lspace/kscience/dataforge/meta/MutableTypedMeta; public static final fun remove (Lspace/kscience/dataforge/meta/MutableMetaProvider;Ljava/lang/String;)V public static final fun remove (Lspace/kscience/dataforge/meta/MutableMetaProvider;Lspace/kscience/dataforge/names/Name;)V + public static final fun reset (Lspace/kscience/dataforge/meta/MutableMeta;Lspace/kscience/dataforge/meta/Meta;)V public static final fun set (Lspace/kscience/dataforge/meta/MutableMetaProvider;Ljava/lang/String;Ljava/lang/Iterable;)V public static final fun set (Lspace/kscience/dataforge/meta/MutableMetaProvider;Ljava/lang/String;Lspace/kscience/dataforge/meta/Meta;)V public static final fun set (Lspace/kscience/dataforge/meta/MutableMetaProvider;Lspace/kscience/dataforge/names/Name;Ljava/lang/Iterable;)V @@ -703,7 +734,6 @@ public final class space/kscience/dataforge/meta/Value$Companion { public final class space/kscience/dataforge/meta/ValueExtensionsKt { public static final fun getBoolean (Lspace/kscience/dataforge/meta/Value;)Z public static final fun getDouble (Lspace/kscience/dataforge/meta/Value;)D - public static final fun getDoubleArray (Lspace/kscience/dataforge/meta/Value;)[D public static final fun getFloat (Lspace/kscience/dataforge/meta/Value;)F public static final fun getInt (Lspace/kscience/dataforge/meta/Value;)I public static final fun getLong (Lspace/kscience/dataforge/meta/Value;)J @@ -722,7 +752,6 @@ public final class space/kscience/dataforge/meta/ValueKt { public static final fun asValue (Ljava/lang/Number;)Lspace/kscience/dataforge/meta/Value; public static final fun asValue (Ljava/lang/String;)Lspace/kscience/dataforge/meta/Value; public static final fun asValue (Z)Lspace/kscience/dataforge/meta/Value; - public static final fun asValue ([B)Lspace/kscience/dataforge/meta/Value; public static final fun asValue ([F)Lspace/kscience/dataforge/meta/Value; public static final fun asValue ([I)Lspace/kscience/dataforge/meta/Value; public static final fun asValue ([J)Lspace/kscience/dataforge/meta/Value; @@ -928,6 +957,9 @@ public final class space/kscience/dataforge/misc/NamedKt { public static final fun isAnonymous (Lspace/kscience/dataforge/misc/Named;)Z } +public abstract interface annotation class space/kscience/dataforge/misc/UnsafeKType : java/lang/annotation/Annotation { +} + public final class space/kscience/dataforge/names/Name { public static final field Companion Lspace/kscience/dataforge/names/Name$Companion; public static final field NAME_SEPARATOR Ljava/lang/String; @@ -945,6 +977,16 @@ public final class space/kscience/dataforge/names/Name$Companion { public final fun serializer ()Lkotlinx/serialization/KSerializer; } +public final class space/kscience/dataforge/names/NameIndexComparator : java/util/Comparator { + public static final field INSTANCE Lspace/kscience/dataforge/names/NameIndexComparator; + public synthetic fun compare (Ljava/lang/Object;Ljava/lang/Object;)I + public fun compare (Ljava/lang/String;Ljava/lang/String;)I +} + +public final class space/kscience/dataforge/names/NameIndexComparatorKt { + public static final fun getIndexedList (Lspace/kscience/dataforge/meta/Meta;Lspace/kscience/dataforge/names/Name;)Ljava/util/List; +} + public final class space/kscience/dataforge/names/NameKt { public static final fun appendFirst (Lspace/kscience/dataforge/names/Name;Ljava/lang/String;)Lspace/kscience/dataforge/names/Name; public static final fun appendLeft (Lspace/kscience/dataforge/names/Name;Ljava/lang/String;)Lspace/kscience/dataforge/names/Name; diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MetaDelegate.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MetaDelegate.kt index a24f7371..8c5a738f 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MetaDelegate.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MetaDelegate.kt @@ -1,15 +1,27 @@ package space.kscience.dataforge.meta +import space.kscience.dataforge.meta.descriptors.Described import space.kscience.dataforge.meta.descriptors.MetaDescriptor import space.kscience.dataforge.misc.DFExperimental import space.kscience.dataforge.names.Name import space.kscience.dataforge.names.asName import kotlin.properties.ReadOnlyProperty +import kotlin.reflect.KProperty /* Meta delegates */ -public fun MetaProvider.node(key: Name? = null): ReadOnlyProperty = ReadOnlyProperty { _, property -> - get(key ?: property.name.asName()) +public interface MetaDelegate : ReadOnlyProperty, Described + + +public fun MetaProvider.node( + key: Name? = null, + descriptor: MetaDescriptor? = null, +): MetaDelegate = object : MetaDelegate { + override val descriptor: MetaDescriptor? = descriptor + + override fun getValue(thisRef: Any?, property: KProperty<*>): Meta? { + return get(key ?: property.name.asName()) + } } /** @@ -18,8 +30,12 @@ public fun MetaProvider.node(key: Name? = null): ReadOnlyProperty = public fun MetaProvider.spec( metaSpec: MetaSpec, key: Name? = null, -): ReadOnlyProperty = ReadOnlyProperty { _, property -> - get(key ?: property.name.asName())?.let { metaSpec.read(it) } +): MetaDelegate = object : MetaDelegate { + override val descriptor: MetaDescriptor? get() = metaSpec.descriptor + + override fun getValue(thisRef: Any?, property: KProperty<*>): T? { + return get(key ?: property.name.asName())?.let { metaSpec.read(it) } + } } /** @@ -27,9 +43,9 @@ public fun MetaProvider.spec( */ @DFExperimental public inline fun MetaProvider.serializable( - descriptor: MetaDescriptor? = null, key: Name? = null, -): ReadOnlyProperty = spec(MetaConverter.serializable(descriptor), key) + descriptor: MetaDescriptor? = null, +): MetaDelegate = spec(MetaConverter.serializable(descriptor), key) @Deprecated("Use convertable", ReplaceWith("convertable(converter, key)")) public fun MetaProvider.node( @@ -43,78 +59,90 @@ public fun MetaProvider.node( public fun Meta.listOfSpec( converter: MetaSpec, key: Name? = null, -): ReadOnlyProperty> = ReadOnlyProperty{_, property -> - val name = key ?: property.name.asName() - getIndexed(name).values.map { converter.read(it) } +): MetaDelegate> = object : MetaDelegate> { + override fun getValue(thisRef: Any?, property: KProperty<*>): List { + val name = key ?: property.name.asName() + return getIndexed(name).values.map { converter.read(it) } + } + + override val descriptor: MetaDescriptor? = converter.descriptor?.copy(multiple = true) } @DFExperimental public inline fun Meta.listOfSerializable( - descriptor: MetaDescriptor? = null, key: Name? = null, -): ReadOnlyProperty> = listOfSpec(MetaConverter.serializable(descriptor), key) + descriptor: MetaDescriptor? = null, +): MetaDelegate> = listOfSpec(MetaConverter.serializable(descriptor), key) /** * A property delegate that uses custom key */ -public fun MetaProvider.value(key: Name? = null): ReadOnlyProperty = ReadOnlyProperty { _, property -> - get(key ?: property.name.asName())?.value +public fun MetaProvider.value( + key: Name? = null, + descriptor: MetaDescriptor? = null, +): MetaDelegate = object : MetaDelegate { + override fun getValue(thisRef: Any?, property: KProperty<*>): Value? = get(key ?: property.name.asName())?.value + + override val descriptor: MetaDescriptor? = descriptor } public fun MetaProvider.value( key: Name? = null, + descriptor: MetaDescriptor? = null, reader: (Value?) -> R, -): ReadOnlyProperty = ReadOnlyProperty { _, property -> - reader(get(key ?: property.name.asName())?.value) +): MetaDelegate = object : MetaDelegate { + override fun getValue(thisRef: Any?, property: KProperty<*>): R = reader(get(key ?: property.name.asName())?.value) + + override val descriptor: MetaDescriptor? = descriptor } //TODO add caching for sealed nodes /* Read-only delegates for [Meta] */ -public fun MetaProvider.string(key: Name? = null): ReadOnlyProperty = value(key) { it?.string } +public fun MetaProvider.string(key: Name? = null): MetaDelegate = value(key = key) { it?.string } -public fun MetaProvider.boolean(key: Name? = null): ReadOnlyProperty = value(key) { it?.boolean } +public fun MetaProvider.boolean(key: Name? = null): MetaDelegate = value(key = key) { it?.boolean } -public fun MetaProvider.number(key: Name? = null): ReadOnlyProperty = value(key) { it?.numberOrNull } +public fun MetaProvider.number(key: Name? = null): MetaDelegate = value(key = key) { it?.numberOrNull } -public fun MetaProvider.double(key: Name? = null): ReadOnlyProperty = value(key) { it?.double } +public fun MetaProvider.double(key: Name? = null): MetaDelegate = value(key = key) { it?.double } -public fun MetaProvider.float(key: Name? = null): ReadOnlyProperty = value(key) { it?.float } +public fun MetaProvider.float(key: Name? = null): MetaDelegate = value(key = key) { it?.float } -public fun MetaProvider.int(key: Name? = null): ReadOnlyProperty = value(key) { it?.int } +public fun MetaProvider.int(key: Name? = null): MetaDelegate = value(key = key) { it?.int } -public fun MetaProvider.long(key: Name? = null): ReadOnlyProperty = value(key) { it?.long } +public fun MetaProvider.long(key: Name? = null): MetaDelegate = value(key = key) { it?.long } -public fun MetaProvider.string(default: String, key: Name? = null): ReadOnlyProperty = - value(key) { it?.string ?: default } +public fun MetaProvider.string(default: String, key: Name? = null): MetaDelegate = + value(key = key) { it?.string ?: default } -public fun MetaProvider.boolean(default: Boolean, key: Name? = null): ReadOnlyProperty = - value(key) { it?.boolean ?: default } +public fun MetaProvider.boolean(default: Boolean, key: Name? = null): MetaDelegate = + value(key = key) { it?.boolean ?: default } -public fun MetaProvider.number(default: Number, key: Name? = null): ReadOnlyProperty = - value(key) { it?.numberOrNull ?: default } +public fun MetaProvider.number(default: Number, key: Name? = null): MetaDelegate = + value(key = key) { it?.numberOrNull ?: default } -public fun MetaProvider.double(default: Double, key: Name? = null): ReadOnlyProperty = - value(key) { it?.double ?: default } +public fun MetaProvider.double(default: Double, key: Name? = null): MetaDelegate = + value(key = key) { it?.double ?: default } -public fun MetaProvider.float(default: Float, key: Name? = null): ReadOnlyProperty = - value(key) { it?.float ?: default } +public fun MetaProvider.float(default: Float, key: Name? = null): MetaDelegate = + value(key = key) { it?.float ?: default } -public fun MetaProvider.int(default: Int, key: Name? = null): ReadOnlyProperty = - value(key) { it?.int ?: default } +public fun MetaProvider.int(default: Int, key: Name? = null): MetaDelegate = + value(key = key) { it?.int ?: default } -public fun MetaProvider.long(default: Long, key: Name? = null): ReadOnlyProperty = - value(key) { it?.long ?: default } +public fun MetaProvider.long(default: Long, key: Name? = null): MetaDelegate = + value(key = key) { it?.long ?: default } -public inline fun > MetaProvider.enum(default: E, key: Name? = null): ReadOnlyProperty = - value(key) { it?.enum() ?: default } +public inline fun > MetaProvider.enum(default: E, key: Name? = null): MetaDelegate = + value(key = key) { it?.enum() ?: default } -public fun MetaProvider.string(key: Name? = null, default: () -> String): ReadOnlyProperty = - value(key) { it?.string ?: default() } +public fun MetaProvider.string(key: Name? = null, default: () -> String): MetaDelegate = + value(key = key) { it?.string ?: default() } -public fun MetaProvider.boolean(key: Name? = null, default: () -> Boolean): ReadOnlyProperty = - value(key) { it?.boolean ?: default() } +public fun MetaProvider.boolean(key: Name? = null, default: () -> Boolean): MetaDelegate = + value(key = key) { it?.boolean ?: default() } -public fun MetaProvider.number(key: Name? = null, default: () -> Number): ReadOnlyProperty = - value(key) { it?.numberOrNull ?: default() } +public fun MetaProvider.number(key: Name? = null, default: () -> Number): MetaDelegate = + value(key = key) { it?.numberOrNull ?: default() } diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MutableMetaDelegate.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MutableMetaDelegate.kt index e308a63d..37140c6f 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MutableMetaDelegate.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MutableMetaDelegate.kt @@ -1,5 +1,6 @@ package space.kscience.dataforge.meta +import space.kscience.dataforge.meta.descriptors.Described import space.kscience.dataforge.meta.descriptors.MetaDescriptor import space.kscience.dataforge.misc.DFExperimental import space.kscience.dataforge.names.Name @@ -8,20 +9,28 @@ import space.kscience.dataforge.names.getIndexedList import kotlin.properties.ReadWriteProperty import kotlin.reflect.KProperty + /* Read-write delegates */ -public fun MutableMetaProvider.node(key: Name? = null): ReadWriteProperty = - object : ReadWriteProperty { - override fun getValue(thisRef: Any?, property: KProperty<*>): Meta? { - return get(key ?: property.name.asName()) - } +public interface MutableMetaDelegate : ReadWriteProperty, Described - override fun setValue(thisRef: Any?, property: KProperty<*>, value: Meta?) { - val name = key ?: property.name.asName() - set(name, value) - } +public fun MutableMetaProvider.node( + key: Name? = null, + descriptor: MetaDescriptor? = null, +): MutableMetaDelegate = object : MutableMetaDelegate { + + override val descriptor: MetaDescriptor? = descriptor + + override fun getValue(thisRef: Any?, property: KProperty<*>): Meta? { + return get(key ?: property.name.asName()) } + override fun setValue(thisRef: Any?, property: KProperty<*>, value: Meta?) { + val name = key ?: property.name.asName() + set(name, value) + } +} + /** * Use [converter] to transform an object to Meta and back. * Note that mutation of the object does not change Meta. @@ -29,21 +38,24 @@ public fun MutableMetaProvider.node(key: Name? = null): ReadWriteProperty MutableMetaProvider.convertable( converter: MetaConverter, key: Name? = null, -): ReadWriteProperty = - object : ReadWriteProperty { - override fun getValue(thisRef: Any?, property: KProperty<*>): T? { - val name = key ?: property.name.asName() - return get(name)?.let { converter.read(it) } - } +): MutableMetaDelegate = object : MutableMetaDelegate { - override fun setValue(thisRef: Any?, property: KProperty<*>, value: T?) { - val name = key ?: property.name.asName() - set(name, value?.let { converter.convert(it) }) - } + override val descriptor: MetaDescriptor? get() = converter.descriptor + + + override fun getValue(thisRef: Any?, property: KProperty<*>): T? { + val name = key ?: property.name.asName() + return get(name)?.let { converter.read(it) } } + override fun setValue(thisRef: Any?, property: KProperty<*>, value: T?) { + val name = key ?: property.name.asName() + set(name, value?.let { converter.convert(it) }) + } +} + @Deprecated("Use convertable", ReplaceWith("convertable(converter, key)")) -public fun MutableMetaProvider.node(key: Name? = null, converter: MetaConverter): ReadWriteProperty = +public fun MutableMetaProvider.node(key: Name? = null, converter: MetaConverter): MutableMetaDelegate = convertable(converter, key) /** @@ -54,7 +66,7 @@ public fun MutableMetaProvider.node(key: Name? = null, converter: MetaConver public inline fun MutableMetaProvider.serializable( descriptor: MetaDescriptor? = null, key: Name? = null, -): ReadWriteProperty = convertable(MetaConverter.serializable(descriptor), key) +): MutableMetaDelegate = convertable(MetaConverter.serializable(descriptor), key) /** * Use [converter] to convert a list of same name siblings meta to object and back. @@ -63,7 +75,9 @@ public inline fun MutableMetaProvider.serializable( public fun MutableMeta.listOfConvertable( converter: MetaConverter, key: Name? = null, -): ReadWriteProperty> = object : ReadWriteProperty> { +): MutableMetaDelegate> = object : MutableMetaDelegate> { + override val descriptor: MetaDescriptor? = converter.descriptor?.copy(multiple = true) + override fun getValue(thisRef: Any?, property: KProperty<*>): List { val name = key ?: property.name.asName() return getIndexedList(name).map { converter.read(it) } @@ -77,26 +91,33 @@ public fun MutableMeta.listOfConvertable( @DFExperimental public inline fun MutableMeta.listOfSerializable( - descriptor: MetaDescriptor? = null, key: Name? = null, -): ReadWriteProperty> = listOfConvertable(MetaConverter.serializable(descriptor), key) + descriptor: MetaDescriptor? = null, +): MutableMetaDelegate> = listOfConvertable(MetaConverter.serializable(descriptor), key) -public fun MutableMetaProvider.value(key: Name? = null): ReadWriteProperty = - object : ReadWriteProperty { - override fun getValue(thisRef: Any?, property: KProperty<*>): Value? = - get(key ?: property.name.asName())?.value +public fun MutableMetaProvider.value( + key: Name? = null, + descriptor: MetaDescriptor? = null, +): MutableMetaDelegate = object : MutableMetaDelegate { + override val descriptor: MetaDescriptor? = descriptor - override fun setValue(thisRef: Any?, property: KProperty<*>, value: Value?) { - setValue(key ?: property.name.asName(), value) - } + override fun getValue(thisRef: Any?, property: KProperty<*>): Value? = + get(key ?: property.name.asName())?.value + + override fun setValue(thisRef: Any?, property: KProperty<*>, value: Value?) { + setValue(key ?: property.name.asName(), value) } +} public fun MutableMetaProvider.value( key: Name? = null, writer: (T) -> Value? = { Value.of(it) }, + descriptor: MetaDescriptor? = null, reader: (Value?) -> T, -): ReadWriteProperty = object : ReadWriteProperty { +): MutableMetaDelegate = object : MutableMetaDelegate { + override val descriptor: MetaDescriptor? = descriptor + override fun getValue(thisRef: Any?, property: KProperty<*>): T = reader(get(key ?: property.name.asName())?.value) @@ -107,65 +128,65 @@ public fun MutableMetaProvider.value( /* Read-write delegates for [MutableItemProvider] */ -public fun MutableMetaProvider.string(key: Name? = null): ReadWriteProperty = +public fun MutableMetaProvider.string(key: Name? = null): MutableMetaDelegate = value(key) { it?.string } -public fun MutableMetaProvider.boolean(key: Name? = null): ReadWriteProperty = +public fun MutableMetaProvider.boolean(key: Name? = null): MutableMetaDelegate = value(key) { it?.boolean } -public fun MutableMetaProvider.number(key: Name? = null): ReadWriteProperty = +public fun MutableMetaProvider.number(key: Name? = null): MutableMetaDelegate = value(key) { it?.number } -public fun MutableMetaProvider.string(default: String, key: Name? = null): ReadWriteProperty = +public fun MutableMetaProvider.string(default: String, key: Name? = null): MutableMetaDelegate = value(key) { it?.string ?: default } -public fun MutableMetaProvider.boolean(default: Boolean, key: Name? = null): ReadWriteProperty = +public fun MutableMetaProvider.boolean(default: Boolean, key: Name? = null): MutableMetaDelegate = value(key) { it?.boolean ?: default } -public fun MutableMetaProvider.number(default: Number, key: Name? = null): ReadWriteProperty = +public fun MutableMetaProvider.number(default: Number, key: Name? = null): MutableMetaDelegate = value(key) { it?.number ?: default } -public fun MutableMetaProvider.string(key: Name? = null, default: () -> String): ReadWriteProperty = +public fun MutableMetaProvider.string(key: Name? = null, default: () -> String): MutableMetaDelegate = value(key) { it?.string ?: default() } -public fun MutableMetaProvider.boolean(key: Name? = null, default: () -> Boolean): ReadWriteProperty = +public fun MutableMetaProvider.boolean(key: Name? = null, default: () -> Boolean): MutableMetaDelegate = value(key) { it?.boolean ?: default() } -public fun MutableMetaProvider.number(key: Name? = null, default: () -> Number): ReadWriteProperty = +public fun MutableMetaProvider.number(key: Name? = null, default: () -> Number): MutableMetaDelegate = value(key) { it?.number ?: default() } public inline fun > MutableMetaProvider.enum( default: E, key: Name? = null, -): ReadWriteProperty = value(key) { value -> value?.string?.let { enumValueOf(it) } ?: default } +): MutableMetaDelegate = value(key) { value -> value?.string?.let { enumValueOf(it) } ?: default } /* Number delegates */ -public fun MutableMetaProvider.int(key: Name? = null): ReadWriteProperty = +public fun MutableMetaProvider.int(key: Name? = null): MutableMetaDelegate = value(key) { it?.int } -public fun MutableMetaProvider.double(key: Name? = null): ReadWriteProperty = +public fun MutableMetaProvider.double(key: Name? = null): MutableMetaDelegate = value(key) { it?.double } -public fun MutableMetaProvider.long(key: Name? = null): ReadWriteProperty = +public fun MutableMetaProvider.long(key: Name? = null): MutableMetaDelegate = value(key) { it?.long } -public fun MutableMetaProvider.float(key: Name? = null): ReadWriteProperty = +public fun MutableMetaProvider.float(key: Name? = null): MutableMetaDelegate = value(key) { it?.float } /* Safe number delegates*/ -public fun MutableMetaProvider.int(default: Int, key: Name? = null): ReadWriteProperty = +public fun MutableMetaProvider.int(default: Int, key: Name? = null): MutableMetaDelegate = value(key) { it?.int ?: default } -public fun MutableMetaProvider.double(default: Double, key: Name? = null): ReadWriteProperty = +public fun MutableMetaProvider.double(default: Double, key: Name? = null): MutableMetaDelegate = value(key) { it?.double ?: default } -public fun MutableMetaProvider.long(default: Long, key: Name? = null): ReadWriteProperty = +public fun MutableMetaProvider.long(default: Long, key: Name? = null): MutableMetaDelegate = value(key) { it?.long ?: default } -public fun MutableMetaProvider.float(default: Float, key: Name? = null): ReadWriteProperty = +public fun MutableMetaProvider.float(default: Float, key: Name? = null): MutableMetaDelegate = value(key) { it?.float ?: default } @@ -174,7 +195,7 @@ public fun MutableMetaProvider.float(default: Float, key: Name? = null): ReadWri public fun MutableMetaProvider.stringList( vararg default: String, key: Name? = null, -): ReadWriteProperty> = value( +): MutableMetaDelegate> = value( key, writer = { list -> list.map { str -> str.asValue() }.asValue() }, reader = { it?.stringList ?: listOf(*default) }, @@ -182,7 +203,7 @@ public fun MutableMetaProvider.stringList( public fun MutableMetaProvider.stringList( key: Name? = null, -): ReadWriteProperty?> = value( +): MutableMetaDelegate?> = value( key, writer = { it -> it?.map { str -> str.asValue() }?.asValue() }, reader = { it?.stringList }, @@ -191,7 +212,7 @@ public fun MutableMetaProvider.stringList( public fun MutableMetaProvider.numberList( vararg default: Number, key: Name? = null, -): ReadWriteProperty> = value( +): MutableMetaDelegate> = value( key, writer = { it.map { num -> num.asValue() }.asValue() }, reader = { it?.list?.map { value -> value.numberOrNull ?: Double.NaN } ?: listOf(*default) }, @@ -202,7 +223,7 @@ public fun MutableMetaProvider.listValue( key: Name? = null, writer: (T) -> Value = { Value.of(it) }, reader: (Value) -> T, -): ReadWriteProperty?> = value( +): MutableMetaDelegate?> = value( key, writer = { it?.map(writer)?.asValue() }, reader = { it?.list?.map(reader) } diff --git a/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/CachingAction.kt b/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/CachingAction.kt new file mode 100644 index 00000000..5f88ab74 --- /dev/null +++ b/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/CachingAction.kt @@ -0,0 +1,19 @@ +package space.kscience.dataforge.workspace + +import space.kscience.dataforge.actions.AbstractAction +import space.kscience.dataforge.data.* +import space.kscience.dataforge.meta.Meta +import kotlin.reflect.KType + +internal class CachingAction(type: KType, private val caching: (NamedData) -> NamedData) : + AbstractAction(type) { + override fun DataSink.generate(source: DataTree, meta: Meta) { + source.forEach { + put(caching(it)) + } + } + + override suspend fun DataSink.update(source: DataTree, meta: Meta, updatedData: DataUpdate) { + put(updatedData.name, updatedData.data?.named(updatedData.name)?.let(caching)) + } +} \ No newline at end of file diff --git a/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/FileWorkspaceCache.kt b/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/FileWorkspaceCache.kt index da09c095..4d2578e5 100644 --- a/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/FileWorkspaceCache.kt +++ b/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/FileWorkspaceCache.kt @@ -1,20 +1,22 @@ package space.kscience.dataforge.workspace -import kotlinx.coroutines.flow.filterIsInstance -import kotlinx.coroutines.flow.map import kotlinx.io.* import kotlinx.serialization.ExperimentalSerializationApi import kotlinx.serialization.KSerializer import kotlinx.serialization.json.Json import kotlinx.serialization.protobuf.ProtoBuf import kotlinx.serialization.serializer +import space.kscience.dataforge.actions.Action +import space.kscience.dataforge.actions.invoke import space.kscience.dataforge.context.error import space.kscience.dataforge.context.logger import space.kscience.dataforge.context.request -import space.kscience.dataforge.data.* +import space.kscience.dataforge.data.Data +import space.kscience.dataforge.data.await +import space.kscience.dataforge.data.named import space.kscience.dataforge.io.* import space.kscience.dataforge.misc.DFExperimental -import space.kscience.dataforge.misc.DFInternal +import space.kscience.dataforge.misc.UnsafeKType import space.kscience.dataforge.names.withIndex import java.nio.file.Path import kotlin.io.path.deleteIfExists @@ -52,7 +54,8 @@ public class FileWorkspaceCache(public val cacheDirectory: Path) : WorkspaceCach // private fun TaskData<*>.checkType(taskType: KType): TaskData = this as TaskData - @OptIn(DFExperimental::class, DFInternal::class) + + @OptIn(DFExperimental::class, UnsafeKType::class) override suspend fun cache(result: TaskResult): TaskResult { val io = result.workspace.context.request(IOPlugin) @@ -60,8 +63,8 @@ public class FileWorkspaceCache(public val cacheDirectory: Path) : WorkspaceCach ?: ProtobufIOFormat(result.dataType) ?: error("Can't resolve IOFormat for ${result.dataType}") - fun cacheOne(data: NamedData): NamedData { + val cachingAction: Action = CachingAction(result.dataType) { data -> val path = cacheDirectory / result.taskName.withIndex(result.taskMeta.hashCode().toString(16)).toString() / data.name.toString() @@ -80,7 +83,7 @@ public class FileWorkspaceCache(public val cacheDirectory: Path) : WorkspaceCach } } - //waiting for data in current scope because Envelope is synchronous + //waiting for data in the current scope because Envelope is synchronous return@Data data.await().also { result -> val envelope = Envelope { meta = data.meta @@ -92,12 +95,10 @@ public class FileWorkspaceCache(public val cacheDirectory: Path) : WorkspaceCach } } - return datum.named(data.name) + datum.named(data.name) } - - val cachedTree = result.asSequence().map { cacheOne(it) } - .toTree(result.dataType, result.updates.filterIsInstance>().map { cacheOne(it) }) + val cachedTree = cachingAction(result) return result.workspace.wrapResult(cachedTree, result.taskName, result.taskMeta) } diff --git a/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/InMemoryWorkspaceCache.kt b/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/InMemoryWorkspaceCache.kt index f3b2ee15..8ba39ec1 100644 --- a/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/InMemoryWorkspaceCache.kt +++ b/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/InMemoryWorkspaceCache.kt @@ -1,9 +1,11 @@ package space.kscience.dataforge.workspace -import kotlinx.coroutines.flow.filterIsInstance -import kotlinx.coroutines.flow.map -import space.kscience.dataforge.data.* +import space.kscience.dataforge.actions.Action +import space.kscience.dataforge.actions.invoke +import space.kscience.dataforge.data.Data +import space.kscience.dataforge.data.named import space.kscience.dataforge.meta.Meta +import space.kscience.dataforge.misc.DFExperimental import space.kscience.dataforge.names.Name import kotlin.reflect.KType import kotlin.reflect.full.isSubtypeOf @@ -20,19 +22,18 @@ public class InMemoryWorkspaceCache : WorkspaceCache { if (type.isSubtypeOf(taskType)) this as Data else error("Cached data type mismatch: expected $taskType but got $type") + @OptIn(DFExperimental::class) override suspend fun cache(result: TaskResult): TaskResult { - fun cacheOne(data: NamedData): NamedData { + val cachingAction: Action = CachingAction(result.dataType) { data -> val cachedData = cache.getOrPut(TaskResultId(result.taskName, result.taskMeta)){ HashMap() }.getOrPut(data.name){ data.data } - return cachedData.checkType(result.dataType).named(data.name) + cachedData.checkType(result.dataType).named(data.name) } - - val cachedTree = result.asSequence().map { cacheOne(it) } - .toTree(result.dataType, result.updates.filterIsInstance>().map { cacheOne(it) }) + val cachedTree = cachingAction(result) return result.workspace.wrapResult(cachedTree, result.taskName, result.taskMeta) } diff --git a/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/SimpleWorkspaceTest.kt b/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/SimpleWorkspaceTest.kt index 837b61bc..39837c15 100644 --- a/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/SimpleWorkspaceTest.kt +++ b/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/SimpleWorkspaceTest.kt @@ -148,16 +148,16 @@ internal class SimpleWorkspaceTest { } @Test - fun testWorkspace() = runTest(timeout = 10.milliseconds) { + fun testWorkspace() = runTest(timeout = 100.milliseconds) { val node = workspace.produce("sum") val res = node.asSequence().single() assertEquals(328350, res.await()) } @Test - fun testMetaPropagation() = runTest(timeout = 10.milliseconds) { + fun testMetaPropagation() = runTest(timeout = 100.milliseconds) { val node = workspace.produce("sum") { "testFlag" put true } - val res = node.data!!.await() + val res = node["sum"]!!.await() } @Test @@ -186,7 +186,7 @@ internal class SimpleWorkspaceTest { val node = workspace.produce("filterOne") { "name" put "myData[12]" } - assertEquals(12, node.data!!.await()) + assertEquals(12, node.asSequence().first().await()) } } \ No newline at end of file -- 2.34.1 From f840ffb473a698ccf8af08dd28ad22078642dcc9 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Sat, 27 Apr 2024 09:47:08 +0300 Subject: [PATCH 43/62] Move data tests to jvm to avoid random failings on native --- .../dataforge/descriptors/reflectiveDescriptors.kt | 7 +------ .../kotlin/space/kscience/dataforge/data/ActionsTest.kt | 0 .../dataforge/meta/descriptors/MetaDescriptorBuilder.kt | 1 + 3 files changed, 2 insertions(+), 6 deletions(-) rename dataforge-data/src/{commonTest => jvmTest}/kotlin/space/kscience/dataforge/data/ActionsTest.kt (100%) diff --git a/dataforge-context/src/jvmMain/kotlin/space/kscience/dataforge/descriptors/reflectiveDescriptors.kt b/dataforge-context/src/jvmMain/kotlin/space/kscience/dataforge/descriptors/reflectiveDescriptors.kt index 9c36410a..b2953018 100644 --- a/dataforge-context/src/jvmMain/kotlin/space/kscience/dataforge/descriptors/reflectiveDescriptors.kt +++ b/dataforge-context/src/jvmMain/kotlin/space/kscience/dataforge/descriptors/reflectiveDescriptors.kt @@ -79,7 +79,7 @@ public fun MetaDescriptor.Companion.forClass( is DescriptorUrl -> loadDescriptorFromUrl(URL(it.url)) } } - kClass.memberProperties.forEach { property-> + kClass.memberProperties.forEach { property -> var flag = false @@ -88,11 +88,6 @@ public fun MetaDescriptor.Companion.forClass( (property.returnType.classifier as? KClass<*>)?.let { from(forClass(it)) } -// -// (property.getDelegate(Unit) as? MetaDelegate<*>)?.descriptor?.let { -// from(it) -// flag = true -// } property.annotations.forEach { when (it) { diff --git a/dataforge-data/src/commonTest/kotlin/space/kscience/dataforge/data/ActionsTest.kt b/dataforge-data/src/jvmTest/kotlin/space/kscience/dataforge/data/ActionsTest.kt similarity index 100% rename from dataforge-data/src/commonTest/kotlin/space/kscience/dataforge/data/ActionsTest.kt rename to dataforge-data/src/jvmTest/kotlin/space/kscience/dataforge/data/ActionsTest.kt diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/descriptors/MetaDescriptorBuilder.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/descriptors/MetaDescriptorBuilder.kt index 2291e3d3..dfdbfeab 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/descriptors/MetaDescriptorBuilder.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/descriptors/MetaDescriptorBuilder.kt @@ -47,6 +47,7 @@ public class MetaDescriptorBuilder @PublishedApi internal constructor() { ): Unit { when (name.length) { 0 -> error("Can't set descriptor to root") + 1 -> { children[name.first().body] = descriptorBuilder } -- 2.34.1 From 2cd18854209cdf332e14581fa7c2cf7145dcf25c Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Sat, 27 Apr 2024 10:01:36 +0300 Subject: [PATCH 44/62] Update readme and changelog --- CHANGELOG.md | 24 +++++++++++++++++------- build.gradle.kts | 3 ++- dataforge-context/README.md | 4 ++-- dataforge-data/README.md | 4 ++-- dataforge-io/README.md | 4 ++-- dataforge-io/dataforge-io-yaml/README.md | 4 ++-- dataforge-meta/README.md | 4 ++-- dataforge-scripting/README.md | 4 ++-- dataforge-workspace/README.md | 4 ++-- 9 files changed, 33 insertions(+), 22 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a56e56e6..a94cb585 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,26 +3,36 @@ ## Unreleased ### Added -- Name index comparator -- Specialized ByteArrayValue ### Changed -- DataSink `branch` is replaced with `putAll` to avoid confusion with DataTree methods -- Meta delegate now uses a specific class that has a descriptor ### Deprecated ### Removed ### Fixed + +### Security + +## 0.8.2 - 2024-04-27 + +### Added + +- Name index comparator +- Specialized ByteArrayValue + +### Changed + +- DataSink `branch` is replaced with `putAll` to avoid confusion with DataTree methods +- Meta delegate now uses a specific class that has a descriptor + +### Fixed + - `listOfScheme` and `listOfConvertable` delegates provides correct items order. - Scheme meta setter works with proper sub-branch. - NameToken.parse improper work with indices. - Proper data handling for cache. - -### Security - ## 0.8.0 - 2024-02-03 ### Added diff --git a/build.gradle.kts b/build.gradle.kts index 78dc6d35..4db098be 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -4,11 +4,12 @@ import space.kscience.gradle.useSPCTeam plugins { id("space.kscience.gradle.project") + id("org.jetbrains.kotlinx.kover") version "0.7.6" } allprojects { group = "space.kscience" - version = "0.8.2-dev-1" + version = "0.8.2" } subprojects { diff --git a/dataforge-context/README.md b/dataforge-context/README.md index f0aff459..7970751c 100644 --- a/dataforge-context/README.md +++ b/dataforge-context/README.md @@ -6,7 +6,7 @@ Context and provider definitions ## Artifact: -The Maven coordinates of this project are `space.kscience:dataforge-context:0.8.0`. +The Maven coordinates of this project are `space.kscience:dataforge-context:0.8.2`. **Gradle Kotlin DSL:** ```kotlin @@ -16,6 +16,6 @@ repositories { } dependencies { - implementation("space.kscience:dataforge-context:0.8.0") + implementation("space.kscience:dataforge-context:0.8.2") } ``` diff --git a/dataforge-data/README.md b/dataforge-data/README.md index d77ed1b9..f08a43c2 100644 --- a/dataforge-data/README.md +++ b/dataforge-data/README.md @@ -6,7 +6,7 @@ ## Artifact: -The Maven coordinates of this project are `space.kscience:dataforge-data:0.8.0`. +The Maven coordinates of this project are `space.kscience:dataforge-data:0.8.2`. **Gradle Kotlin DSL:** ```kotlin @@ -16,6 +16,6 @@ repositories { } dependencies { - implementation("space.kscience:dataforge-data:0.8.0") + implementation("space.kscience:dataforge-data:0.8.2") } ``` diff --git a/dataforge-io/README.md b/dataforge-io/README.md index 85e49e5a..5168797b 100644 --- a/dataforge-io/README.md +++ b/dataforge-io/README.md @@ -6,7 +6,7 @@ IO module ## Artifact: -The Maven coordinates of this project are `space.kscience:dataforge-io:0.8.0`. +The Maven coordinates of this project are `space.kscience:dataforge-io:0.8.2`. **Gradle Kotlin DSL:** ```kotlin @@ -16,6 +16,6 @@ repositories { } dependencies { - implementation("space.kscience:dataforge-io:0.8.0") + implementation("space.kscience:dataforge-io:0.8.2") } ``` diff --git a/dataforge-io/dataforge-io-yaml/README.md b/dataforge-io/dataforge-io-yaml/README.md index 20f5b4f6..cc1ecd8c 100644 --- a/dataforge-io/dataforge-io-yaml/README.md +++ b/dataforge-io/dataforge-io-yaml/README.md @@ -6,7 +6,7 @@ YAML meta IO ## Artifact: -The Maven coordinates of this project are `space.kscience:dataforge-io-yaml:0.8.0`. +The Maven coordinates of this project are `space.kscience:dataforge-io-yaml:0.8.2`. **Gradle Kotlin DSL:** ```kotlin @@ -16,6 +16,6 @@ repositories { } dependencies { - implementation("space.kscience:dataforge-io-yaml:0.8.0") + implementation("space.kscience:dataforge-io-yaml:0.8.2") } ``` diff --git a/dataforge-meta/README.md b/dataforge-meta/README.md index bd11ebf1..a3209760 100644 --- a/dataforge-meta/README.md +++ b/dataforge-meta/README.md @@ -6,7 +6,7 @@ Meta definition and basic operations on meta ## Artifact: -The Maven coordinates of this project are `space.kscience:dataforge-meta:0.8.0`. +The Maven coordinates of this project are `space.kscience:dataforge-meta:0.8.2`. **Gradle Kotlin DSL:** ```kotlin @@ -16,6 +16,6 @@ repositories { } dependencies { - implementation("space.kscience:dataforge-meta:0.8.0") + implementation("space.kscience:dataforge-meta:0.8.2") } ``` diff --git a/dataforge-scripting/README.md b/dataforge-scripting/README.md index fbc5cb69..3167e1b4 100644 --- a/dataforge-scripting/README.md +++ b/dataforge-scripting/README.md @@ -6,7 +6,7 @@ ## Artifact: -The Maven coordinates of this project are `space.kscience:dataforge-scripting:0.8.0`. +The Maven coordinates of this project are `space.kscience:dataforge-scripting:0.8.2`. **Gradle Kotlin DSL:** ```kotlin @@ -16,6 +16,6 @@ repositories { } dependencies { - implementation("space.kscience:dataforge-scripting:0.8.0") + implementation("space.kscience:dataforge-scripting:0.8.2") } ``` diff --git a/dataforge-workspace/README.md b/dataforge-workspace/README.md index cea37368..f7461f30 100644 --- a/dataforge-workspace/README.md +++ b/dataforge-workspace/README.md @@ -6,7 +6,7 @@ ## Artifact: -The Maven coordinates of this project are `space.kscience:dataforge-workspace:0.8.0`. +The Maven coordinates of this project are `space.kscience:dataforge-workspace:0.8.2`. **Gradle Kotlin DSL:** ```kotlin @@ -16,6 +16,6 @@ repositories { } dependencies { - implementation("space.kscience:dataforge-workspace:0.8.0") + implementation("space.kscience:dataforge-workspace:0.8.2") } ``` -- 2.34.1 From 3de5691c84dab040e7e570567b8b1fa38228b744 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Sat, 25 May 2024 21:19:06 +0300 Subject: [PATCH 45/62] Add custom coroutine context during new Context creation --- .../space/kscience/dataforge/context/Context.kt | 4 +++- .../kscience/dataforge/context/ContextBuilder.kt | 13 +++++++++++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/context/Context.kt b/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/context/Context.kt index bb74d605..c614598c 100644 --- a/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/context/Context.kt +++ b/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/context/Context.kt @@ -10,6 +10,7 @@ import space.kscience.dataforge.misc.ThreadSafe import space.kscience.dataforge.names.Name import space.kscience.dataforge.provider.Provider import kotlin.coroutines.CoroutineContext +import kotlin.coroutines.EmptyCoroutineContext /** * The local environment for anything being done in DataForge framework. Contexts are organized into tree structure with [Global] at the top. @@ -26,6 +27,7 @@ public open class Context internal constructor( public val parent: Context?, plugins: Set, // set of unattached plugins meta: Meta, + coroutineContext: CoroutineContext = EmptyCoroutineContext, ) : Named, MetaRepr, Provider, CoroutineScope { /** @@ -65,7 +67,7 @@ public open class Context internal constructor( override val coroutineContext: CoroutineContext by lazy { (parent ?: Global).coroutineContext.let { parenContext -> - parenContext + SupervisorJob(parenContext[Job]) + parenContext + coroutineContext + SupervisorJob(parenContext[Job]) } } diff --git a/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/context/ContextBuilder.kt b/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/context/ContextBuilder.kt index c0db4314..894c5f15 100644 --- a/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/context/ContextBuilder.kt +++ b/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/context/ContextBuilder.kt @@ -13,6 +13,8 @@ import space.kscience.dataforge.names.plus import kotlin.collections.component1 import kotlin.collections.component2 import kotlin.collections.set +import kotlin.coroutines.CoroutineContext +import kotlin.coroutines.EmptyCoroutineContext /** * A convenience builder for context @@ -59,8 +61,15 @@ public class ContextBuilder internal constructor( plugin(DeFactoPluginFactory(plugin)) } + private var coroutineContext: CoroutineContext = EmptyCoroutineContext + + public fun coroutineContext(coroutineContext: CoroutineContext) { + this.coroutineContext = coroutineContext + } + + public fun build(): Context { - val contextName = name ?: NameToken("@auto",hashCode().toUInt().toString(16)).asName() + val contextName = name ?: NameToken("@auto", hashCode().toUInt().toString(16)).asName() val plugins = HashMap() fun addPlugin(factory: PluginFactory<*>, meta: Meta) { @@ -86,7 +95,7 @@ public class ContextBuilder internal constructor( addPlugin(factory, meta) } - return Context(contextName, parent, plugins.values.toSet(), meta.seal()) + return Context(contextName, parent, plugins.values.toSet(), meta.seal(), coroutineContext) } } -- 2.34.1 From 5a75b05acd8893371fefb60e8190e5e688c3a014 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Tue, 4 Jun 2024 17:44:44 +0300 Subject: [PATCH 46/62] 0.9-RC --- CHANGELOG.md | 5 + build.gradle.kts | 10 +- .../dataforge/properties/metaAsFlow.kt | 2 +- .../descriptors/reflectiveDescriptors.kt | 129 +++++++++--------- .../descriptors/TestAutoDescriptors.kt | 53 ++++--- .../kscience/dataforge/data/dataTransform.kt | 18 +++ .../kscience/dataforge/data/ActionsTest.kt | 1 + .../space/kscience/dataforge/meta/Laminate.kt | 2 + .../space/kscience/dataforge/meta/Meta.kt | 8 +- .../kscience/dataforge/meta/MetaConverter.kt | 2 +- .../kscience/dataforge/meta/MetaDelegate.kt | 12 +- .../meta/{MetaSpec.kt => MetaReader.kt} | 8 +- .../space/kscience/dataforge/meta/MetaRef.kt | 64 +++++++++ .../kscience/dataforge/meta/MutableMeta.kt | 3 + .../kscience/dataforge/meta/ObservableMeta.kt | 3 + .../dataforge/meta/ObservableMetaWrapper.kt | 3 + .../space/kscience/dataforge/meta/Scheme.kt | 11 +- .../kscience/dataforge/meta/SealedMeta.kt | 3 + .../meta/descriptors/MetaDescriptorBuilder.kt | 2 +- .../space/kscience/dataforge/misc/cast.kt | 3 - .../kscience/dataforge/meta/DynamicMeta.kt | 3 + .../space/kscience/dataforge/misc/castJs.kt | 5 - .../space/kscience/dataforge/misc/castJvm.kt | 4 - .../kscience/dataforge/misc/castNative.kt | 4 - .../space/kscience/dataforge/misc/castWasm.kt | 4 - .../kscience/dataforge/workspace/Task.kt | 12 +- .../dataforge/workspace/WorkspaceBuilder.kt | 4 +- gradle.properties | 7 +- 28 files changed, 234 insertions(+), 151 deletions(-) rename dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/{MetaSpec.kt => MetaReader.kt} (59%) create mode 100644 dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MetaRef.kt delete mode 100644 dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/misc/cast.kt delete mode 100644 dataforge-meta/src/jsMain/kotlin/space/kscience/dataforge/misc/castJs.kt delete mode 100644 dataforge-meta/src/jvmMain/kotlin/space/kscience/dataforge/misc/castJvm.kt delete mode 100644 dataforge-meta/src/nativeMain/kotlin/space/kscience/dataforge/misc/castNative.kt delete mode 100644 dataforge-meta/src/wasmJsMain/kotlin/space/kscience/dataforge/misc/castWasm.kt diff --git a/CHANGELOG.md b/CHANGELOG.md index a94cb585..d75dd1a2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,12 +3,17 @@ ## Unreleased ### Added +- Custom CoroutineContext during `Context` creation. ### Changed +- Kotlin 2.0 +- `MetaSpec` renamed to `MetaReader`. MetaSpec is now reserved for builder-based generation of meta descriptors. +- Add self-type for Meta. Remove unsafe cast method for meta instances. ### Deprecated ### Removed +- Automatic descriptors for schema. It is not possible to implement them without heavy reflection. ### Fixed diff --git a/build.gradle.kts b/build.gradle.kts index 4db098be..815a7077 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -3,21 +3,21 @@ import space.kscience.gradle.useApache2Licence import space.kscience.gradle.useSPCTeam plugins { - id("space.kscience.gradle.project") - id("org.jetbrains.kotlinx.kover") version "0.7.6" + alias(spclibs.plugins.kscience.project) + alias(spclibs.plugins.kotlinx.kover) } allprojects { group = "space.kscience" - version = "0.8.2" + version = "0.9.0-dev-1" } subprojects { apply(plugin = "maven-publish") tasks.withType { - kotlinOptions { - freeCompilerArgs = freeCompilerArgs + "-Xcontext-receivers" + compilerOptions { + freeCompilerArgs.add("-Xcontext-receivers") } } } diff --git a/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/properties/metaAsFlow.kt b/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/properties/metaAsFlow.kt index 90fafc5e..da539fcb 100644 --- a/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/properties/metaAsFlow.kt +++ b/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/properties/metaAsFlow.kt @@ -10,7 +10,7 @@ import space.kscience.dataforge.meta.* import space.kscience.dataforge.misc.DFExperimental @DFExperimental -public fun ObservableMeta.asFlow(converter: MetaSpec): Flow = callbackFlow { +public fun ObservableMeta.asFlow(converter: MetaReader): Flow = callbackFlow { onChange(this){ trySend(converter.read(this)) } diff --git a/dataforge-context/src/jvmMain/kotlin/space/kscience/dataforge/descriptors/reflectiveDescriptors.kt b/dataforge-context/src/jvmMain/kotlin/space/kscience/dataforge/descriptors/reflectiveDescriptors.kt index b2953018..590324d7 100644 --- a/dataforge-context/src/jvmMain/kotlin/space/kscience/dataforge/descriptors/reflectiveDescriptors.kt +++ b/dataforge-context/src/jvmMain/kotlin/space/kscience/dataforge/descriptors/reflectiveDescriptors.kt @@ -4,18 +4,12 @@ import kotlinx.serialization.ExperimentalSerializationApi import kotlinx.serialization.json.Json import kotlinx.serialization.json.decodeFromStream import org.slf4j.LoggerFactory -import space.kscience.dataforge.meta.Scheme -import space.kscience.dataforge.meta.SchemeSpec -import space.kscience.dataforge.meta.ValueType import space.kscience.dataforge.meta.descriptors.MetaDescriptor import space.kscience.dataforge.meta.descriptors.MetaDescriptorBuilder -import space.kscience.dataforge.meta.descriptors.node import space.kscience.dataforge.misc.DFExperimental import java.net.URL -import kotlin.reflect.KClass -import kotlin.reflect.full.isSubclassOf -import kotlin.reflect.full.memberProperties - +import kotlin.reflect.KAnnotatedElement +import kotlin.reflect.KProperty /** * Description text for meta property, node or whole object @@ -59,18 +53,8 @@ private fun MetaDescriptorBuilder.loadDescriptorFromResource(resource: Descripto } @DFExperimental -public fun MetaDescriptor.Companion.forClass( - kClass: KClass, - mod: MetaDescriptorBuilder.() -> Unit = {}, -): MetaDescriptor = MetaDescriptor { - when { - kClass.isSubclassOf(Number::class) -> valueType(ValueType.NUMBER) - kClass == String::class -> ValueType.STRING - kClass == Boolean::class -> ValueType.BOOLEAN - kClass == DoubleArray::class -> ValueType.LIST - } - - kClass.annotations.forEach { +public fun MetaDescriptorBuilder.forAnnotatedElement(element: KAnnotatedElement) { + element.annotations.forEach { when (it) { is Description -> description = it.value @@ -79,47 +63,70 @@ public fun MetaDescriptor.Companion.forClass( is DescriptorUrl -> loadDescriptorFromUrl(URL(it.url)) } } - kClass.memberProperties.forEach { property -> - - var flag = false - - val descriptor = MetaDescriptor { - //use base type descriptor as a base - (property.returnType.classifier as? KClass<*>)?.let { - from(forClass(it)) - } - - property.annotations.forEach { - when (it) { - is Description -> { - description = it.value - flag = true - } - - is Multiple -> { - multiple = true - flag = true - } - - is DescriptorResource -> { - loadDescriptorFromResource(it) - flag = true - } - - is DescriptorUrl -> { - loadDescriptorFromUrl(URL(it.url)) - flag = true - } - } - } - } - if (flag) { - node(property.name, descriptor) - } - } - mod() } @DFExperimental -public inline fun SchemeSpec.autoDescriptor(noinline mod: MetaDescriptorBuilder.() -> Unit = {}): MetaDescriptor = - MetaDescriptor.forClass(T::class, mod) \ No newline at end of file +public fun MetaDescriptorBuilder.forProperty(property: KProperty<*>) { + property.annotations.forEach { + when (it) { + is Description -> description = it.value + + is DescriptorResource -> loadDescriptorFromResource(it) + + is DescriptorUrl -> loadDescriptorFromUrl(URL(it.url)) + } + } +} +// +//@DFExperimental +//public fun MetaDescriptor.Companion.forScheme( +// spec: SchemeSpec, +// mod: MetaDescriptorBuilder.() -> Unit = {}, +//): MetaDescriptor = MetaDescriptor { +// val scheme = spec.empty() +// val kClass: KClass = scheme::class as KClass +// when { +// kClass.isSubclassOf(Number::class) -> valueType(ValueType.NUMBER) +// kClass == String::class -> ValueType.STRING +// kClass == Boolean::class -> ValueType.BOOLEAN +// kClass == DoubleArray::class -> ValueType.LIST +// kClass == ByteArray::class -> ValueType.LIST +// } +// +// forAnnotatedElement(kClass) +// kClass.memberProperties.forEach { property -> +// node(property.name) { +// +// (property.getDelegate(scheme) as? MetaDelegate<*>)?.descriptor?.let { +// from(it) +// } +// +// property.annotations.forEach { +// when (it) { +// is Description -> { +// description = it.value +// } +// +// is Multiple -> { +// multiple = true +// } +// +// is DescriptorResource -> { +// loadDescriptorFromResource(it) +// } +// +// is DescriptorUrl -> { +// loadDescriptorFromUrl(URL(it.url)) +// } +// } +// } +// } +// +// } +// mod() +//} +// +//@DFExperimental +//public inline fun SchemeSpec.autoDescriptor( +// noinline mod: MetaDescriptorBuilder.() -> Unit = {}, +//): MetaDescriptor = MetaDescriptor.forScheme(this, mod) \ No newline at end of file diff --git a/dataforge-context/src/jvmTest/kotlin/space/kscience/dataforge/descriptors/TestAutoDescriptors.kt b/dataforge-context/src/jvmTest/kotlin/space/kscience/dataforge/descriptors/TestAutoDescriptors.kt index df849ce5..3b1fce3d 100644 --- a/dataforge-context/src/jvmTest/kotlin/space/kscience/dataforge/descriptors/TestAutoDescriptors.kt +++ b/dataforge-context/src/jvmTest/kotlin/space/kscience/dataforge/descriptors/TestAutoDescriptors.kt @@ -2,35 +2,28 @@ package space.kscience.dataforge.descriptors -import kotlinx.serialization.encodeToString -import kotlinx.serialization.json.Json -import org.junit.jupiter.api.Test -import space.kscience.dataforge.meta.Scheme -import space.kscience.dataforge.meta.SchemeSpec -import space.kscience.dataforge.meta.descriptors.MetaDescriptor -import space.kscience.dataforge.meta.int -import space.kscience.dataforge.meta.string import space.kscience.dataforge.misc.DFExperimental -private class TestScheme : Scheme() { - - @Description("A") - val a by string() - - @Description("B") - val b by int() - - val c by int() - - companion object : SchemeSpec(::TestScheme) { - override val descriptor: MetaDescriptor = autoDescriptor() - } -} - -class TestAutoDescriptors { - @Test - fun autoDescriptor() { - val autoDescriptor = MetaDescriptor.forClass(TestScheme::class) - println(Json { prettyPrint = true }.encodeToString(autoDescriptor)) - } -} \ No newline at end of file +// +//class TestScheme : Scheme() { +// +// @Description("A") +// val a by string() +// +// @Description("B") +// val b by int() +// +// val c by int() +// +// companion object : SchemeSpec(::TestScheme) { +// override val descriptor: MetaDescriptor = autoDescriptor() +// } +//} +// +//class TestAutoDescriptors { +// @Test +// fun autoDescriptor() { +// val autoDescriptor = MetaDescriptor.forScheme(TestScheme) +// println(Json { prettyPrint = true }.encodeToString(autoDescriptor)) +// } +//} \ No newline at end of file diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataTransform.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataTransform.kt index ab54eb3e..c0d92e9e 100644 --- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataTransform.kt +++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataTransform.kt @@ -18,6 +18,24 @@ public data class NamedValueWithMeta(val name: Name, val value: T, val meta: public suspend fun NamedData.awaitWithMeta(): NamedValueWithMeta = NamedValueWithMeta(name, await(), meta) +/** + * Lazily transform this data to another data. By convention [block] should not use external data (be pure). + * @param type explicit type of the resulting [Data] + * @param coroutineContext additional [CoroutineContext] elements used for data computation. + * @param meta for the resulting data. By default equals input data. + * @param block the transformation itself + */ +@UnsafeKType +public fun Data.transform( + type: KType, + meta: Meta = this.meta, + coroutineContext: CoroutineContext = EmptyCoroutineContext, + block: suspend (T) -> R, +): Data = Data(type, meta, coroutineContext, listOf(this)) { + block(await()) +} + + /** * Lazily transform this data to another data. By convention [block] should not use external data (be pure). diff --git a/dataforge-data/src/jvmTest/kotlin/space/kscience/dataforge/data/ActionsTest.kt b/dataforge-data/src/jvmTest/kotlin/space/kscience/dataforge/data/ActionsTest.kt index 477ca592..13660eee 100644 --- a/dataforge-data/src/jvmTest/kotlin/space/kscience/dataforge/data/ActionsTest.kt +++ b/dataforge-data/src/jvmTest/kotlin/space/kscience/dataforge/data/ActionsTest.kt @@ -43,6 +43,7 @@ internal class ActionsTest { repeat(10) { source.updateValue(it.toString(), it) } + result.updates.take(10).onEach { println(it.name) }.collect() assertEquals(2, result["1"]?.await()) diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/Laminate.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/Laminate.kt index 87284107..0ae84bb6 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/Laminate.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/Laminate.kt @@ -9,6 +9,8 @@ import space.kscience.dataforge.names.NameToken */ public class Laminate internal constructor(public val layers: List) : TypedMeta { + override val self: Laminate get() = this + override val value: Value? = layers.firstNotNullOfOrNull { it.value } override val items: Map by lazy { diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/Meta.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/Meta.kt index 5cf53c75..fd953085 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/Meta.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/Meta.kt @@ -3,7 +3,6 @@ package space.kscience.dataforge.meta import kotlinx.serialization.Serializable import kotlinx.serialization.json.Json import space.kscience.dataforge.misc.DfType -import space.kscience.dataforge.misc.unsafeCast import space.kscience.dataforge.names.* import kotlin.jvm.JvmName @@ -151,6 +150,8 @@ public interface TypedMeta> : Meta { override val items: Map + public val self: M + override fun get(name: Name): M? { tailrec fun M.find(name: Name): M? = if (name.isEmpty()) { this @@ -164,11 +165,6 @@ public interface TypedMeta> : Meta { override fun toMeta(): Meta = this } -/** - * Access self as a recursive type instance - */ -public inline val > TypedMeta.self: M get() = unsafeCast() - //public typealias Meta = TypedMeta<*> public operator fun > TypedMeta?.get(token: NameToken): M? = this?.items?.get(token) diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MetaConverter.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MetaConverter.kt index 124f49f5..9baf0087 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MetaConverter.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MetaConverter.kt @@ -11,7 +11,7 @@ import space.kscience.dataforge.misc.DFExperimental /** * A converter of generic object to and from [Meta] */ -public interface MetaConverter: MetaSpec { +public interface MetaConverter: MetaReader { /** * A descriptor for resulting meta diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MetaDelegate.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MetaDelegate.kt index 8c5a738f..1b506b44 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MetaDelegate.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MetaDelegate.kt @@ -25,16 +25,16 @@ public fun MetaProvider.node( } /** - * Use [metaSpec] to read the Meta node + * Use [metaReader] to read the Meta node */ public fun MetaProvider.spec( - metaSpec: MetaSpec, + metaReader: MetaReader, key: Name? = null, ): MetaDelegate = object : MetaDelegate { - override val descriptor: MetaDescriptor? get() = metaSpec.descriptor + override val descriptor: MetaDescriptor? get() = metaReader.descriptor override fun getValue(thisRef: Any?, property: KProperty<*>): T? { - return get(key ?: property.name.asName())?.let { metaSpec.read(it) } + return get(key ?: property.name.asName())?.let { metaReader.read(it) } } } @@ -50,14 +50,14 @@ public inline fun MetaProvider.serializable( @Deprecated("Use convertable", ReplaceWith("convertable(converter, key)")) public fun MetaProvider.node( key: Name? = null, - converter: MetaSpec, + converter: MetaReader, ): ReadOnlyProperty = spec(converter, key) /** * Use [converter] to convert a list of same name siblings meta to object */ public fun Meta.listOfSpec( - converter: MetaSpec, + converter: MetaReader, key: Name? = null, ): MetaDelegate> = object : MetaDelegate> { override fun getValue(thisRef: Any?, property: KProperty<*>): List { diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MetaSpec.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MetaReader.kt similarity index 59% rename from dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MetaSpec.kt rename to dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MetaReader.kt index 9918d504..a8514d63 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MetaSpec.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MetaReader.kt @@ -2,7 +2,7 @@ package space.kscience.dataforge.meta import space.kscience.dataforge.meta.descriptors.Described -public interface MetaSpec : Described { +public interface MetaReader : Described { /** * Read the source meta into an object and return null if Meta could not be interpreted as a target type @@ -10,12 +10,12 @@ public interface MetaSpec : Described { public fun readOrNull(source: Meta): T? /** - * Read generic read-only meta with this [MetaSpec] producing instance of the desired type. + * Read generic read-only meta with this [MetaReader] producing instance of the desired type. * Throws an error if conversion could not be done. */ public fun read(source: Meta): T = readOrNull(source) ?: error("Meta $source could not be interpreted by $this") } -public fun MetaSpec.readNullable(item: Meta?): T? = item?.let { read(it) } -public fun MetaSpec.readValue(value: Value): T? = read(Meta(value)) +public fun MetaReader.readNullable(item: Meta?): T? = item?.let { read(it) } +public fun MetaReader.readValue(value: Value): T? = read(Meta(value)) diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MetaRef.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MetaRef.kt new file mode 100644 index 00000000..413fe404 --- /dev/null +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MetaRef.kt @@ -0,0 +1,64 @@ +package space.kscience.dataforge.meta + +import space.kscience.dataforge.meta.descriptors.Described +import space.kscience.dataforge.meta.descriptors.MetaDescriptor +import space.kscience.dataforge.meta.descriptors.MetaDescriptorBuilder +import space.kscience.dataforge.misc.DFExperimental +import space.kscience.dataforge.names.Name +import space.kscience.dataforge.names.asName +import kotlin.properties.PropertyDelegateProvider +import kotlin.properties.ReadOnlyProperty + + +/** + * A reference to a read-only value of type [T] inside [MetaProvider] + */ +@DFExperimental +public data class MetaRef( + public val name: Name, + public val converter: MetaConverter, + override val descriptor: MetaDescriptor? = converter.descriptor, +) : Described + +@DFExperimental +public operator fun MetaProvider.get(ref: MetaRef): T? = get(ref.name)?.let { ref.converter.readOrNull(it) } + +@DFExperimental +public operator fun MutableMetaProvider.set(ref: MetaRef, value: T) { + set(ref.name, ref.converter.convert(value)) +} + +@DFExperimental +public class MetaSpec( + private val configuration: MetaDescriptorBuilder.() -> Unit = {}, +) : Described { + private val refs: MutableList> = mutableListOf() + + private fun registerRef(ref: MetaRef<*>) { + refs.add(ref) + } + + public fun item( + converter: MetaConverter, + descriptor: MetaDescriptor? = converter.descriptor, + key: Name? = null, + ): PropertyDelegateProvider>> = + PropertyDelegateProvider { _, property -> + val ref = MetaRef(key ?: property.name.asName(), converter, descriptor) + registerRef(ref) + ReadOnlyProperty { _, _ -> + ref + } + } + + override val descriptor: MetaDescriptor by lazy { + MetaDescriptor { + refs.forEach { ref -> + ref.descriptor?.let { + node(ref.name, ref.descriptor) + } + } + configuration() + } + } +} \ No newline at end of file diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MutableMeta.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MutableMeta.kt index 69e221f5..7be16cc9 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MutableMeta.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MutableMeta.kt @@ -253,6 +253,9 @@ private class MutableMetaImpl( value: Value?, children: Map = emptyMap(), ) : AbstractObservableMeta(), ObservableMutableMeta { + + override val self get() = this + override var value = value @ThreadSafe set(value) { val oldValue = field diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/ObservableMeta.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/ObservableMeta.kt index b481962e..7cd28746 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/ObservableMeta.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/ObservableMeta.kt @@ -39,6 +39,9 @@ public interface ObservableMeta : Meta { * A [Meta] which is both observable and mutable */ public interface ObservableMutableMeta : ObservableMeta, MutableMeta, MutableTypedMeta { + + override val self: ObservableMutableMeta get() = this + override fun getOrCreate(name: Name): ObservableMutableMeta override fun get(name: Name): ObservableMutableMeta? { diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/ObservableMetaWrapper.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/ObservableMetaWrapper.kt index abd2deb6..7c62f692 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/ObservableMetaWrapper.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/ObservableMetaWrapper.kt @@ -14,6 +14,9 @@ private class ObservableMetaWrapper( val nodeName: Name, val listeners: MutableSet, ) : ObservableMutableMeta { + + override val self get() = this + override val items: Map get() = root[nodeName]?.items?.keys?.associateWith { ObservableMetaWrapper(root, nodeName + it, listeners) diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/Scheme.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/Scheme.kt index 85b5087a..fe121c42 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/Scheme.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/Scheme.kt @@ -12,7 +12,7 @@ import kotlin.reflect.KProperty import kotlin.reflect.KProperty1 /** - * A base for delegate-based or descriptor-based scheme. [Scheme] has an empty constructor to simplify usage from [MetaSpec]. + * A base for delegate-based or descriptor-based scheme. [Scheme] has an empty constructor to simplify usage from [MetaReader]. * * @param prototype default values provided by this scheme */ @@ -80,6 +80,9 @@ public open class Scheme( override fun toString(): String = meta.toString() private inner class SchemeMeta(val pathName: Name) : ObservableMutableMeta { + + override val self get() = this + override var value: Value? get() = target[pathName]?.value ?: prototype?.get(pathName)?.value @@ -216,7 +219,7 @@ public fun Configurable.updateWith( /** - * A delegate that uses a [MetaSpec] to wrap a child of this provider + * A delegate that uses a [MetaReader] to wrap a child of this provider */ public fun MutableMeta.scheme( spec: SchemeSpec, @@ -239,7 +242,7 @@ public fun Scheme.scheme( ): ReadWriteProperty = meta.scheme(spec, key) /** - * A delegate that uses a [MetaSpec] to wrap a child of this provider. + * A delegate that uses a [MetaReader] to wrap a child of this provider. * Returns null if meta with given name does not exist. */ public fun MutableMeta.schemeOrNull( @@ -264,7 +267,7 @@ public fun Scheme.schemeOrNull( ): ReadWriteProperty = meta.schemeOrNull(spec, key) /** - * A delegate that uses a [MetaSpec] to wrap a list of child providers. + * A delegate that uses a [MetaReader] to wrap a list of child providers. * If children are mutable, the changes in list elements are reflected on them. * The list is a snapshot of children state, so change in structure is not reflected on its composition. */ diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/SealedMeta.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/SealedMeta.kt index b3fdf062..b218fad6 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/SealedMeta.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/SealedMeta.kt @@ -13,6 +13,9 @@ public class SealedMeta( override val value: Value?, override val items: Map, ) : TypedMeta { + + override val self: SealedMeta get() = this + override fun toString(): String = Meta.toString(this) override fun equals(other: Any?): Boolean = Meta.equals(this, other as? Meta) diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/descriptors/MetaDescriptorBuilder.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/descriptors/MetaDescriptorBuilder.kt index dfdbfeab..2590273e 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/descriptors/MetaDescriptorBuilder.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/descriptors/MetaDescriptorBuilder.kt @@ -58,7 +58,7 @@ public class MetaDescriptorBuilder @PublishedApi internal constructor() { } } - internal fun node( + public fun node( name: Name, descriptorBuilder: MetaDescriptor, ): Unit { diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/misc/cast.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/misc/cast.kt deleted file mode 100644 index e714d596..00000000 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/misc/cast.kt +++ /dev/null @@ -1,3 +0,0 @@ -package space.kscience.dataforge.misc - -public expect inline fun Any?.unsafeCast(): T \ No newline at end of file diff --git a/dataforge-meta/src/jsMain/kotlin/space/kscience/dataforge/meta/DynamicMeta.kt b/dataforge-meta/src/jsMain/kotlin/space/kscience/dataforge/meta/DynamicMeta.kt index b38d5891..57d324c7 100644 --- a/dataforge-meta/src/jsMain/kotlin/space/kscience/dataforge/meta/DynamicMeta.kt +++ b/dataforge-meta/src/jsMain/kotlin/space/kscience/dataforge/meta/DynamicMeta.kt @@ -31,6 +31,9 @@ public fun Meta.toDynamic(): dynamic { } public class DynamicMeta(internal val obj: dynamic) : TypedMeta { + + override val self: DynamicMeta get() = this + private fun keys(): Array = js("Object").keys(obj) as Array private fun isArray(obj: dynamic): Boolean = diff --git a/dataforge-meta/src/jsMain/kotlin/space/kscience/dataforge/misc/castJs.kt b/dataforge-meta/src/jsMain/kotlin/space/kscience/dataforge/misc/castJs.kt deleted file mode 100644 index b057bcbe..00000000 --- a/dataforge-meta/src/jsMain/kotlin/space/kscience/dataforge/misc/castJs.kt +++ /dev/null @@ -1,5 +0,0 @@ -package space.kscience.dataforge.misc -import kotlin.js.unsafeCast as unsafeCastJs - -@Suppress("NOTHING_TO_INLINE") -public actual inline fun Any?.unsafeCast(): T = unsafeCastJs() \ No newline at end of file diff --git a/dataforge-meta/src/jvmMain/kotlin/space/kscience/dataforge/misc/castJvm.kt b/dataforge-meta/src/jvmMain/kotlin/space/kscience/dataforge/misc/castJvm.kt deleted file mode 100644 index 27d399fe..00000000 --- a/dataforge-meta/src/jvmMain/kotlin/space/kscience/dataforge/misc/castJvm.kt +++ /dev/null @@ -1,4 +0,0 @@ -package space.kscience.dataforge.misc - -@Suppress("UNCHECKED_CAST", "NOTHING_TO_INLINE") -public actual inline fun Any?.unsafeCast(): T = this as T \ No newline at end of file diff --git a/dataforge-meta/src/nativeMain/kotlin/space/kscience/dataforge/misc/castNative.kt b/dataforge-meta/src/nativeMain/kotlin/space/kscience/dataforge/misc/castNative.kt deleted file mode 100644 index 4d9aa758..00000000 --- a/dataforge-meta/src/nativeMain/kotlin/space/kscience/dataforge/misc/castNative.kt +++ /dev/null @@ -1,4 +0,0 @@ -package space.kscience.dataforge.misc - -@Suppress("UNCHECKED_CAST") -public actual inline fun Any?.unsafeCast(): T = this as T \ No newline at end of file diff --git a/dataforge-meta/src/wasmJsMain/kotlin/space/kscience/dataforge/misc/castWasm.kt b/dataforge-meta/src/wasmJsMain/kotlin/space/kscience/dataforge/misc/castWasm.kt deleted file mode 100644 index 27d399fe..00000000 --- a/dataforge-meta/src/wasmJsMain/kotlin/space/kscience/dataforge/misc/castWasm.kt +++ /dev/null @@ -1,4 +0,0 @@ -package space.kscience.dataforge.misc - -@Suppress("UNCHECKED_CAST", "NOTHING_TO_INLINE") -public actual inline fun Any?.unsafeCast(): T = this as T \ No newline at end of file diff --git a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/Task.kt b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/Task.kt index 372b119f..a1a754a4 100644 --- a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/Task.kt +++ b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/Task.kt @@ -5,8 +5,8 @@ import space.kscience.dataforge.data.DataSink import space.kscience.dataforge.data.GoalExecutionRestriction import space.kscience.dataforge.data.MutableDataTree import space.kscience.dataforge.meta.Meta +import space.kscience.dataforge.meta.MetaReader import space.kscience.dataforge.meta.MetaRepr -import space.kscience.dataforge.meta.MetaSpec import space.kscience.dataforge.meta.descriptors.Described import space.kscience.dataforge.meta.descriptors.MetaDescriptor import space.kscience.dataforge.misc.DfType @@ -44,10 +44,10 @@ public interface Task : Described { } /** - * A [Task] with [MetaSpec] for wrapping and unwrapping task configuration + * A [Task] with [MetaReader] for wrapping and unwrapping task configuration */ public interface TaskWithSpec : Task { - public val spec: MetaSpec + public val spec: MetaReader override val descriptor: MetaDescriptor? get() = spec.descriptor public suspend fun execute(workspace: Workspace, taskName: Name, configuration: C): TaskResult @@ -122,10 +122,10 @@ public inline fun Task( @Suppress("FunctionName") public fun Task( resultType: KType, - specification: MetaSpec, + specification: MetaReader, builder: suspend TaskResultBuilder.(C) -> Unit, ): TaskWithSpec = object : TaskWithSpec { - override val spec: MetaSpec = specification + override val spec: MetaReader = specification override suspend fun execute( workspace: Workspace, @@ -143,6 +143,6 @@ public fun Task( } public inline fun Task( - specification: MetaSpec, + specification: MetaReader, noinline builder: suspend TaskResultBuilder.(C) -> Unit, ): Task = Task(typeOf(), specification, builder) \ No newline at end of file diff --git a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/WorkspaceBuilder.kt b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/WorkspaceBuilder.kt index cf263a46..013c0171 100644 --- a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/WorkspaceBuilder.kt +++ b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/WorkspaceBuilder.kt @@ -71,10 +71,10 @@ public inline fun TaskContainer.task( } /** - * Create a task based on [MetaSpec] + * Create a task based on [MetaReader] */ public inline fun TaskContainer.task( - specification: MetaSpec, + specification: MetaReader, noinline builder: suspend TaskResultBuilder.(C) -> Unit, ): PropertyDelegateProvider>> = PropertyDelegateProvider { _, property -> val taskName = Name.parse(property.name) diff --git a/gradle.properties b/gradle.properties index 3734d13e..ffc318d9 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,10 +1,9 @@ +kotlin.code.style=official + org.gradle.parallel=true org.gradle.jvmargs=-Xmx4096m -kotlin.code.style=official kotlin.mpp.stability.nowarn=true -kotlin.incremental.js.ir=true kotlin.native.ignoreDisabledTargets=true -toolsVersion=0.15.2-kotlin-1.9.21 -#kotlin.experimental.tryK2=true \ No newline at end of file +toolsVersion=0.15.4-kotlin-2.0.0 \ No newline at end of file -- 2.34.1 From 04238295e277430e5b22771e470e87f343bd2fd1 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Tue, 4 Jun 2024 17:55:25 +0300 Subject: [PATCH 47/62] Remove sequence with list builder in Name::parse to fix problem in Wasm --- .../kotlin/space/kscience/dataforge/names/Name.kt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/names/Name.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/names/Name.kt index b46a3507..7867330c 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/names/Name.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/names/Name.kt @@ -58,7 +58,7 @@ public class Name(public val tokens: List) { */ public fun parse(string: String): Name { if (string.isBlank()) return EMPTY - val tokens = sequence { + val tokens = buildList { var bodyBuilder = StringBuilder() var queryBuilder = StringBuilder() var bracketCount = 0 @@ -91,7 +91,7 @@ public class Name(public val tokens: List) { else -> when (it) { '.' -> { val query = if (queryBuilder.isEmpty()) null else queryBuilder.toString() - yield(NameToken(bodyBuilder.toString(), query)) + add(NameToken(bodyBuilder.toString(), query)) bodyBuilder = StringBuilder() queryBuilder = StringBuilder() } @@ -106,9 +106,9 @@ public class Name(public val tokens: List) { } } val query = if (queryBuilder.isEmpty()) null else queryBuilder.toString() - yield(NameToken(bodyBuilder.toString(), query)) + add(NameToken(bodyBuilder.toString(), query)) } - return Name(tokens.toList()) + return Name(tokens) } } } -- 2.34.1 From c62dc2c698ad802e38542c2490f9996d4944ae9f Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Tue, 4 Jun 2024 18:00:10 +0300 Subject: [PATCH 48/62] 0.9 release --- CHANGELOG.md | 21 +++++-- build.gradle.kts | 2 +- dataforge-context/README.md | 4 +- dataforge-context/api/dataforge-context.api | 23 ++----- dataforge-data/README.md | 4 +- dataforge-io/README.md | 4 +- dataforge-io/dataforge-io-yaml/README.md | 4 +- dataforge-meta/README.md | 4 +- dataforge-meta/api/dataforge-meta.api | 67 +++++++++++---------- dataforge-scripting/README.md | 4 +- dataforge-workspace/README.md | 4 +- 11 files changed, 72 insertions(+), 69 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d75dd1a2..1f59fbae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,22 +3,33 @@ ## Unreleased ### Added -- Custom CoroutineContext during `Context` creation. ### Changed -- Kotlin 2.0 -- `MetaSpec` renamed to `MetaReader`. MetaSpec is now reserved for builder-based generation of meta descriptors. -- Add self-type for Meta. Remove unsafe cast method for meta instances. ### Deprecated ### Removed -- Automatic descriptors for schema. It is not possible to implement them without heavy reflection. ### Fixed ### Security +## 0.9.0 - 2024-06-04 + +### Added + +- Custom CoroutineContext during `Context` creation. + +### Changed + +- Kotlin 2.0 +- `MetaSpec` renamed to `MetaReader`. MetaSpec is now reserved for builder-based generation of meta descriptors. +- Add self-type for Meta. Remove unsafe cast method for meta instances. + +### Removed + +- Automatic descriptors for schema. It is not possible to implement them without heavy reflection. + ## 0.8.2 - 2024-04-27 ### Added diff --git a/build.gradle.kts b/build.gradle.kts index 815a7077..72f61abc 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -9,7 +9,7 @@ plugins { allprojects { group = "space.kscience" - version = "0.9.0-dev-1" + version = "0.9.0" } subprojects { diff --git a/dataforge-context/README.md b/dataforge-context/README.md index 7970751c..2cd53fd1 100644 --- a/dataforge-context/README.md +++ b/dataforge-context/README.md @@ -6,7 +6,7 @@ Context and provider definitions ## Artifact: -The Maven coordinates of this project are `space.kscience:dataforge-context:0.8.2`. +The Maven coordinates of this project are `space.kscience:dataforge-context:0.9.0-dev-1`. **Gradle Kotlin DSL:** ```kotlin @@ -16,6 +16,6 @@ repositories { } dependencies { - implementation("space.kscience:dataforge-context:0.8.2") + implementation("space.kscience:dataforge-context:0.9.0-dev-1") } ``` diff --git a/dataforge-context/api/dataforge-context.api b/dataforge-context/api/dataforge-context.api index eb94f89d..67cd0cba 100644 --- a/dataforge-context/api/dataforge-context.api +++ b/dataforge-context/api/dataforge-context.api @@ -57,6 +57,7 @@ public abstract interface class space/kscience/dataforge/context/ContextAware { public final class space/kscience/dataforge/context/ContextBuilder { public final fun build ()Lspace/kscience/dataforge/context/Context; + public final fun coroutineContext (Lkotlin/coroutines/CoroutineContext;)V public final fun getName ()Lspace/kscience/dataforge/names/Name; public final fun plugin (Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Lkotlin/jvm/functions/Function1;)V public final fun plugin (Lspace/kscience/dataforge/context/Plugin;)V @@ -69,9 +70,6 @@ public final class space/kscience/dataforge/context/ContextBuilder { public final fun properties (Lkotlin/jvm/functions/Function1;)V } -public final class space/kscience/dataforge/context/ContextBuilderKt { -} - public final class space/kscience/dataforge/context/DefaultLogManager : space/kscience/dataforge/context/AbstractPlugin, space/kscience/dataforge/context/LogManager { public static final field Companion Lspace/kscience/dataforge/context/DefaultLogManager$Companion; public fun ()V @@ -212,14 +210,14 @@ public final class space/kscience/dataforge/context/PluginTag : space/kscience/d public fun toString ()Ljava/lang/String; } -public final class space/kscience/dataforge/context/PluginTag$$serializer : kotlinx/serialization/internal/GeneratedSerializer { +public synthetic class space/kscience/dataforge/context/PluginTag$$serializer : kotlinx/serialization/internal/GeneratedSerializer { public static final field INSTANCE Lspace/kscience/dataforge/context/PluginTag$$serializer; - public fun childSerializers ()[Lkotlinx/serialization/KSerializer; + public final fun childSerializers ()[Lkotlinx/serialization/KSerializer; public synthetic fun deserialize (Lkotlinx/serialization/encoding/Decoder;)Ljava/lang/Object; - public fun deserialize (Lkotlinx/serialization/encoding/Decoder;)Lspace/kscience/dataforge/context/PluginTag; - public fun getDescriptor ()Lkotlinx/serialization/descriptors/SerialDescriptor; + public final fun deserialize (Lkotlinx/serialization/encoding/Decoder;)Lspace/kscience/dataforge/context/PluginTag; + public final fun getDescriptor ()Lkotlinx/serialization/descriptors/SerialDescriptor; public synthetic fun serialize (Lkotlinx/serialization/encoding/Encoder;Ljava/lang/Object;)V - public fun serialize (Lkotlinx/serialization/encoding/Encoder;Lspace/kscience/dataforge/context/PluginTag;)V + public final fun serialize (Lkotlinx/serialization/encoding/Encoder;Lspace/kscience/dataforge/context/PluginTag;)V public fun typeParametersSerializers ()[Lkotlinx/serialization/KSerializer; } @@ -264,15 +262,6 @@ public abstract interface annotation class space/kscience/dataforge/descriptors/ public abstract interface annotation class space/kscience/dataforge/descriptors/Multiple : java/lang/annotation/Annotation { } -public final class space/kscience/dataforge/descriptors/ReflectiveDescriptorsKt { -} - -public final class space/kscience/dataforge/properties/MetaAsFlowKt { -} - -public final class space/kscience/dataforge/provider/DfTypeKt { -} - public final class space/kscience/dataforge/provider/Path : java/lang/Iterable, kotlin/jvm/internal/markers/KMappedMarker { public static final field Companion Lspace/kscience/dataforge/provider/Path$Companion; public static final field PATH_SEGMENT_SEPARATOR Ljava/lang/String; diff --git a/dataforge-data/README.md b/dataforge-data/README.md index f08a43c2..35aaa4e6 100644 --- a/dataforge-data/README.md +++ b/dataforge-data/README.md @@ -6,7 +6,7 @@ ## Artifact: -The Maven coordinates of this project are `space.kscience:dataforge-data:0.8.2`. +The Maven coordinates of this project are `space.kscience:dataforge-data:0.9.0-dev-1`. **Gradle Kotlin DSL:** ```kotlin @@ -16,6 +16,6 @@ repositories { } dependencies { - implementation("space.kscience:dataforge-data:0.8.2") + implementation("space.kscience:dataforge-data:0.9.0-dev-1") } ``` diff --git a/dataforge-io/README.md b/dataforge-io/README.md index 5168797b..5a9979a9 100644 --- a/dataforge-io/README.md +++ b/dataforge-io/README.md @@ -6,7 +6,7 @@ IO module ## Artifact: -The Maven coordinates of this project are `space.kscience:dataforge-io:0.8.2`. +The Maven coordinates of this project are `space.kscience:dataforge-io:0.9.0-dev-1`. **Gradle Kotlin DSL:** ```kotlin @@ -16,6 +16,6 @@ repositories { } dependencies { - implementation("space.kscience:dataforge-io:0.8.2") + implementation("space.kscience:dataforge-io:0.9.0-dev-1") } ``` diff --git a/dataforge-io/dataforge-io-yaml/README.md b/dataforge-io/dataforge-io-yaml/README.md index cc1ecd8c..f70a1490 100644 --- a/dataforge-io/dataforge-io-yaml/README.md +++ b/dataforge-io/dataforge-io-yaml/README.md @@ -6,7 +6,7 @@ YAML meta IO ## Artifact: -The Maven coordinates of this project are `space.kscience:dataforge-io-yaml:0.8.2`. +The Maven coordinates of this project are `space.kscience:dataforge-io-yaml:0.9.0-dev-1`. **Gradle Kotlin DSL:** ```kotlin @@ -16,6 +16,6 @@ repositories { } dependencies { - implementation("space.kscience:dataforge-io-yaml:0.8.2") + implementation("space.kscience:dataforge-io-yaml:0.9.0-dev-1") } ``` diff --git a/dataforge-meta/README.md b/dataforge-meta/README.md index a3209760..e4fcacb1 100644 --- a/dataforge-meta/README.md +++ b/dataforge-meta/README.md @@ -6,7 +6,7 @@ Meta definition and basic operations on meta ## Artifact: -The Maven coordinates of this project are `space.kscience:dataforge-meta:0.8.2`. +The Maven coordinates of this project are `space.kscience:dataforge-meta:0.9.0-dev-1`. **Gradle Kotlin DSL:** ```kotlin @@ -16,6 +16,6 @@ repositories { } dependencies { - implementation("space.kscience:dataforge-meta:0.8.2") + implementation("space.kscience:dataforge-meta:0.9.0-dev-1") } ``` diff --git a/dataforge-meta/api/dataforge-meta.api b/dataforge-meta/api/dataforge-meta.api index 620e64d4..0bb4a908 100644 --- a/dataforge-meta/api/dataforge-meta.api +++ b/dataforge-meta/api/dataforge-meta.api @@ -103,6 +103,8 @@ public final class space/kscience/dataforge/meta/Laminate : space/kscience/dataf public synthetic fun get (Lspace/kscience/dataforge/names/Name;)Lspace/kscience/dataforge/meta/TypedMeta; public fun getItems ()Ljava/util/Map; public final fun getLayers ()Ljava/util/List; + public fun getSelf ()Lspace/kscience/dataforge/meta/Laminate; + public synthetic fun getSelf ()Lspace/kscience/dataforge/meta/TypedMeta; public fun getValue ()Lspace/kscience/dataforge/meta/Value; public fun hashCode ()I public final fun merge ()Lspace/kscience/dataforge/meta/SealedMeta; @@ -198,7 +200,7 @@ public final class space/kscience/dataforge/meta/MetaBuilder : space/kscience/da public abstract interface annotation class space/kscience/dataforge/meta/MetaBuilderMarker : java/lang/annotation/Annotation { } -public abstract interface class space/kscience/dataforge/meta/MetaConverter : space/kscience/dataforge/meta/MetaSpec { +public abstract interface class space/kscience/dataforge/meta/MetaConverter : space/kscience/dataforge/meta/MetaReader { public static final field Companion Lspace/kscience/dataforge/meta/MetaConverter$Companion; public abstract fun convert (Ljava/lang/Object;)Lspace/kscience/dataforge/meta/Meta; public fun getDescriptor ()Lspace/kscience/dataforge/meta/descriptors/MetaDescriptor; @@ -246,15 +248,15 @@ public final class space/kscience/dataforge/meta/MetaDelegateKt { public static final fun int (Lspace/kscience/dataforge/meta/MetaProvider;Lspace/kscience/dataforge/names/Name;)Lspace/kscience/dataforge/meta/MetaDelegate; public static synthetic fun int$default (Lspace/kscience/dataforge/meta/MetaProvider;ILspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lspace/kscience/dataforge/meta/MetaDelegate; public static synthetic fun int$default (Lspace/kscience/dataforge/meta/MetaProvider;Lspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lspace/kscience/dataforge/meta/MetaDelegate; - public static final fun listOfSpec (Lspace/kscience/dataforge/meta/Meta;Lspace/kscience/dataforge/meta/MetaSpec;Lspace/kscience/dataforge/names/Name;)Lspace/kscience/dataforge/meta/MetaDelegate; - public static synthetic fun listOfSpec$default (Lspace/kscience/dataforge/meta/Meta;Lspace/kscience/dataforge/meta/MetaSpec;Lspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lspace/kscience/dataforge/meta/MetaDelegate; + public static final fun listOfSpec (Lspace/kscience/dataforge/meta/Meta;Lspace/kscience/dataforge/meta/MetaReader;Lspace/kscience/dataforge/names/Name;)Lspace/kscience/dataforge/meta/MetaDelegate; + public static synthetic fun listOfSpec$default (Lspace/kscience/dataforge/meta/Meta;Lspace/kscience/dataforge/meta/MetaReader;Lspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lspace/kscience/dataforge/meta/MetaDelegate; public static final fun long (Lspace/kscience/dataforge/meta/MetaProvider;JLspace/kscience/dataforge/names/Name;)Lspace/kscience/dataforge/meta/MetaDelegate; public static final fun long (Lspace/kscience/dataforge/meta/MetaProvider;Lspace/kscience/dataforge/names/Name;)Lspace/kscience/dataforge/meta/MetaDelegate; public static synthetic fun long$default (Lspace/kscience/dataforge/meta/MetaProvider;JLspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lspace/kscience/dataforge/meta/MetaDelegate; public static synthetic fun long$default (Lspace/kscience/dataforge/meta/MetaProvider;Lspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lspace/kscience/dataforge/meta/MetaDelegate; - public static final fun node (Lspace/kscience/dataforge/meta/MetaProvider;Lspace/kscience/dataforge/names/Name;Lspace/kscience/dataforge/meta/MetaSpec;)Lkotlin/properties/ReadOnlyProperty; + public static final fun node (Lspace/kscience/dataforge/meta/MetaProvider;Lspace/kscience/dataforge/names/Name;Lspace/kscience/dataforge/meta/MetaReader;)Lkotlin/properties/ReadOnlyProperty; public static final fun node (Lspace/kscience/dataforge/meta/MetaProvider;Lspace/kscience/dataforge/names/Name;Lspace/kscience/dataforge/meta/descriptors/MetaDescriptor;)Lspace/kscience/dataforge/meta/MetaDelegate; - public static synthetic fun node$default (Lspace/kscience/dataforge/meta/MetaProvider;Lspace/kscience/dataforge/names/Name;Lspace/kscience/dataforge/meta/MetaSpec;ILjava/lang/Object;)Lkotlin/properties/ReadOnlyProperty; + public static synthetic fun node$default (Lspace/kscience/dataforge/meta/MetaProvider;Lspace/kscience/dataforge/names/Name;Lspace/kscience/dataforge/meta/MetaReader;ILjava/lang/Object;)Lkotlin/properties/ReadOnlyProperty; public static synthetic fun node$default (Lspace/kscience/dataforge/meta/MetaProvider;Lspace/kscience/dataforge/names/Name;Lspace/kscience/dataforge/meta/descriptors/MetaDescriptor;ILjava/lang/Object;)Lspace/kscience/dataforge/meta/MetaDelegate; public static final fun number (Lspace/kscience/dataforge/meta/MetaProvider;Ljava/lang/Number;Lspace/kscience/dataforge/names/Name;)Lspace/kscience/dataforge/meta/MetaDelegate; public static final fun number (Lspace/kscience/dataforge/meta/MetaProvider;Lspace/kscience/dataforge/names/Name;)Lspace/kscience/dataforge/meta/MetaDelegate; @@ -262,8 +264,8 @@ public final class space/kscience/dataforge/meta/MetaDelegateKt { public static synthetic fun number$default (Lspace/kscience/dataforge/meta/MetaProvider;Ljava/lang/Number;Lspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lspace/kscience/dataforge/meta/MetaDelegate; public static synthetic fun number$default (Lspace/kscience/dataforge/meta/MetaProvider;Lspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lspace/kscience/dataforge/meta/MetaDelegate; public static synthetic fun number$default (Lspace/kscience/dataforge/meta/MetaProvider;Lspace/kscience/dataforge/names/Name;Lkotlin/jvm/functions/Function0;ILjava/lang/Object;)Lspace/kscience/dataforge/meta/MetaDelegate; - public static final fun spec (Lspace/kscience/dataforge/meta/MetaProvider;Lspace/kscience/dataforge/meta/MetaSpec;Lspace/kscience/dataforge/names/Name;)Lspace/kscience/dataforge/meta/MetaDelegate; - public static synthetic fun spec$default (Lspace/kscience/dataforge/meta/MetaProvider;Lspace/kscience/dataforge/meta/MetaSpec;Lspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lspace/kscience/dataforge/meta/MetaDelegate; + public static final fun spec (Lspace/kscience/dataforge/meta/MetaProvider;Lspace/kscience/dataforge/meta/MetaReader;Lspace/kscience/dataforge/names/Name;)Lspace/kscience/dataforge/meta/MetaDelegate; + public static synthetic fun spec$default (Lspace/kscience/dataforge/meta/MetaProvider;Lspace/kscience/dataforge/meta/MetaReader;Lspace/kscience/dataforge/names/Name;ILjava/lang/Object;)Lspace/kscience/dataforge/meta/MetaDelegate; public static final fun string (Lspace/kscience/dataforge/meta/MetaProvider;Ljava/lang/String;Lspace/kscience/dataforge/names/Name;)Lspace/kscience/dataforge/meta/MetaDelegate; public static final fun string (Lspace/kscience/dataforge/meta/MetaProvider;Lspace/kscience/dataforge/names/Name;)Lspace/kscience/dataforge/meta/MetaDelegate; public static final fun string (Lspace/kscience/dataforge/meta/MetaProvider;Lspace/kscience/dataforge/names/Name;Lkotlin/jvm/functions/Function0;)Lspace/kscience/dataforge/meta/MetaDelegate; @@ -294,7 +296,6 @@ public final class space/kscience/dataforge/meta/MetaKt { public static final fun getLong (Lspace/kscience/dataforge/meta/Meta;)Ljava/lang/Long; public static final synthetic fun getNonNullable (Lspace/kscience/dataforge/meta/Meta;Lspace/kscience/dataforge/names/NameToken;)Lspace/kscience/dataforge/meta/Meta; public static final fun getNumber (Lspace/kscience/dataforge/meta/Meta;)Ljava/lang/Number; - public static final fun getSelf (Lspace/kscience/dataforge/meta/TypedMeta;)Lspace/kscience/dataforge/meta/TypedMeta; public static final fun getShort (Lspace/kscience/dataforge/meta/Meta;)Ljava/lang/Short; public static final fun getString (Lspace/kscience/dataforge/meta/Meta;)Ljava/lang/String; public static final fun getStringList (Lspace/kscience/dataforge/meta/Meta;)Ljava/util/List; @@ -311,6 +312,16 @@ public abstract interface class space/kscience/dataforge/meta/MetaProvider : spa public fun getValue (Lspace/kscience/dataforge/names/Name;)Lspace/kscience/dataforge/meta/Value; } +public abstract interface class space/kscience/dataforge/meta/MetaReader : space/kscience/dataforge/meta/descriptors/Described { + public fun read (Lspace/kscience/dataforge/meta/Meta;)Ljava/lang/Object; + public abstract fun readOrNull (Lspace/kscience/dataforge/meta/Meta;)Ljava/lang/Object; +} + +public final class space/kscience/dataforge/meta/MetaReaderKt { + public static final fun readNullable (Lspace/kscience/dataforge/meta/MetaReader;Lspace/kscience/dataforge/meta/Meta;)Ljava/lang/Object; + public static final fun readValue (Lspace/kscience/dataforge/meta/MetaReader;Lspace/kscience/dataforge/meta/Value;)Ljava/lang/Object; +} + public abstract interface class space/kscience/dataforge/meta/MetaRepr { public abstract fun toMeta ()Lspace/kscience/dataforge/meta/Meta; } @@ -324,16 +335,6 @@ public final class space/kscience/dataforge/meta/MetaSerializer : kotlinx/serial public fun serialize (Lkotlinx/serialization/encoding/Encoder;Lspace/kscience/dataforge/meta/Meta;)V } -public abstract interface class space/kscience/dataforge/meta/MetaSpec : space/kscience/dataforge/meta/descriptors/Described { - public fun read (Lspace/kscience/dataforge/meta/Meta;)Ljava/lang/Object; - public abstract fun readOrNull (Lspace/kscience/dataforge/meta/Meta;)Ljava/lang/Object; -} - -public final class space/kscience/dataforge/meta/MetaSpecKt { - public static final fun readNullable (Lspace/kscience/dataforge/meta/MetaSpec;Lspace/kscience/dataforge/meta/Meta;)Ljava/lang/Object; - public static final fun readValue (Lspace/kscience/dataforge/meta/MetaSpec;Lspace/kscience/dataforge/meta/Value;)Ljava/lang/Object; -} - public final class space/kscience/dataforge/meta/MetaTransformation { public static final field Companion Lspace/kscience/dataforge/meta/MetaTransformation$Companion; public static final fun apply-impl (Ljava/util/Collection;Lspace/kscience/dataforge/meta/Meta;)Lspace/kscience/dataforge/meta/Meta; @@ -551,6 +552,8 @@ public abstract interface class space/kscience/dataforge/meta/ObservableMutableM public fun get (Lspace/kscience/dataforge/names/Name;)Lspace/kscience/dataforge/meta/ObservableMutableMeta; public synthetic fun get (Lspace/kscience/dataforge/names/Name;)Lspace/kscience/dataforge/meta/TypedMeta; public abstract fun getOrCreate (Lspace/kscience/dataforge/names/Name;)Lspace/kscience/dataforge/meta/ObservableMutableMeta; + public fun getSelf ()Lspace/kscience/dataforge/meta/ObservableMutableMeta; + public synthetic fun getSelf ()Lspace/kscience/dataforge/meta/TypedMeta; } public final class space/kscience/dataforge/meta/RegexItemTransformationRule : space/kscience/dataforge/meta/TransformationRule { @@ -624,19 +627,21 @@ public final class space/kscience/dataforge/meta/SealedMeta : space/kscience/dat public fun (Lspace/kscience/dataforge/meta/Value;Ljava/util/Map;)V public fun equals (Ljava/lang/Object;)Z public fun getItems ()Ljava/util/Map; + public fun getSelf ()Lspace/kscience/dataforge/meta/SealedMeta; + public synthetic fun getSelf ()Lspace/kscience/dataforge/meta/TypedMeta; public fun getValue ()Lspace/kscience/dataforge/meta/Value; public fun hashCode ()I public fun toString ()Ljava/lang/String; } -public final class space/kscience/dataforge/meta/SealedMeta$$serializer : kotlinx/serialization/internal/GeneratedSerializer { +public synthetic class space/kscience/dataforge/meta/SealedMeta$$serializer : kotlinx/serialization/internal/GeneratedSerializer { public static final field INSTANCE Lspace/kscience/dataforge/meta/SealedMeta$$serializer; - public fun childSerializers ()[Lkotlinx/serialization/KSerializer; + public final fun childSerializers ()[Lkotlinx/serialization/KSerializer; public synthetic fun deserialize (Lkotlinx/serialization/encoding/Decoder;)Ljava/lang/Object; - public fun deserialize (Lkotlinx/serialization/encoding/Decoder;)Lspace/kscience/dataforge/meta/SealedMeta; - public fun getDescriptor ()Lkotlinx/serialization/descriptors/SerialDescriptor; + public final fun deserialize (Lkotlinx/serialization/encoding/Decoder;)Lspace/kscience/dataforge/meta/SealedMeta; + public final fun getDescriptor ()Lkotlinx/serialization/descriptors/SerialDescriptor; public synthetic fun serialize (Lkotlinx/serialization/encoding/Encoder;Ljava/lang/Object;)V - public fun serialize (Lkotlinx/serialization/encoding/Encoder;Lspace/kscience/dataforge/meta/SealedMeta;)V + public final fun serialize (Lkotlinx/serialization/encoding/Encoder;Lspace/kscience/dataforge/meta/SealedMeta;)V public fun typeParametersSerializers ()[Lkotlinx/serialization/KSerializer; } @@ -710,6 +715,7 @@ public abstract interface class space/kscience/dataforge/meta/TypedMeta : space/ public synthetic fun get (Lspace/kscience/dataforge/names/Name;)Lspace/kscience/dataforge/meta/Meta; public fun get (Lspace/kscience/dataforge/names/Name;)Lspace/kscience/dataforge/meta/TypedMeta; public abstract fun getItems ()Ljava/util/Map; + public abstract fun getSelf ()Lspace/kscience/dataforge/meta/TypedMeta; public fun toMeta ()Lspace/kscience/dataforge/meta/Meta; } @@ -844,14 +850,14 @@ public final class space/kscience/dataforge/meta/descriptors/MetaDescriptor { public fun toString ()Ljava/lang/String; } -public final class space/kscience/dataforge/meta/descriptors/MetaDescriptor$$serializer : kotlinx/serialization/internal/GeneratedSerializer { +public synthetic class space/kscience/dataforge/meta/descriptors/MetaDescriptor$$serializer : kotlinx/serialization/internal/GeneratedSerializer { public static final field INSTANCE Lspace/kscience/dataforge/meta/descriptors/MetaDescriptor$$serializer; - public fun childSerializers ()[Lkotlinx/serialization/KSerializer; + public final fun childSerializers ()[Lkotlinx/serialization/KSerializer; public synthetic fun deserialize (Lkotlinx/serialization/encoding/Decoder;)Ljava/lang/Object; - public fun deserialize (Lkotlinx/serialization/encoding/Decoder;)Lspace/kscience/dataforge/meta/descriptors/MetaDescriptor; - public fun getDescriptor ()Lkotlinx/serialization/descriptors/SerialDescriptor; + public final fun deserialize (Lkotlinx/serialization/encoding/Decoder;)Lspace/kscience/dataforge/meta/descriptors/MetaDescriptor; + public final fun getDescriptor ()Lkotlinx/serialization/descriptors/SerialDescriptor; public synthetic fun serialize (Lkotlinx/serialization/encoding/Encoder;Ljava/lang/Object;)V - public fun serialize (Lkotlinx/serialization/encoding/Encoder;Lspace/kscience/dataforge/meta/descriptors/MetaDescriptor;)V + public final fun serialize (Lkotlinx/serialization/encoding/Encoder;Lspace/kscience/dataforge/meta/descriptors/MetaDescriptor;)V public fun typeParametersSerializers ()[Lkotlinx/serialization/KSerializer; } @@ -876,6 +882,7 @@ public final class space/kscience/dataforge/meta/descriptors/MetaDescriptorBuild public final fun getMultiple ()Z public final fun getValueRestriction ()Lspace/kscience/dataforge/meta/descriptors/ValueRestriction; public final fun getValueTypes ()Ljava/util/List; + public final fun node (Lspace/kscience/dataforge/names/Name;Lspace/kscience/dataforge/meta/descriptors/MetaDescriptor;)V public final fun setAllowedValues (Ljava/util/List;)V public final fun setAttributes (Lspace/kscience/dataforge/meta/MutableMeta;)V public final fun setChildren (Ljava/util/Map;)V @@ -927,10 +934,6 @@ public final class space/kscience/dataforge/meta/descriptors/ValueRestriction$Co public final fun serializer ()Lkotlinx/serialization/KSerializer; } -public final class space/kscience/dataforge/misc/CastJvmKt { - public static final fun unsafeCast (Ljava/lang/Object;)Ljava/lang/Object; -} - public abstract interface annotation class space/kscience/dataforge/misc/DFBuilder : java/lang/annotation/Annotation { } diff --git a/dataforge-scripting/README.md b/dataforge-scripting/README.md index 3167e1b4..1f650bea 100644 --- a/dataforge-scripting/README.md +++ b/dataforge-scripting/README.md @@ -6,7 +6,7 @@ ## Artifact: -The Maven coordinates of this project are `space.kscience:dataforge-scripting:0.8.2`. +The Maven coordinates of this project are `space.kscience:dataforge-scripting:0.9.0-dev-1`. **Gradle Kotlin DSL:** ```kotlin @@ -16,6 +16,6 @@ repositories { } dependencies { - implementation("space.kscience:dataforge-scripting:0.8.2") + implementation("space.kscience:dataforge-scripting:0.9.0-dev-1") } ``` diff --git a/dataforge-workspace/README.md b/dataforge-workspace/README.md index f7461f30..87b38c6e 100644 --- a/dataforge-workspace/README.md +++ b/dataforge-workspace/README.md @@ -6,7 +6,7 @@ ## Artifact: -The Maven coordinates of this project are `space.kscience:dataforge-workspace:0.8.2`. +The Maven coordinates of this project are `space.kscience:dataforge-workspace:0.9.0-dev-1`. **Gradle Kotlin DSL:** ```kotlin @@ -16,6 +16,6 @@ repositories { } dependencies { - implementation("space.kscience:dataforge-workspace:0.8.2") + implementation("space.kscience:dataforge-workspace:0.9.0-dev-1") } ``` -- 2.34.1 From bb9afb329fbb9dcea3859a07f3a4718675d7c454 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Wed, 5 Jun 2024 10:53:35 +0300 Subject: [PATCH 49/62] MetaProvider `spec` is replaced by `readable`. `listOfSpec` replaced with `listOfReadable` --- CHANGELOG.md | 1 + build.gradle.kts | 2 +- .../kscience/dataforge/meta/MetaDelegate.kt | 68 +++++++++++++++---- .../dataforge/meta/MutableMetaDelegate.kt | 31 +++++++-- 4 files changed, 84 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1f59fbae..788a7374 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ ### Changed ### Deprecated +- MetaProvider `spec` is replaced by `readable`. `listOfSpec` replaced with `listOfReadable` ### Removed diff --git a/build.gradle.kts b/build.gradle.kts index 72f61abc..9f9632de 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -9,7 +9,7 @@ plugins { allprojects { group = "space.kscience" - version = "0.9.0" + version = "0.9.1-dev-1" } subprojects { diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MetaDelegate.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MetaDelegate.kt index 1b506b44..2fa0f528 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MetaDelegate.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MetaDelegate.kt @@ -24,20 +24,45 @@ public fun MetaProvider.node( } } + /** - * Use [metaReader] to read the Meta node + * Use [reader] to read the Meta node */ -public fun MetaProvider.spec( - metaReader: MetaReader, +public fun MetaProvider.readable( + reader: MetaReader, key: Name? = null, ): MetaDelegate = object : MetaDelegate { - override val descriptor: MetaDescriptor? get() = metaReader.descriptor + override val descriptor: MetaDescriptor? get() = reader.descriptor override fun getValue(thisRef: Any?, property: KProperty<*>): T? { - return get(key ?: property.name.asName())?.let { metaReader.read(it) } + return get(key ?: property.name.asName())?.let { reader.read(it) } } } +/** + * Use [reader] to read the Meta node or return [default] if node does not exist + */ +public fun MetaProvider.readable( + reader: MetaReader, + default: T, + key: Name? = null, +): MetaDelegate = object : MetaDelegate { + override val descriptor: MetaDescriptor? get() = reader.descriptor + + override fun getValue(thisRef: Any?, property: KProperty<*>): T { + return get(key ?: property.name.asName())?.let { reader.read(it) } ?: default + } +} + +/** + * Use [reader] to read the Meta node + */ +@Deprecated("Replace with reading", ReplaceWith("reading(metaReader, key)")) +public fun MetaProvider.spec( + reader: MetaReader, + key: Name? = null, +): MetaDelegate = readable(reader, key) + /** * Use object serializer to transform it to Meta and back */ @@ -45,34 +70,51 @@ public fun MetaProvider.spec( public inline fun MetaProvider.serializable( key: Name? = null, descriptor: MetaDescriptor? = null, -): MetaDelegate = spec(MetaConverter.serializable(descriptor), key) +): MetaDelegate = readable(MetaConverter.serializable(descriptor), key) + +@DFExperimental +public inline fun MetaProvider.serializable( + key: Name? = null, + default: T, + descriptor: MetaDescriptor? = null, +): MetaDelegate = readable(MetaConverter.serializable(descriptor), default, key) @Deprecated("Use convertable", ReplaceWith("convertable(converter, key)")) public fun MetaProvider.node( key: Name? = null, converter: MetaReader, -): ReadOnlyProperty = spec(converter, key) +): ReadOnlyProperty = readable(converter, key) /** - * Use [converter] to convert a list of same name siblings meta to object + * Use [reader] to convert a list of same name siblings meta to object */ -public fun Meta.listOfSpec( - converter: MetaReader, +public fun Meta.listOfReadable( + reader: MetaReader, key: Name? = null, ): MetaDelegate> = object : MetaDelegate> { override fun getValue(thisRef: Any?, property: KProperty<*>): List { val name = key ?: property.name.asName() - return getIndexed(name).values.map { converter.read(it) } + return getIndexed(name).values.map { reader.read(it) } } - override val descriptor: MetaDescriptor? = converter.descriptor?.copy(multiple = true) + override val descriptor: MetaDescriptor? = reader.descriptor?.copy(multiple = true) } + +/** + * Use [converter] to convert a list of same name siblings meta to object + */ +@Deprecated("Replace with readingList", ReplaceWith("readingList(converter, key)")) +public fun Meta.listOfSpec( + converter: MetaReader, + key: Name? = null, +): MetaDelegate> = listOfReadable(converter, key) + @DFExperimental public inline fun Meta.listOfSerializable( key: Name? = null, descriptor: MetaDescriptor? = null, -): MetaDelegate> = listOfSpec(MetaConverter.serializable(descriptor), key) +): MetaDelegate> = listOfReadable(MetaConverter.serializable(descriptor), key) /** * A property delegate that uses custom key diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MutableMetaDelegate.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MutableMetaDelegate.kt index 37140c6f..28a550d8 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MutableMetaDelegate.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MutableMetaDelegate.kt @@ -54,9 +54,25 @@ public fun MutableMetaProvider.convertable( } } -@Deprecated("Use convertable", ReplaceWith("convertable(converter, key)")) -public fun MutableMetaProvider.node(key: Name? = null, converter: MetaConverter): MutableMetaDelegate = - convertable(converter, key) +public fun MutableMetaProvider.convertable( + converter: MetaConverter, + default: T, + key: Name? = null, +): MutableMetaDelegate = object : MutableMetaDelegate { + + override val descriptor: MetaDescriptor? get() = converter.descriptor + + + override fun getValue(thisRef: Any?, property: KProperty<*>): T { + val name = key ?: property.name.asName() + return get(name)?.let { converter.read(it) } ?: default + } + + override fun setValue(thisRef: Any?, property: KProperty<*>, value: T) { + val name = key ?: property.name.asName() + set(name, value?.let { converter.convert(it) }) + } +} /** * Use object serializer to transform it to Meta and back. @@ -66,7 +82,14 @@ public fun MutableMetaProvider.node(key: Name? = null, converter: MetaConver public inline fun MutableMetaProvider.serializable( descriptor: MetaDescriptor? = null, key: Name? = null, -): MutableMetaDelegate = convertable(MetaConverter.serializable(descriptor), key) +): MutableMetaDelegate = convertable(MetaConverter.serializable(descriptor), key) + +@DFExperimental +public inline fun MutableMetaProvider.serializable( + descriptor: MetaDescriptor? = null, + default: T, + key: Name? = null, +): MutableMetaDelegate = convertable(MetaConverter.serializable(descriptor), default, key) /** * Use [converter] to convert a list of same name siblings meta to object and back. -- 2.34.1 From f79b7faeafeafab51a5c6432be0255d5c3e14251 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Wed, 5 Jun 2024 12:15:27 +0300 Subject: [PATCH 50/62] Simplify inheritance logic in `MutableTypedMeta` --- CHANGELOG.md | 1 + .../space/kscience/dataforge/meta/MutableMeta.kt | 14 ++++++++++++-- .../kscience/dataforge/meta/ObservableMeta.kt | 16 ---------------- 3 files changed, 13 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 788a7374..43e6bf4e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ ### Added ### Changed +- Simplify inheritance logic in `MutableTypedMeta` ### Deprecated - MetaProvider `spec` is replaced by `readable`. `listOfSpec` replaced with `listOfReadable` diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MutableMeta.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MutableMeta.kt index 7be16cc9..274671bf 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MutableMeta.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MutableMeta.kt @@ -159,7 +159,17 @@ public interface MutableTypedMeta> : TypedMeta, Mutab */ @DFExperimental public fun attach(name: Name, node: M) - override fun get(name: Name): M? + + override fun get(name: Name): M? { + tailrec fun M.find(name: Name): M? = if (name.isEmpty()) { + self + } else { + items[name.firstOrNull()!!]?.find(name.cutFirst()) + } + + return self.find(name) + } + override fun getOrCreate(name: Name): M } @@ -388,7 +398,7 @@ public fun MutableMeta.reset(newMeta: Meta) { (items.keys - newMeta.items.keys).forEach { remove(it.asName()) } - newMeta.items.forEach { (token, item)-> + newMeta.items.forEach { (token, item) -> set(token, item) } } diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/ObservableMeta.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/ObservableMeta.kt index 7cd28746..20388a92 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/ObservableMeta.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/ObservableMeta.kt @@ -2,9 +2,6 @@ package space.kscience.dataforge.meta import space.kscience.dataforge.misc.ThreadSafe import space.kscience.dataforge.names.Name -import space.kscience.dataforge.names.cutFirst -import space.kscience.dataforge.names.firstOrNull -import space.kscience.dataforge.names.isEmpty internal data class MetaListener( @@ -39,20 +36,7 @@ public interface ObservableMeta : Meta { * A [Meta] which is both observable and mutable */ public interface ObservableMutableMeta : ObservableMeta, MutableMeta, MutableTypedMeta { - override val self: ObservableMutableMeta get() = this - - override fun getOrCreate(name: Name): ObservableMutableMeta - - override fun get(name: Name): ObservableMutableMeta? { - tailrec fun ObservableMutableMeta.find(name: Name): ObservableMutableMeta? = if (name.isEmpty()) { - this - } else { - items[name.firstOrNull()!!]?.find(name.cutFirst()) - } - - return find(name) - } } internal abstract class AbstractObservableMeta : ObservableMeta { -- 2.34.1 From 332d38df77895ea3f72e6ba2a5b2d41233c08d29 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Mon, 10 Jun 2024 20:28:57 +0300 Subject: [PATCH 51/62] First implementation for protobuf converter --- .../kscience/dataforge/data/ActionsTest.kt | 16 ++-- .../dataforge-io-proto/build.gradle.kts | 32 +++++++ .../src/commonMain/proto/meta.proto | 19 ++++ .../src/jvmMain/kotlin/ProtoMetaFormat.kt | 92 +++++++++++++++++++ .../src/jvmTest/kotlin/ProtoBufTest.kt | 39 ++++++++ .../space/kscience/dataforge/meta/JsonMeta.kt | 5 +- .../space/kscience/dataforge/meta/Scheme.kt | 5 +- .../kscience/dataforge/names/NameToken.kt | 27 +++++- .../kscience/dataforge/names/NameTest.kt | 14 ++- settings.gradle.kts | 1 + 10 files changed, 233 insertions(+), 17 deletions(-) create mode 100644 dataforge-io/dataforge-io-proto/build.gradle.kts create mode 100644 dataforge-io/dataforge-io-proto/src/commonMain/proto/meta.proto create mode 100644 dataforge-io/dataforge-io-proto/src/jvmMain/kotlin/ProtoMetaFormat.kt create mode 100644 dataforge-io/dataforge-io-proto/src/jvmTest/kotlin/ProtoBufTest.kt diff --git a/dataforge-data/src/jvmTest/kotlin/space/kscience/dataforge/data/ActionsTest.kt b/dataforge-data/src/jvmTest/kotlin/space/kscience/dataforge/data/ActionsTest.kt index 13660eee..d608cbd3 100644 --- a/dataforge-data/src/jvmTest/kotlin/space/kscience/dataforge/data/ActionsTest.kt +++ b/dataforge-data/src/jvmTest/kotlin/space/kscience/dataforge/data/ActionsTest.kt @@ -1,8 +1,5 @@ package space.kscience.dataforge.data -import kotlinx.coroutines.flow.collect -import kotlinx.coroutines.flow.onEach -import kotlinx.coroutines.flow.take import kotlinx.coroutines.test.runTest import space.kscience.dataforge.actions.Action import space.kscience.dataforge.actions.invoke @@ -16,27 +13,28 @@ import kotlin.time.Duration.Companion.milliseconds internal class ActionsTest { @Test fun testStaticMapAction() = runTest(timeout = 500.milliseconds) { + val plusOne = Action.mapping { + result { it + 1 } + } + val data: DataTree = DataTree { repeat(10) { putValue(it.toString(), it) } } - val plusOne = Action.mapping { - result { it + 1 } - } val result = plusOne(data) assertEquals(2, result["1"]?.await()) } @Test fun testDynamicMapAction() = runTest(timeout = 500.milliseconds) { - val source: MutableDataTree = MutableDataTree() - val plusOne = Action.mapping { result { it + 1 } } + val source: MutableDataTree = MutableDataTree() + val result = plusOne(source) @@ -44,7 +42,7 @@ internal class ActionsTest { source.updateValue(it.toString(), it) } - result.updates.take(10).onEach { println(it.name) }.collect() +// result.updates.take(10).onEach { println(it.name) }.collect() assertEquals(2, result["1"]?.await()) } diff --git a/dataforge-io/dataforge-io-proto/build.gradle.kts b/dataforge-io/dataforge-io-proto/build.gradle.kts new file mode 100644 index 00000000..2899a2b1 --- /dev/null +++ b/dataforge-io/dataforge-io-proto/build.gradle.kts @@ -0,0 +1,32 @@ +plugins { + id("space.kscience.gradle.mpp") + id("com.squareup.wire") version "4.9.9" +} + +description = "ProtoBuf meta IO" + +kscience { + jvm() + dependencies { + api(projects.dataforgeIo) + api("com.squareup.wire:wire-runtime:4.9.9") + } + useSerialization { + protobuf() + } +} + +wire { + kotlin { + sourcePath { + srcDir("src/commonMain/proto") + } + } +} + +readme { + maturity = space.kscience.gradle.Maturity.PROTOTYPE + description = """ + ProtoBuf Meta representation + """.trimIndent() +} diff --git a/dataforge-io/dataforge-io-proto/src/commonMain/proto/meta.proto b/dataforge-io/dataforge-io-proto/src/commonMain/proto/meta.proto new file mode 100644 index 00000000..2fd33eab --- /dev/null +++ b/dataforge-io/dataforge-io-proto/src/commonMain/proto/meta.proto @@ -0,0 +1,19 @@ +syntax = "proto3"; +package space.kscience.dataforge.io.proto; + +message ProtoMeta { + message ProtoValue { + oneof value { + string stringValue = 2; + bool booleanValue = 3; + double doubleValue = 4; + float floatValue = 5; + int32 int32Value = 6; + int64 int64Value = 7; + bytes bytesValue = 8; + } + } + repeated ProtoValue value = 1; + + map items = 2; +} \ No newline at end of file diff --git a/dataforge-io/dataforge-io-proto/src/jvmMain/kotlin/ProtoMetaFormat.kt b/dataforge-io/dataforge-io-proto/src/jvmMain/kotlin/ProtoMetaFormat.kt new file mode 100644 index 00000000..5e55099b --- /dev/null +++ b/dataforge-io/dataforge-io-proto/src/jvmMain/kotlin/ProtoMetaFormat.kt @@ -0,0 +1,92 @@ +package space.kscience.dataforge.io.proto + +import kotlinx.io.Sink +import kotlinx.io.Source +import kotlinx.io.asInputStream +import kotlinx.io.asOutputStream +import org.slf4j.LoggerFactory +import space.kscience.dataforge.io.MetaFormat +import space.kscience.dataforge.meta.* +import space.kscience.dataforge.meta.descriptors.MetaDescriptor +import space.kscience.dataforge.names.NameToken + +internal class ProtoMetaWrapper(private val proto: ProtoMeta) : Meta { + + private fun ProtoMeta.ProtoValue.toValue(): Value = when { + stringValue != null -> stringValue.asValue() + booleanValue != null -> booleanValue.asValue() + doubleValue != null -> doubleValue.asValue() + floatValue != null -> floatValue.asValue() + int32Value != null -> int32Value.asValue() + int64Value != null -> int64Value.asValue() + bytesValue != null -> bytesValue.toByteArray().asValue() + else -> Null + } + + override val value: Value? + get() = when (proto.value_.size) { + 0 -> null + 1 -> proto.value_[0].toValue() + else -> proto.value_.map { it.toValue() }.asValue() + } + + + override val items: Map + get() = proto.items.entries.associate { NameToken.parse(it.key) to ProtoMetaWrapper(it.value) } + + override fun toString(): String = Meta.toString(this) + + override fun equals(other: Any?): Boolean = Meta.equals(this, other as? Meta) + + override fun hashCode(): Int = Meta.hashCode(this) +} + +internal fun Meta.toProto(): ProtoMeta { + + + fun MutableList.appendProtoValues(value: Value): Unit { + when (value.type) { + ValueType.NULL -> { + //do nothing + } + + ValueType.NUMBER -> when (value.value) { + is Int, is Short, is Byte -> add(ProtoMeta.ProtoValue(int32Value = value.int)) + is Long -> add(ProtoMeta.ProtoValue(int64Value = value.long)) + is Float -> add(ProtoMeta.ProtoValue(floatValue = value.float)) + else -> { + LoggerFactory.getLogger(ProtoMeta::class.java) + .warn("Unknown number type ${value.value} encoded as Double") + add(ProtoMeta.ProtoValue(doubleValue = value.double)) + } + } + + ValueType.STRING -> add(ProtoMeta.ProtoValue(stringValue = value.string)) + ValueType.BOOLEAN -> add(ProtoMeta.ProtoValue(booleanValue = value.boolean)) + ValueType.LIST -> { + value.list.forEach { + if (it.type == ValueType.LIST) { + error("Nested lists are not supported") + } else { + appendProtoValues(it) + } + } + } + } + } + + return ProtoMeta( + value_ = buildList { value?.let { appendProtoValues(it) } }, + items.entries.associate { it.key.toString() to it.value.toProto() } + ) +} + + +public object ProtoMetaFormat : MetaFormat { + override fun writeMeta(sink: Sink, meta: Meta, descriptor: MetaDescriptor?) { + ProtoMeta.ADAPTER.encode(sink.asOutputStream(), meta.toProto()) + } + + override fun readMeta(source: Source, descriptor: MetaDescriptor?): Meta = + ProtoMetaWrapper(ProtoMeta.ADAPTER.decode(source.asInputStream())) +} \ No newline at end of file diff --git a/dataforge-io/dataforge-io-proto/src/jvmTest/kotlin/ProtoBufTest.kt b/dataforge-io/dataforge-io-proto/src/jvmTest/kotlin/ProtoBufTest.kt new file mode 100644 index 00000000..bf819b6e --- /dev/null +++ b/dataforge-io/dataforge-io-proto/src/jvmTest/kotlin/ProtoBufTest.kt @@ -0,0 +1,39 @@ +package space.kscience.dataforge.io.proto + +import space.kscience.dataforge.meta.Meta +import space.kscience.dataforge.meta.get +import kotlin.test.Test +import kotlin.test.assertEquals + +class ProtoBufTest { + + @Test + fun testProtoBufMetaFormat(){ + val meta = Meta { + "a" put 22 + "node" put { + "b" put "DDD" + "c" put 11.1 + "d" put { + "d1" put { + "d11" put "aaa" + "d12" put "bbb" + } + "d2" put 2 + } + "array" put doubleArrayOf(1.0, 2.0, 3.0) + } + } + val buffer = kotlinx.io.Buffer() + ProtoMetaFormat.writeTo(buffer,meta) + val result = ProtoMetaFormat.readFrom(buffer) + + println(result["a"]?.value) + + meta.items.keys.forEach { + assertEquals(meta[it],result[it],"${meta[it]} != ${result[it]}") + } + + assertEquals(meta, result) + } +} \ No newline at end of file diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/JsonMeta.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/JsonMeta.kt index 8da8b2d3..049c1733 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/JsonMeta.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/JsonMeta.kt @@ -117,8 +117,11 @@ private fun MutableMap.addJsonElement( } else { val indexKey = descriptor?.indexKey ?: Meta.INDEX_KEY element.forEachIndexed { serial, childElement -> - val index = (childElement as? JsonObject)?.get(indexKey)?.jsonPrimitive?.content + + val index = (childElement as? JsonObject) + ?.get(indexKey)?.jsonPrimitive?.content ?: serial.toString() + val child: SealedMeta = when (childElement) { is JsonObject -> childElement.toMeta(descriptor) is JsonArray -> { diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/Scheme.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/Scheme.kt index fe121c42..bc05cb5d 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/Scheme.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/Scheme.kt @@ -221,13 +221,14 @@ public fun Configurable.updateWith( /** * A delegate that uses a [MetaReader] to wrap a child of this provider */ -public fun MutableMeta.scheme( +public fun MutableMetaProvider.scheme( spec: SchemeSpec, key: Name? = null, ): ReadWriteProperty = object : ReadWriteProperty { override fun getValue(thisRef: Any?, property: KProperty<*>): T { val name = key ?: property.name.asName() - return spec.write(getOrCreate(name)) + val node = get(name)?: MutableMeta().also { set(name,it) } + return spec.write(node) } override fun setValue(thisRef: Any?, property: KProperty<*>, value: T) { diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/names/NameToken.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/names/NameToken.kt index 83752b9a..3994ef27 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/names/NameToken.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/names/NameToken.kt @@ -67,10 +67,29 @@ public class NameToken(public val body: String, public val index: String? = null * Parse name token from a string */ public fun parse(string: String): NameToken { - val body = string.substringBefore('[') - val index = string.substringAfter('[', "") - if (index.isNotEmpty() && !index.endsWith(']')) error("NameToken with index must end with ']'") - return NameToken(body, index.removeSuffix("]")) + var indexStart = -1 + var indexEnd = -1 + string.forEachIndexed { index, c -> + when (c) { + '[' -> when { + indexStart >= 0 -> error("Second opening bracket not allowed in NameToken: $string") + else -> indexStart = index + } + + ']' -> when { + indexStart < 0 -> error("Closing index bracket could not be used before opening bracket in NameToken: $string") + indexEnd >= 0 -> error("Second closing bracket not allowed in NameToken: $string") + else -> indexEnd = index + } + + else -> if(indexEnd>=0) error("Symbols not allowed after index in NameToken: $string") + } + } + if(indexStart>=0 && indexEnd<0) error("Opening bracket without closing bracket not allowed in NameToken: $string") + return NameToken( + if(indexStart>=0) string.substring(0, indexStart) else string, + if(indexStart>=0) string.substring(indexStart + 1, indexEnd) else null + ) } } } diff --git a/dataforge-meta/src/commonTest/kotlin/space/kscience/dataforge/names/NameTest.kt b/dataforge-meta/src/commonTest/kotlin/space/kscience/dataforge/names/NameTest.kt index db630487..25725333 100644 --- a/dataforge-meta/src/commonTest/kotlin/space/kscience/dataforge/names/NameTest.kt +++ b/dataforge-meta/src/commonTest/kotlin/space/kscience/dataforge/names/NameTest.kt @@ -56,10 +56,22 @@ class NameTest { val token2 = NameToken.parse("token-body") assertEquals("token-body", token2.body) - assertEquals("", token2.index) + assertEquals(null, token2.index) + +// val token3 = NameToken.parse("[token-index]") +// assertEquals("", token3.body) +// assertEquals("token-index", token3.index) + + assertFails{ + NameToken.parse("[token-index]") + } assertFails { NameToken.parse("token[22") } + + assertFails { + NameToken.parse("token[22]ddd") + } } } \ No newline at end of file diff --git a/settings.gradle.kts b/settings.gradle.kts index ca872038..35eae74e 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -43,6 +43,7 @@ include( ":dataforge-meta", ":dataforge-io", ":dataforge-io:dataforge-io-yaml", + ":dataforge-io:dataforge-io-proto", ":dataforge-context", ":dataforge-data", ":dataforge-workspace", -- 2.34.1 From 7fa6617e7e85d341d7740cc252e46753cfd4c573 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Mon, 10 Jun 2024 21:22:00 +0300 Subject: [PATCH 52/62] First implementation for protobuf converter --- CHANGELOG.md | 2 ++ .../space/kscience/dataforge/meta/JsonMeta.kt | 14 +++++++++----- .../kscience/dataforge/meta/MetaConverter.kt | 8 +++++++- .../kscience/dataforge/meta/ConvertersTest.kt | 17 +++++++++++++++++ 4 files changed, 35 insertions(+), 6 deletions(-) create mode 100644 dataforge-meta/src/commonTest/kotlin/space/kscience/dataforge/meta/ConvertersTest.kt diff --git a/CHANGELOG.md b/CHANGELOG.md index 43e6bf4e..f7a2e363 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,8 @@ ### Removed ### Fixed +- Fixed NameToken parsing. +- Top level string list meta conversion. ### Security diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/JsonMeta.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/JsonMeta.kt index 049c1733..39a98927 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/JsonMeta.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/JsonMeta.kt @@ -163,11 +163,15 @@ public fun JsonObject.toMeta(descriptor: MetaDescriptor? = null): SealedMeta { public fun JsonElement.toMeta(descriptor: MetaDescriptor? = null): SealedMeta = when (this) { is JsonPrimitive -> Meta(toValue(descriptor)) is JsonObject -> toMeta(descriptor) - is JsonArray -> SealedMeta(null, - linkedMapOf().apply { - addJsonElement(Meta.JSON_ARRAY_KEY, this@toMeta, null) - } - ) + is JsonArray -> if (any { it is JsonObject }) { + SealedMeta(null, + linkedMapOf().apply { + addJsonElement(Meta.JSON_ARRAY_KEY, this@toMeta, null) + } + ) + } else{ + Meta(map { it.toValueOrNull(descriptor) ?: kotlin.error("Unreachable: should not contain objects") }.asValue()) + } } // diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MetaConverter.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MetaConverter.kt index 9baf0087..0f0e8efe 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MetaConverter.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MetaConverter.kt @@ -11,7 +11,7 @@ import space.kscience.dataforge.misc.DFExperimental /** * A converter of generic object to and from [Meta] */ -public interface MetaConverter: MetaReader { +public interface MetaConverter : MetaReader { /** * A descriptor for resulting meta @@ -116,6 +116,12 @@ public interface MetaConverter: MetaReader { override fun convert(obj: E): Meta = Meta(obj.asValue()) } + public val stringList: MetaConverter> = object : MetaConverter> { + override fun convert(obj: List): Meta = Meta(obj.map { it.asValue() }.asValue()) + + override fun readOrNull(source: Meta): List? = source.stringList + } + public fun valueList( writer: (T) -> Value = { Value.of(it) }, reader: (Value) -> T, diff --git a/dataforge-meta/src/commonTest/kotlin/space/kscience/dataforge/meta/ConvertersTest.kt b/dataforge-meta/src/commonTest/kotlin/space/kscience/dataforge/meta/ConvertersTest.kt new file mode 100644 index 00000000..fda978e4 --- /dev/null +++ b/dataforge-meta/src/commonTest/kotlin/space/kscience/dataforge/meta/ConvertersTest.kt @@ -0,0 +1,17 @@ +package space.kscience.dataforge.meta + +import kotlin.test.Test +import kotlin.test.assertEquals + +class ConvertersTest { + + @Test + fun stringListConversion() { + val list = listOf("A", "B", "C") + val meta = MetaConverter.stringList.convert(list) + val json = meta.toJson() + val reconstructedMeta = json.toMeta() + val reconstructed = MetaConverter.stringList.read(reconstructedMeta) + assertEquals(list,reconstructed) + } +} \ No newline at end of file -- 2.34.1 From e7f2f7e96f7b8d9b797ce921de209fbf931e2712 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Tue, 11 Jun 2024 12:06:48 +0300 Subject: [PATCH 53/62] Protobuf meta representation --- .../src/commonMain/proto/meta.proto | 13 ++++- .../src/jvmMain/kotlin/ProtoMetaFormat.kt | 54 +++++++------------ .../src/jvmTest/kotlin/ProtoBufTest.kt | 7 ++- 3 files changed, 38 insertions(+), 36 deletions(-) diff --git a/dataforge-io/dataforge-io-proto/src/commonMain/proto/meta.proto b/dataforge-io/dataforge-io-proto/src/commonMain/proto/meta.proto index 2fd33eab..24d33420 100644 --- a/dataforge-io/dataforge-io-proto/src/commonMain/proto/meta.proto +++ b/dataforge-io/dataforge-io-proto/src/commonMain/proto/meta.proto @@ -11,9 +11,20 @@ message ProtoMeta { int32 int32Value = 6; int64 int64Value = 7; bytes bytesValue = 8; + ProtoValueList listValue = 9; + Float64List float64ListValue = 10; } } - repeated ProtoValue value = 1; + + message ProtoValueList{ + repeated ProtoValue values = 1; + } + + message Float64List{ + repeated double values = 1 [packed=true]; + } + + ProtoValue protoValue = 1; map items = 2; } \ No newline at end of file diff --git a/dataforge-io/dataforge-io-proto/src/jvmMain/kotlin/ProtoMetaFormat.kt b/dataforge-io/dataforge-io-proto/src/jvmMain/kotlin/ProtoMetaFormat.kt index 5e55099b..e4a33549 100644 --- a/dataforge-io/dataforge-io-proto/src/jvmMain/kotlin/ProtoMetaFormat.kt +++ b/dataforge-io/dataforge-io-proto/src/jvmMain/kotlin/ProtoMetaFormat.kt @@ -12,7 +12,7 @@ import space.kscience.dataforge.names.NameToken internal class ProtoMetaWrapper(private val proto: ProtoMeta) : Meta { - private fun ProtoMeta.ProtoValue.toValue(): Value = when { + private fun ProtoMeta.ProtoValue.toValue(): Value? = when { stringValue != null -> stringValue.asValue() booleanValue != null -> booleanValue.asValue() doubleValue != null -> doubleValue.asValue() @@ -20,15 +20,13 @@ internal class ProtoMetaWrapper(private val proto: ProtoMeta) : Meta { int32Value != null -> int32Value.asValue() int64Value != null -> int64Value.asValue() bytesValue != null -> bytesValue.toByteArray().asValue() - else -> Null + listValue != null -> listValue.values.mapNotNull { it.toValue() }.asValue() + float64ListValue != null -> float64ListValue.values.map { it.asValue() }.asValue() + else -> null } override val value: Value? - get() = when (proto.value_.size) { - 0 -> null - 1 -> proto.value_[0].toValue() - else -> proto.value_.map { it.toValue() }.asValue() - } + get() = proto.protoValue?.toValue() override val items: Map @@ -44,39 +42,27 @@ internal class ProtoMetaWrapper(private val proto: ProtoMeta) : Meta { internal fun Meta.toProto(): ProtoMeta { - fun MutableList.appendProtoValues(value: Value): Unit { - when (value.type) { - ValueType.NULL -> { - //do nothing - } + fun Value.toProto(): ProtoMeta.ProtoValue = when (type) { + ValueType.NULL -> ProtoMeta.ProtoValue() - ValueType.NUMBER -> when (value.value) { - is Int, is Short, is Byte -> add(ProtoMeta.ProtoValue(int32Value = value.int)) - is Long -> add(ProtoMeta.ProtoValue(int64Value = value.long)) - is Float -> add(ProtoMeta.ProtoValue(floatValue = value.float)) - else -> { - LoggerFactory.getLogger(ProtoMeta::class.java) - .warn("Unknown number type ${value.value} encoded as Double") - add(ProtoMeta.ProtoValue(doubleValue = value.double)) - } - } - - ValueType.STRING -> add(ProtoMeta.ProtoValue(stringValue = value.string)) - ValueType.BOOLEAN -> add(ProtoMeta.ProtoValue(booleanValue = value.boolean)) - ValueType.LIST -> { - value.list.forEach { - if (it.type == ValueType.LIST) { - error("Nested lists are not supported") - } else { - appendProtoValues(it) - } - } + ValueType.NUMBER -> when (value) { + is Int, is Short, is Byte -> ProtoMeta.ProtoValue(int32Value = int) + is Long -> ProtoMeta.ProtoValue(int64Value = long) + is Float -> ProtoMeta.ProtoValue(floatValue = float) + else -> { + LoggerFactory.getLogger(ProtoMeta::class.java) + .warn("Unknown number type ${value} encoded as Double") + ProtoMeta.ProtoValue(doubleValue = double) } } + + ValueType.STRING -> ProtoMeta.ProtoValue(stringValue = string) + ValueType.BOOLEAN -> ProtoMeta.ProtoValue(booleanValue = boolean) + ValueType.LIST -> ProtoMeta.ProtoValue(listValue = ProtoMeta.ProtoValueList(list.map { it.toProto() })) } return ProtoMeta( - value_ = buildList { value?.let { appendProtoValues(it) } }, + protoValue = value?.toProto(), items.entries.associate { it.key.toString() to it.value.toProto() } ) } diff --git a/dataforge-io/dataforge-io-proto/src/jvmTest/kotlin/ProtoBufTest.kt b/dataforge-io/dataforge-io-proto/src/jvmTest/kotlin/ProtoBufTest.kt index bf819b6e..5757092c 100644 --- a/dataforge-io/dataforge-io-proto/src/jvmTest/kotlin/ProtoBufTest.kt +++ b/dataforge-io/dataforge-io-proto/src/jvmTest/kotlin/ProtoBufTest.kt @@ -1,6 +1,7 @@ package space.kscience.dataforge.io.proto import space.kscience.dataforge.meta.Meta +import space.kscience.dataforge.meta.asValue import space.kscience.dataforge.meta.get import kotlin.test.Test import kotlin.test.assertEquals @@ -22,13 +23,17 @@ class ProtoBufTest { "d2" put 2 } "array" put doubleArrayOf(1.0, 2.0, 3.0) + "array2d" put listOf( + doubleArrayOf(1.0, 2.0, 3.0).asValue(), + doubleArrayOf(1.0, 2.0, 3.0).asValue() + ).asValue() } } val buffer = kotlinx.io.Buffer() ProtoMetaFormat.writeTo(buffer,meta) val result = ProtoMetaFormat.readFrom(buffer) - println(result["a"]?.value) +// println(result["a"]?.value) meta.items.keys.forEach { assertEquals(meta[it],result[it],"${meta[it]} != ${result[it]}") -- 2.34.1 From b77fc9a0d5bdd28f100e1f1788464ce856c4462e Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Tue, 11 Jun 2024 16:24:15 +0300 Subject: [PATCH 54/62] Add Envelope encoding to protobuf --- .../dataforge-io-proto/build.gradle.kts | 1 + .../commonMain/kotlin/ProtoEnvelopeFormat.kt | 32 +++++++ .../kotlin/ProtoMetaFormat.kt | 16 ++-- .../src/commonMain/proto/meta.proto | 5 ++ .../src/commonTest/kotlin/ProtoBufTest.kt | 83 +++++++++++++++++++ .../src/jvmTest/kotlin/ProtoBufTest.kt | 44 ---------- .../space/kscience/dataforge/io/Envelope.kt | 5 +- 7 files changed, 131 insertions(+), 55 deletions(-) create mode 100644 dataforge-io/dataforge-io-proto/src/commonMain/kotlin/ProtoEnvelopeFormat.kt rename dataforge-io/dataforge-io-proto/src/{jvmMain => commonMain}/kotlin/ProtoMetaFormat.kt (85%) create mode 100644 dataforge-io/dataforge-io-proto/src/commonTest/kotlin/ProtoBufTest.kt delete mode 100644 dataforge-io/dataforge-io-proto/src/jvmTest/kotlin/ProtoBufTest.kt diff --git a/dataforge-io/dataforge-io-proto/build.gradle.kts b/dataforge-io/dataforge-io-proto/build.gradle.kts index 2899a2b1..a9099cb5 100644 --- a/dataforge-io/dataforge-io-proto/build.gradle.kts +++ b/dataforge-io/dataforge-io-proto/build.gradle.kts @@ -7,6 +7,7 @@ description = "ProtoBuf meta IO" kscience { jvm() +// js() dependencies { api(projects.dataforgeIo) api("com.squareup.wire:wire-runtime:4.9.9") diff --git a/dataforge-io/dataforge-io-proto/src/commonMain/kotlin/ProtoEnvelopeFormat.kt b/dataforge-io/dataforge-io-proto/src/commonMain/kotlin/ProtoEnvelopeFormat.kt new file mode 100644 index 00000000..d60539f5 --- /dev/null +++ b/dataforge-io/dataforge-io-proto/src/commonMain/kotlin/ProtoEnvelopeFormat.kt @@ -0,0 +1,32 @@ +package pace.kscience.dataforge.io.proto + +import kotlinx.io.Sink +import kotlinx.io.Source +import kotlinx.io.readByteArray +import okio.ByteString +import okio.ByteString.Companion.toByteString +import space.kscience.dataforge.io.Envelope +import space.kscience.dataforge.io.EnvelopeFormat +import space.kscience.dataforge.io.asBinary +import space.kscience.dataforge.io.proto.ProtoEnvelope +import space.kscience.dataforge.io.toByteArray +import space.kscience.dataforge.meta.Meta + + +public object ProtoEnvelopeFormat : EnvelopeFormat { + override fun readFrom(source: Source): Envelope { + val protoEnvelope = ProtoEnvelope.ADAPTER.decode(source.readByteArray()) + return Envelope( + meta = protoEnvelope.meta?.let { ProtoMetaWrapper(it) } ?: Meta.EMPTY, + data = protoEnvelope.dataBytes.toByteArray().asBinary() + ) + } + + override fun writeTo(sink: Sink, obj: Envelope) { + val protoEnvelope = ProtoEnvelope( + obj.meta.toProto(), + obj.data?.toByteArray()?.toByteString() ?: ByteString.EMPTY + ) + sink.write(ProtoEnvelope.ADAPTER.encode(protoEnvelope)) + } +} \ No newline at end of file diff --git a/dataforge-io/dataforge-io-proto/src/jvmMain/kotlin/ProtoMetaFormat.kt b/dataforge-io/dataforge-io-proto/src/commonMain/kotlin/ProtoMetaFormat.kt similarity index 85% rename from dataforge-io/dataforge-io-proto/src/jvmMain/kotlin/ProtoMetaFormat.kt rename to dataforge-io/dataforge-io-proto/src/commonMain/kotlin/ProtoMetaFormat.kt index e4a33549..8857832a 100644 --- a/dataforge-io/dataforge-io-proto/src/jvmMain/kotlin/ProtoMetaFormat.kt +++ b/dataforge-io/dataforge-io-proto/src/commonMain/kotlin/ProtoMetaFormat.kt @@ -1,11 +1,10 @@ -package space.kscience.dataforge.io.proto +package pace.kscience.dataforge.io.proto import kotlinx.io.Sink import kotlinx.io.Source -import kotlinx.io.asInputStream -import kotlinx.io.asOutputStream -import org.slf4j.LoggerFactory +import kotlinx.io.readByteArray import space.kscience.dataforge.io.MetaFormat +import space.kscience.dataforge.io.proto.ProtoMeta import space.kscience.dataforge.meta.* import space.kscience.dataforge.meta.descriptors.MetaDescriptor import space.kscience.dataforge.names.NameToken @@ -50,8 +49,8 @@ internal fun Meta.toProto(): ProtoMeta { is Long -> ProtoMeta.ProtoValue(int64Value = long) is Float -> ProtoMeta.ProtoValue(floatValue = float) else -> { - LoggerFactory.getLogger(ProtoMeta::class.java) - .warn("Unknown number type ${value} encoded as Double") +// LoggerFactory.getLogger(ProtoMeta::class.java) +// .warn("Unknown number type ${value} encoded as Double") ProtoMeta.ProtoValue(doubleValue = double) } } @@ -67,12 +66,11 @@ internal fun Meta.toProto(): ProtoMeta { ) } - public object ProtoMetaFormat : MetaFormat { override fun writeMeta(sink: Sink, meta: Meta, descriptor: MetaDescriptor?) { - ProtoMeta.ADAPTER.encode(sink.asOutputStream(), meta.toProto()) + sink.write(ProtoMeta.ADAPTER.encode(meta.toProto())) } override fun readMeta(source: Source, descriptor: MetaDescriptor?): Meta = - ProtoMetaWrapper(ProtoMeta.ADAPTER.decode(source.asInputStream())) + ProtoMetaWrapper(ProtoMeta.ADAPTER.decode(source.readByteArray())) } \ No newline at end of file diff --git a/dataforge-io/dataforge-io-proto/src/commonMain/proto/meta.proto b/dataforge-io/dataforge-io-proto/src/commonMain/proto/meta.proto index 24d33420..c6233872 100644 --- a/dataforge-io/dataforge-io-proto/src/commonMain/proto/meta.proto +++ b/dataforge-io/dataforge-io-proto/src/commonMain/proto/meta.proto @@ -27,4 +27,9 @@ message ProtoMeta { ProtoValue protoValue = 1; map items = 2; +} + +message ProtoEnvelope{ + ProtoMeta meta = 1; + bytes dataBytes = 2; } \ No newline at end of file diff --git a/dataforge-io/dataforge-io-proto/src/commonTest/kotlin/ProtoBufTest.kt b/dataforge-io/dataforge-io-proto/src/commonTest/kotlin/ProtoBufTest.kt new file mode 100644 index 00000000..f8abef24 --- /dev/null +++ b/dataforge-io/dataforge-io-proto/src/commonTest/kotlin/ProtoBufTest.kt @@ -0,0 +1,83 @@ +package pace.kscience.dataforge.io.proto + +import kotlinx.io.writeString +import space.kscience.dataforge.io.Envelope +import space.kscience.dataforge.io.toByteArray +import space.kscience.dataforge.meta.Meta +import space.kscience.dataforge.meta.asValue +import space.kscience.dataforge.meta.get +import kotlin.test.Test +import kotlin.test.assertContentEquals +import kotlin.test.assertEquals + +class ProtoBufTest { + + @Test + fun testProtoBufMetaFormat(){ + val meta = Meta { + "a" put 22 + "node" put { + "b" put "DDD" + "c" put 11.1 + "d" put { + "d1" put { + "d11" put "aaa" + "d12" put "bbb" + } + "d2" put 2 + } + "array" put doubleArrayOf(1.0, 2.0, 3.0) + "array2d" put listOf( + doubleArrayOf(1.0, 2.0, 3.0).asValue(), + doubleArrayOf(1.0, 2.0, 3.0).asValue() + ).asValue() + } + } + val buffer = kotlinx.io.Buffer() + ProtoMetaFormat.writeTo(buffer,meta) + val result = ProtoMetaFormat.readFrom(buffer) + +// println(result["a"]?.value) + + meta.items.keys.forEach { + assertEquals(meta[it],result[it],"${meta[it]} != ${result[it]}") + } + + assertEquals(meta, result) + } + + @Test + fun testProtoBufEnvelopeFormat(){ + val envelope = Envelope{ + meta { + "a" put 22 + "node" put { + "b" put "DDD" + "c" put 11.1 + "d" put { + "d1" put { + "d11" put "aaa" + "d12" put "bbb" + } + "d2" put 2 + } + "array" put doubleArrayOf(1.0, 2.0, 3.0) + "array2d" put listOf( + doubleArrayOf(1.0, 2.0, 3.0).asValue(), + doubleArrayOf(1.0, 2.0, 3.0).asValue() + ).asValue() + } + } + data { + writeString("Hello world!") + } + } + + val buffer = kotlinx.io.Buffer() + ProtoEnvelopeFormat.writeTo(buffer,envelope) + val result = ProtoEnvelopeFormat.readFrom(buffer) + + assertEquals(envelope.meta, result.meta) + assertContentEquals(envelope.data?.toByteArray(), result.data?.toByteArray()) + } +} \ No newline at end of file diff --git a/dataforge-io/dataforge-io-proto/src/jvmTest/kotlin/ProtoBufTest.kt b/dataforge-io/dataforge-io-proto/src/jvmTest/kotlin/ProtoBufTest.kt deleted file mode 100644 index 5757092c..00000000 --- a/dataforge-io/dataforge-io-proto/src/jvmTest/kotlin/ProtoBufTest.kt +++ /dev/null @@ -1,44 +0,0 @@ -package space.kscience.dataforge.io.proto - -import space.kscience.dataforge.meta.Meta -import space.kscience.dataforge.meta.asValue -import space.kscience.dataforge.meta.get -import kotlin.test.Test -import kotlin.test.assertEquals - -class ProtoBufTest { - - @Test - fun testProtoBufMetaFormat(){ - val meta = Meta { - "a" put 22 - "node" put { - "b" put "DDD" - "c" put 11.1 - "d" put { - "d1" put { - "d11" put "aaa" - "d12" put "bbb" - } - "d2" put 2 - } - "array" put doubleArrayOf(1.0, 2.0, 3.0) - "array2d" put listOf( - doubleArrayOf(1.0, 2.0, 3.0).asValue(), - doubleArrayOf(1.0, 2.0, 3.0).asValue() - ).asValue() - } - } - val buffer = kotlinx.io.Buffer() - ProtoMetaFormat.writeTo(buffer,meta) - val result = ProtoMetaFormat.readFrom(buffer) - -// println(result["a"]?.value) - - meta.items.keys.forEach { - assertEquals(meta[it],result[it],"${meta[it]} != ${result[it]}") - } - - assertEquals(meta, result) - } -} \ No newline at end of file diff --git a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/Envelope.kt b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/Envelope.kt index 728a0e69..43df520e 100644 --- a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/Envelope.kt +++ b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/Envelope.kt @@ -2,7 +2,6 @@ package space.kscience.dataforge.io import space.kscience.dataforge.meta.Laminate import space.kscience.dataforge.meta.Meta -import space.kscience.dataforge.meta.get import space.kscience.dataforge.meta.string import space.kscience.dataforge.names.Name import space.kscience.dataforge.names.asName @@ -34,7 +33,9 @@ public interface Envelope { } } -internal class SimpleEnvelope(override val meta: Meta, override val data: Binary?) : Envelope +internal class SimpleEnvelope(override val meta: Meta, override val data: Binary?) : Envelope{ + override fun toString(): String = "Envelope(meta=$meta, data=$data)" +} public fun Envelope(meta: Meta, data: Binary?): Envelope = SimpleEnvelope(meta, data) -- 2.34.1 From 1f1f894e0d87da2b9b8f3b65b5a4d451a402169a Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Mon, 22 Jul 2024 10:01:48 +0300 Subject: [PATCH 55/62] Documentation update --- dataforge-io/build.gradle.kts | 2 +- .../jvmMain/kotlin/performanceComparison.kt | 51 +++++++++++++++++++ .../meta/{MetaRef.kt => MetaSpec.kt} | 29 ++++++++--- .../dataforge/meta/ValueSerializer.kt | 3 ++ .../kscience/dataforge/meta/exoticValues.kt | 10 +++- 5 files changed, 87 insertions(+), 8 deletions(-) create mode 100644 dataforge-io/dataforge-io-proto/src/jvmMain/kotlin/performanceComparison.kt rename dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/{MetaRef.kt => MetaSpec.kt} (73%) diff --git a/dataforge-io/build.gradle.kts b/dataforge-io/build.gradle.kts index 5be52e61..16a6fbfc 100644 --- a/dataforge-io/build.gradle.kts +++ b/dataforge-io/build.gradle.kts @@ -4,7 +4,7 @@ plugins { description = "IO module" -val ioVersion = "0.3.1" +val ioVersion = "0.4.0" kscience { jvm() diff --git a/dataforge-io/dataforge-io-proto/src/jvmMain/kotlin/performanceComparison.kt b/dataforge-io/dataforge-io-proto/src/jvmMain/kotlin/performanceComparison.kt new file mode 100644 index 00000000..74939882 --- /dev/null +++ b/dataforge-io/dataforge-io-proto/src/jvmMain/kotlin/performanceComparison.kt @@ -0,0 +1,51 @@ +package pace.kscience.dataforge.io.proto + +import kotlinx.io.writeString +import space.kscience.dataforge.io.Envelope +import space.kscience.dataforge.meta.asValue +import kotlin.concurrent.thread +import kotlin.time.measureTime + +public fun main() { + val envelope = Envelope { + meta { + "a" put 22 + "node" put { + "b" put "DDD" + "c" put 11.1 + "d" put { + "d1" put { + "d11" put "aaa" + "d12" put "bbb" + } + "d2" put 2 + } + "array" put doubleArrayOf(1.0, 2.0, 3.0) + "array2d" put listOf( + doubleArrayOf(1.0, 2.0, 3.0).asValue(), + doubleArrayOf(1.0, 2.0, 3.0).asValue() + ).asValue() + } + } + data { + writeString("Hello world!") + } + } + + val format = ProtoEnvelopeFormat + + measureTime { + val threads = List(100) { + thread { + repeat(100000) { + val buffer = kotlinx.io.Buffer() + format.writeTo(buffer, envelope) +// println(buffer.size) + val r = format.readFrom(buffer) + } + } + } + + threads.forEach { it.join() } + }.also { println(it) } +} \ No newline at end of file diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MetaRef.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MetaSpec.kt similarity index 73% rename from dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MetaRef.kt rename to dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MetaSpec.kt index 413fe404..7af7594c 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MetaRef.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MetaSpec.kt @@ -11,7 +11,7 @@ import kotlin.properties.ReadOnlyProperty /** - * A reference to a read-only value of type [T] inside [MetaProvider] + * A reference to a read-only value of type [T] inside [MetaProvider] or writable value in [MutableMetaProvider] */ @DFExperimental public data class MetaRef( @@ -20,21 +20,36 @@ public data class MetaRef( override val descriptor: MetaDescriptor? = converter.descriptor, ) : Described +/** + * Get a value from provider by [ref] or return null if node with given name is missing + */ @DFExperimental public operator fun MetaProvider.get(ref: MetaRef): T? = get(ref.name)?.let { ref.converter.readOrNull(it) } +/** + * Set a value in a mutable provider by [ref] + */ @DFExperimental public operator fun MutableMetaProvider.set(ref: MetaRef, value: T) { set(ref.name, ref.converter.convert(value)) } +/** + * Remove a node corresponding to [ref] from a mutable provider if it exists + */ @DFExperimental -public class MetaSpec( - private val configuration: MetaDescriptorBuilder.() -> Unit = {}, -) : Described { +public fun MutableMetaProvider.remove(ref: MetaRef<*>) { + remove(ref.name) +} + +/** + * A base class for [Meta] specification that stores references to meta nodes + */ +@DFExperimental +public abstract class MetaSpec : Described { private val refs: MutableList> = mutableListOf() - private fun registerRef(ref: MetaRef<*>) { + protected fun registerRef(ref: MetaRef<*>) { refs.add(ref) } @@ -51,6 +66,8 @@ public class MetaSpec( } } + protected open fun MetaDescriptorBuilder.buildDescriptor(): Unit = Unit + override val descriptor: MetaDescriptor by lazy { MetaDescriptor { refs.forEach { ref -> @@ -58,7 +75,7 @@ public class MetaSpec( node(ref.name, ref.descriptor) } } - configuration() + buildDescriptor() } } } \ No newline at end of file diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/ValueSerializer.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/ValueSerializer.kt index dc13ef4c..0379187f 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/ValueSerializer.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/ValueSerializer.kt @@ -8,6 +8,9 @@ import kotlinx.serialization.descriptors.element import kotlinx.serialization.encoding.Decoder import kotlinx.serialization.encoding.Encoder +/** + * A serializer for [Value] + */ public object ValueSerializer : KSerializer { private val listSerializer by lazy { ListSerializer(ValueSerializer) } diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/exoticValues.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/exoticValues.kt index eb39e985..91811390 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/exoticValues.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/exoticValues.kt @@ -21,6 +21,9 @@ public class LazyParsedValue(public val string: String) : Value { override fun hashCode(): Int = string.hashCode() } +/** + * Read this string as lazily parsed value + */ public fun String.lazyParseValue(): LazyParsedValue = LazyParsedValue(this) /** @@ -47,6 +50,9 @@ public class DoubleArrayValue(override val value: DoubleArray) : Value, Iterable override fun iterator(): Iterator = value.iterator() } +/** + * A zero-copy wrapping of this [DoubleArray] in a [Value] + */ public fun DoubleArray.asValue(): Value = if (isEmpty()) Null else DoubleArrayValue(this) public val Value.doubleArray: DoubleArray @@ -75,7 +81,9 @@ public fun MutableMetaProvider.doubleArray( reader = { it?.doubleArray ?: doubleArrayOf(*default) }, ) - +/** + * A [Value] wrapping a [ByteArray] + */ public class ByteArrayValue(override val value: ByteArray) : Value, Iterable { override val type: ValueType get() = ValueType.LIST override val list: List get() = value.map { NumberValue(it) } -- 2.34.1 From b4ebdfe089f13e592cb022e994955c00df4b1f37 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Mon, 22 Jul 2024 11:34:29 +0300 Subject: [PATCH 56/62] Fix json to meta mapping --- .../space/kscience/dataforge/meta/JsonMeta.kt | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/JsonMeta.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/JsonMeta.kt index 39a98927..1817ba64 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/JsonMeta.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/JsonMeta.kt @@ -34,9 +34,9 @@ private fun Meta.toJsonWithIndex(descriptor: MetaDescriptor?, index: String?): J val childDescriptor = descriptor?.nodes?.get(body) if (list.size == 1) { val (token, element) = list.first() - //do not add an empty element - val child: JsonElement = element.toJsonWithIndex(childDescriptor, token.index) - if(token.index == null) { + //do not add an empty element + val child: JsonElement = element.toJsonWithIndex(childDescriptor, token.index) + if (token.index == null) { body to child } else { body to JsonArray(listOf(child)) @@ -106,7 +106,7 @@ private fun JsonElement.toValueOrNull(descriptor: MetaDescriptor?): Value? = whe private fun MutableMap.addJsonElement( key: String, element: JsonElement, - descriptor: MetaDescriptor? + descriptor: MetaDescriptor?, ) { when (element) { is JsonPrimitive -> put(NameToken(key), Meta(element.toValue(descriptor))) @@ -136,12 +136,14 @@ private fun MutableMap.addJsonElement( Meta(childValue) } } + is JsonPrimitive -> Meta(childElement.toValue(null)) } put(NameToken(key, index), child) } } } + is JsonObject -> { val indexKey = descriptor?.indexKey ?: Meta.INDEX_KEY val index = element[indexKey]?.jsonPrimitive?.content @@ -163,14 +165,14 @@ public fun JsonObject.toMeta(descriptor: MetaDescriptor? = null): SealedMeta { public fun JsonElement.toMeta(descriptor: MetaDescriptor? = null): SealedMeta = when (this) { is JsonPrimitive -> Meta(toValue(descriptor)) is JsonObject -> toMeta(descriptor) - is JsonArray -> if (any { it is JsonObject }) { + is JsonArray -> if (all { it is JsonPrimitive }) { + Meta(map { it.toValueOrNull(descriptor) ?: error("Unreachable: should not contain objects") }.asValue()) + } else { SealedMeta(null, linkedMapOf().apply { addJsonElement(Meta.JSON_ARRAY_KEY, this@toMeta, null) } ) - } else{ - Meta(map { it.toValueOrNull(descriptor) ?: kotlin.error("Unreachable: should not contain objects") }.asValue()) } } -- 2.34.1 From 523db20e4a0e47d4a78b414a55811010fdb8d005 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Mon, 22 Jul 2024 11:45:50 +0300 Subject: [PATCH 57/62] Fix flaky coroutines tests --- .../kscience/dataforge/data/ActionsTest.kt | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/dataforge-data/src/jvmTest/kotlin/space/kscience/dataforge/data/ActionsTest.kt b/dataforge-data/src/jvmTest/kotlin/space/kscience/dataforge/data/ActionsTest.kt index d608cbd3..ee4b05be 100644 --- a/dataforge-data/src/jvmTest/kotlin/space/kscience/dataforge/data/ActionsTest.kt +++ b/dataforge-data/src/jvmTest/kotlin/space/kscience/dataforge/data/ActionsTest.kt @@ -1,6 +1,11 @@ package space.kscience.dataforge.data +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.ExperimentalCoroutinesApi +import kotlinx.coroutines.delay +import kotlinx.coroutines.test.advanceUntilIdle import kotlinx.coroutines.test.runTest +import kotlinx.coroutines.withContext import space.kscience.dataforge.actions.Action import space.kscience.dataforge.actions.invoke import space.kscience.dataforge.actions.mapping @@ -9,7 +14,7 @@ import kotlin.test.Test import kotlin.test.assertEquals import kotlin.time.Duration.Companion.milliseconds -@OptIn(DFExperimental::class) +@OptIn(DFExperimental::class, ExperimentalCoroutinesApi::class) internal class ActionsTest { @Test fun testStaticMapAction() = runTest(timeout = 500.milliseconds) { @@ -24,6 +29,8 @@ internal class ActionsTest { } val result = plusOne(data) + + advanceUntilIdle() assertEquals(2, result["1"]?.await()) } @@ -38,8 +45,12 @@ internal class ActionsTest { val result = plusOne(source) - repeat(10) { - source.updateValue(it.toString(), it) + withContext(Dispatchers.Default) { + repeat(10) { + source.updateValue(it.toString(), it) + } + + delay(50) } // result.updates.take(10).onEach { println(it.name) }.collect() -- 2.34.1 From 5196d85da1cd87996a1232683855a3119f8318c1 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Mon, 22 Jul 2024 11:48:39 +0300 Subject: [PATCH 58/62] Add MetaConverter for DoubleArray and ByteArray --- .../kscience/dataforge/meta/exoticValues.kt | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/exoticValues.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/exoticValues.kt index 91811390..17436990 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/exoticValues.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/exoticValues.kt @@ -50,6 +50,7 @@ public class DoubleArrayValue(override val value: DoubleArray) : Value, Iterable override fun iterator(): Iterator = value.iterator() } + /** * A zero-copy wrapping of this [DoubleArray] in a [Value] */ @@ -81,6 +82,14 @@ public fun MutableMetaProvider.doubleArray( reader = { it?.doubleArray ?: doubleArrayOf(*default) }, ) +private object DoubleArrayMetaConverter : MetaConverter { + override fun readOrNull(source: Meta): DoubleArray? = source.doubleArray + + override fun convert(obj: DoubleArray): Meta = Meta(obj.asValue()) +} + +public val MetaConverter.Companion.doubleArray: MetaConverter get() = DoubleArrayMetaConverter + /** * A [Value] wrapping a [ByteArray] */ @@ -131,4 +140,12 @@ public fun MutableMetaProvider.byteArray( key, writer = { ByteArrayValue(it) }, reader = { it?.byteArray ?: byteArrayOf(*default) }, -) \ No newline at end of file +) + +private object ByteArrayMetaConverter : MetaConverter { + override fun readOrNull(source: Meta): ByteArray? = source.byteArray + + override fun convert(obj: ByteArray): Meta = Meta(obj.asValue()) +} + +public val MetaConverter.Companion.byteArray: MetaConverter get() = ByteArrayMetaConverter \ No newline at end of file -- 2.34.1 From 3807960cda2c2d7223c4057d581b215fd707e785 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Mon, 22 Jul 2024 12:08:45 +0300 Subject: [PATCH 59/62] Implement MetaRef mechanics and tests --- .../space/kscience/dataforge/meta/MetaRef.kt | 202 ++++++++++++++++++ .../kscience/dataforge/meta/MetaRefTest.kt | 35 +++ 2 files changed, 237 insertions(+) create mode 100644 dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MetaRef.kt create mode 100644 dataforge-meta/src/commonTest/kotlin/space/kscience/dataforge/meta/MetaRefTest.kt diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MetaRef.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MetaRef.kt new file mode 100644 index 00000000..ef265f0d --- /dev/null +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MetaRef.kt @@ -0,0 +1,202 @@ +package space.kscience.dataforge.meta + +import kotlinx.serialization.json.Json +import space.kscience.dataforge.meta.descriptors.Described +import space.kscience.dataforge.meta.descriptors.MetaDescriptor +import space.kscience.dataforge.meta.descriptors.MetaDescriptorBuilder +import space.kscience.dataforge.misc.DFExperimental +import space.kscience.dataforge.names.Name +import space.kscience.dataforge.names.asName +import kotlin.properties.PropertyDelegateProvider +import kotlin.properties.ReadOnlyProperty + + +/** + * A reference to a read-only value of type [T] inside [MetaProvider] or writable value in [MutableMetaProvider] + */ +@DFExperimental +public data class MetaRef( + public val name: Name, + public val converter: MetaConverter, + override val descriptor: MetaDescriptor? = converter.descriptor, +) : Described + +/** + * Get a value from provider by [ref] or return null if node with given name is missing + */ +@DFExperimental +public operator fun MetaProvider.get(ref: MetaRef): T? = get(ref.name)?.let { ref.converter.readOrNull(it) } + +/** + * Set a value in a mutable provider by [ref] + */ +@DFExperimental +public operator fun MutableMetaProvider.set(ref: MetaRef, value: T) { + set(ref.name, ref.converter.convert(value)) +} + +/** + * Remove a node corresponding to [ref] from a mutable provider if it exists + */ +@DFExperimental +public fun MutableMetaProvider.remove(ref: MetaRef<*>) { + remove(ref.name) +} + +/** + * Base storage of [MetaRef] + */ +@OptIn(DFExperimental::class) +public interface MetaRefStore : Described { + public val refs: List> +} + +/** + * A base class for [Meta] specification that stores references to meta nodes. + */ +@DFExperimental +public abstract class MetaSpec : MetaRefStore { + private val _refs: MutableList> = mutableListOf() + override val refs: List> get() = _refs + + /** + * Register a ref in this specification + */ + protected fun registerRef(ref: MetaRef<*>) { + _refs.add(ref) + } + + /** + * Create and register a ref by property name and provided converter. + * By default, uses descriptor from the converter + */ + public fun item( + converter: MetaConverter, + key: Name? = null, + descriptor: MetaDescriptor? = converter.descriptor, + ): PropertyDelegateProvider>> = + PropertyDelegateProvider { _, property -> + val ref = MetaRef(key ?: property.name.asName(), converter, descriptor) + registerRef(ref) + ReadOnlyProperty { _, _ -> + ref + } + } + + /** + * Override to provide custom [MetaDescriptor] + */ + protected open fun MetaDescriptorBuilder.buildDescriptor(): Unit = Unit + + override val descriptor: MetaDescriptor by lazy { + MetaDescriptor { + refs.forEach { ref -> + ref.descriptor?.let { + node(ref.name, ref.descriptor) + } + } + buildDescriptor() + } + } +} + +/** + * Register an item using a [descriptorBuilder] to customize descriptor + */ +@DFExperimental +public fun MetaSpec.item( + converter: MetaConverter, + key: Name? = null, + descriptorBuilder: MetaDescriptorBuilder.() -> Unit = {}, +): PropertyDelegateProvider>> = item(converter, key, MetaDescriptor { + converter.descriptor?.let { from(it) } + descriptorBuilder() +}) + +//utility methods to add different nodes + +@DFExperimental +public fun MetaSpec.metaItem( + key: Name? = null, + descriptorBuilder: MetaDescriptorBuilder.() -> Unit = {}, +): PropertyDelegateProvider>> = + item(MetaConverter.meta, key, descriptorBuilder) + +@DFExperimental +public fun MetaSpec.string( + key: Name? = null, + descriptorBuilder: MetaDescriptorBuilder.() -> Unit = {}, +): PropertyDelegateProvider>> = + item(MetaConverter.string, key, descriptorBuilder) + +@DFExperimental +public fun MetaSpec.boolean( + key: Name? = null, + descriptorBuilder: MetaDescriptorBuilder.() -> Unit = {}, +): PropertyDelegateProvider>> = + item(MetaConverter.boolean, key, descriptorBuilder) + +@DFExperimental +public fun MetaSpec.stringList( + key: Name? = null, + descriptorBuilder: MetaDescriptorBuilder.() -> Unit = {}, +): PropertyDelegateProvider>>> = + item(MetaConverter.stringList, key, descriptorBuilder) + +@DFExperimental +public fun MetaSpec.float( + key: Name? = null, + descriptorBuilder: MetaDescriptorBuilder.() -> Unit = {}, +): PropertyDelegateProvider>> = + item(MetaConverter.float, key, descriptorBuilder) + +@DFExperimental +public fun MetaSpec.double( + key: Name? = null, + descriptorBuilder: MetaDescriptorBuilder.() -> Unit = {}, +): PropertyDelegateProvider>> = + item(MetaConverter.double, key, descriptorBuilder) + +@DFExperimental +public fun MetaSpec.int( + key: Name? = null, + descriptorBuilder: MetaDescriptorBuilder.() -> Unit = {}, +): PropertyDelegateProvider>> = + item(MetaConverter.int, key, descriptorBuilder) + +@DFExperimental +public fun MetaSpec.long( + key: Name? = null, + descriptorBuilder: MetaDescriptorBuilder.() -> Unit = {}, +): PropertyDelegateProvider>> = + item(MetaConverter.long, key, descriptorBuilder) + + +@DFExperimental +public fun MetaSpec.doubleArray( + key: Name? = null, + descriptorBuilder: MetaDescriptorBuilder.() -> Unit = {}, +): PropertyDelegateProvider>> = + item(MetaConverter.doubleArray, key, descriptorBuilder) + +@DFExperimental +public fun MetaSpec.byteArray( + key: Name? = null, + descriptorBuilder: MetaDescriptorBuilder.() -> Unit = {}, +): PropertyDelegateProvider>> = + item(MetaConverter.byteArray, key, descriptorBuilder) + +@DFExperimental +public inline fun > MetaSpec.enum( + key: Name? = null, + noinline descriptorBuilder: MetaDescriptorBuilder.() -> Unit = {}, +): PropertyDelegateProvider>> = + item(MetaConverter.enum(), key, descriptorBuilder) + +@DFExperimental +public inline fun MetaSpec.serializable( + key: Name? = null, + jsonEncoder: Json = Json, + noinline descriptorBuilder: MetaDescriptorBuilder.() -> Unit = {}, +): PropertyDelegateProvider>> = + item(MetaConverter.serializable(jsonEncoder = jsonEncoder), key, descriptorBuilder) \ No newline at end of file diff --git a/dataforge-meta/src/commonTest/kotlin/space/kscience/dataforge/meta/MetaRefTest.kt b/dataforge-meta/src/commonTest/kotlin/space/kscience/dataforge/meta/MetaRefTest.kt new file mode 100644 index 00000000..a93841d9 --- /dev/null +++ b/dataforge-meta/src/commonTest/kotlin/space/kscience/dataforge/meta/MetaRefTest.kt @@ -0,0 +1,35 @@ +package space.kscience.dataforge.meta + +import kotlinx.serialization.Serializable +import space.kscience.dataforge.misc.DFExperimental +import kotlin.test.Test +import kotlin.test.assertEquals + +@DFExperimental +internal class MetaRefTest { + + @Serializable + data class XY(val x: Double, val y: Double) + + object TestMetaSpec : MetaSpec() { + val integer by int { description = "Integer value" } + val string by string { description = "String value" } + val custom by item(MetaConverter.serializable()) { description = "custom value" } + } + + @Test + fun specWriteRead() = with(TestMetaSpec){ + val meta = MutableMeta() + + meta[integer] = 22 + meta[string] = "33" + val xy = XY(33.0, -33.0) + meta[custom] = xy + + val sealed = meta.seal() + + assertEquals(22, sealed[integer]) + assertEquals("33", sealed[string]) + assertEquals(xy, sealed[custom]) + } +} \ No newline at end of file -- 2.34.1 From 8723f49efde5a699ea4b55c59abbbaef97044f9f Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Mon, 22 Jul 2024 12:08:48 +0300 Subject: [PATCH 60/62] Implement MetaRef mechanics and tests --- .../space/kscience/dataforge/meta/MetaSpec.kt | 81 ------------------- 1 file changed, 81 deletions(-) delete mode 100644 dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MetaSpec.kt diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MetaSpec.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MetaSpec.kt deleted file mode 100644 index 7af7594c..00000000 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MetaSpec.kt +++ /dev/null @@ -1,81 +0,0 @@ -package space.kscience.dataforge.meta - -import space.kscience.dataforge.meta.descriptors.Described -import space.kscience.dataforge.meta.descriptors.MetaDescriptor -import space.kscience.dataforge.meta.descriptors.MetaDescriptorBuilder -import space.kscience.dataforge.misc.DFExperimental -import space.kscience.dataforge.names.Name -import space.kscience.dataforge.names.asName -import kotlin.properties.PropertyDelegateProvider -import kotlin.properties.ReadOnlyProperty - - -/** - * A reference to a read-only value of type [T] inside [MetaProvider] or writable value in [MutableMetaProvider] - */ -@DFExperimental -public data class MetaRef( - public val name: Name, - public val converter: MetaConverter, - override val descriptor: MetaDescriptor? = converter.descriptor, -) : Described - -/** - * Get a value from provider by [ref] or return null if node with given name is missing - */ -@DFExperimental -public operator fun MetaProvider.get(ref: MetaRef): T? = get(ref.name)?.let { ref.converter.readOrNull(it) } - -/** - * Set a value in a mutable provider by [ref] - */ -@DFExperimental -public operator fun MutableMetaProvider.set(ref: MetaRef, value: T) { - set(ref.name, ref.converter.convert(value)) -} - -/** - * Remove a node corresponding to [ref] from a mutable provider if it exists - */ -@DFExperimental -public fun MutableMetaProvider.remove(ref: MetaRef<*>) { - remove(ref.name) -} - -/** - * A base class for [Meta] specification that stores references to meta nodes - */ -@DFExperimental -public abstract class MetaSpec : Described { - private val refs: MutableList> = mutableListOf() - - protected fun registerRef(ref: MetaRef<*>) { - refs.add(ref) - } - - public fun item( - converter: MetaConverter, - descriptor: MetaDescriptor? = converter.descriptor, - key: Name? = null, - ): PropertyDelegateProvider>> = - PropertyDelegateProvider { _, property -> - val ref = MetaRef(key ?: property.name.asName(), converter, descriptor) - registerRef(ref) - ReadOnlyProperty { _, _ -> - ref - } - } - - protected open fun MetaDescriptorBuilder.buildDescriptor(): Unit = Unit - - override val descriptor: MetaDescriptor by lazy { - MetaDescriptor { - refs.forEach { ref -> - ref.descriptor?.let { - node(ref.name, ref.descriptor) - } - } - buildDescriptor() - } - } -} \ No newline at end of file -- 2.34.1 From 904262a22a35c8ba8bd58db99c4bc6429433798d Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Mon, 22 Jul 2024 12:20:05 +0300 Subject: [PATCH 61/62] Add observability via MetaRef --- .../space/kscience/dataforge/meta/MetaRef.kt | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MetaRef.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MetaRef.kt index ef265f0d..2e6f3452 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MetaRef.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MetaRef.kt @@ -7,6 +7,7 @@ import space.kscience.dataforge.meta.descriptors.MetaDescriptorBuilder import space.kscience.dataforge.misc.DFExperimental import space.kscience.dataforge.names.Name import space.kscience.dataforge.names.asName +import space.kscience.dataforge.names.startsWith import kotlin.properties.PropertyDelegateProvider import kotlin.properties.ReadOnlyProperty @@ -35,6 +36,24 @@ public operator fun MutableMetaProvider.set(ref: MetaRef, value: T) { set(ref.name, ref.converter.convert(value)) } +/** + * Observe changes to specific property via given [ref]. + * + * This listener should be removed in a same way as [ObservableMeta.onChange]. + * + * @param callback an action to be performed on each change of item. Null means that the item is not present or malformed. + */ +@DFExperimental +public fun ObservableMeta.onValueChange(owner: Any?, ref: MetaRef, callback: (T?) -> Unit) { + onChange(owner) { name -> + if (name.startsWith(ref.name)) { + get(name)?.let { value -> + callback(ref.converter.readOrNull(value)) + } + } + } +} + /** * Remove a node corresponding to [ref] from a mutable provider if it exists */ -- 2.34.1 From e5902c0bfed8a33c6527918a3e143478e5e5c8dc Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Sat, 3 Aug 2024 10:04:24 +0300 Subject: [PATCH 62/62] Coroutine exception logging in context --- CHANGELOG.md | 1 + .../kotlin/space/kscience/dataforge/context/Context.kt | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f7a2e363..6c856ac5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## Unreleased ### Added +- Coroutine exception logging in context ### Changed - Simplify inheritance logic in `MutableTypedMeta` diff --git a/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/context/Context.kt b/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/context/Context.kt index c614598c..457c39fc 100644 --- a/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/context/Context.kt +++ b/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/context/Context.kt @@ -1,5 +1,6 @@ package space.kscience.dataforge.context +import kotlinx.coroutines.CoroutineExceptionHandler import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Job import kotlinx.coroutines.SupervisorJob @@ -67,7 +68,9 @@ public open class Context internal constructor( override val coroutineContext: CoroutineContext by lazy { (parent ?: Global).coroutineContext.let { parenContext -> - parenContext + coroutineContext + SupervisorJob(parenContext[Job]) + parenContext + coroutineContext + SupervisorJob(parenContext[Job]) + CoroutineExceptionHandler { _, throwable -> + logger.error(throwable) { "Exception in context $name" } + } } } -- 2.34.1