diff --git a/CHANGELOG.md b/CHANGELOG.md index 49956a9e..48da46a3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ ## [Unreleased] ### Added +- File cache for workspace +- Smart task metadata transformation for workspace - Add `readOnly` property to descriptors - Add `specOrNull` delegate to meta and Scheme - Suspended read methods to the `Binary` @@ -9,11 +11,13 @@ - More fine-grained types in Action builders. ### Changed +- `PluginFactory` no longer requires plugin class +- Collection toMap -> associateByName - Simplified `DFTL` envelope format. Closing symbols are unnecessary. Properties are discontinued. - Meta `get` method allows nullable receiver - `withDefault` functions do not add new keys to meta children and are consistent. - `dataforge.meta.values` package is merged into `dataforge.meta` for better star imports -- Kotlin 1.7.20 +- Kotlin 1.8.20 - `Factory` is now `fun interface` and uses `build` instead of `invoke`. `invoke moved to an extension. - KTor 2.0 - DataTree `items` call is blocking. diff --git a/build.gradle.kts b/build.gradle.kts index f7d70463..8df17860 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -9,7 +9,7 @@ plugins { allprojects { group = "space.kscience" - version = "0.6.1-dev-4" + version = "0.6.1-dev-5" } subprojects { diff --git a/dataforge-context/build.gradle.kts b/dataforge-context/build.gradle.kts index b8637203..be9036d0 100644 --- a/dataforge-context/build.gradle.kts +++ b/dataforge-context/build.gradle.kts @@ -9,6 +9,7 @@ kscience { js() native() useCoroutines() + useSerialization() dependencies { api(project(":dataforge-meta")) } diff --git a/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/context/AbstractPlugin.kt b/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/context/AbstractPlugin.kt index 3357a0f1..e83d301c 100644 --- a/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/context/AbstractPlugin.kt +++ b/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/context/AbstractPlugin.kt @@ -1,6 +1,7 @@ package space.kscience.dataforge.context import space.kscience.dataforge.meta.Meta +import space.kscience.dataforge.misc.DFInternal import space.kscience.dataforge.misc.Named import space.kscience.dataforge.names.Name import kotlin.properties.ReadOnlyProperty @@ -24,25 +25,33 @@ public abstract class AbstractPlugin(override val meta: Meta = Meta.EMPTY) : Plu this._context = null } - final override fun dependsOn(): Map, Meta> = dependencies + override fun dependsOn(): Map, Meta> = dependencies + + protected fun

require( + factory: PluginFactory

, + type: KClass

, + meta: Meta = Meta.EMPTY, + ): ReadOnlyProperty { + dependencies[factory] = meta + return PluginDependencyDelegate(factory, type) + } /** * Register plugin dependency and return a delegate which provides lazily initialized reference to dependent plugin */ - protected fun

require( + protected inline fun require( factory: PluginFactory

, meta: Meta = Meta.EMPTY, - ): ReadOnlyProperty { - dependencies[factory] = meta - return PluginDependencyDelegate(factory.type) - } + ): ReadOnlyProperty = require(factory, P::class, meta) } -public fun Collection.toMap(): Map = associate { it.name to it } +public fun Collection.associateByName(): Map = associate { it.name to it } -private class PluginDependencyDelegate

(val type: KClass) : ReadOnlyProperty { +private class PluginDependencyDelegate

(val factory: PluginFactory

, val type: KClass

) : + ReadOnlyProperty { + @OptIn(DFInternal::class) override fun getValue(thisRef: AbstractPlugin, property: KProperty<*>): P { if (!thisRef.isAttached) error("Plugin dependency must not be called eagerly during initialization.") - return thisRef.context.plugins[type] ?: error("Plugin with type $type not found") + return thisRef.context.plugins.getByType(type, factory.tag) ?: error("Plugin ${factory.tag} not found") } } \ No newline at end of file diff --git a/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/context/Context.kt b/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/context/Context.kt index 427d8611..bb74d605 100644 --- a/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/context/Context.kt +++ b/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/context/Context.kt @@ -6,10 +6,10 @@ import kotlinx.coroutines.SupervisorJob import space.kscience.dataforge.meta.* import space.kscience.dataforge.misc.DFExperimental import space.kscience.dataforge.misc.Named +import space.kscience.dataforge.misc.ThreadSafe import space.kscience.dataforge.names.Name import space.kscience.dataforge.provider.Provider import kotlin.coroutines.CoroutineContext -import kotlin.jvm.Synchronized /** * The local environment for anything being done in DataForge framework. Contexts are organized into tree structure with [Global] at the top. @@ -76,10 +76,10 @@ public open class Context internal constructor( * @param name the relative (tail) name of the new context. If null, uses context hash code as a marker. */ @OptIn(DFExperimental::class) - @Synchronized + @ThreadSafe public fun buildContext(name: Name? = null, block: ContextBuilder.() -> Unit = {}): Context { val existing = name?.let { childrenContexts[name] } - return existing?.modify(block)?: ContextBuilder(this, name).apply(block).build().also { + return existing?.modify(block) ?: ContextBuilder(this, name).apply(block).build().also { childrenContexts[it.name] = it } } diff --git a/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/context/LogManager.kt b/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/context/LogManager.kt index d11d0c2f..4f551b88 100644 --- a/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/context/LogManager.kt +++ b/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/context/LogManager.kt @@ -4,7 +4,6 @@ import space.kscience.dataforge.meta.Meta import space.kscience.dataforge.misc.Named import space.kscience.dataforge.names.Name import space.kscience.dataforge.names.plus -import kotlin.reflect.KClass public fun interface Logger { public fun log(tag: String, body: () -> String) @@ -66,7 +65,6 @@ public class DefaultLogManager : AbstractPlugin(), LogManager { override fun build(context: Context, meta: Meta): DefaultLogManager = DefaultLogManager() override val tag: PluginTag = PluginTag(group = PluginTag.DATAFORGE_GROUP, name = "log.default") - override val type: KClass = DefaultLogManager::class } } diff --git a/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/context/PluginBuilder.kt b/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/context/PluginBuilder.kt new file mode 100644 index 00000000..c931880c --- /dev/null +++ b/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/context/PluginBuilder.kt @@ -0,0 +1,57 @@ +package space.kscience.dataforge.context + +import space.kscience.dataforge.meta.Meta +import space.kscience.dataforge.misc.Named +import space.kscience.dataforge.names.Name + +/** + * A convenience factory to build simple plugins + */ +public class PluginBuilder( + name: String, + group: String = "", + version: String = "", +) { + public val tag: PluginTag = PluginTag(name, group, version) + + private val content = HashMap>() + private val dependencies = HashMap, Meta>() + + public fun requires( + factory: PluginFactory<*>, + meta: Meta = Meta.EMPTY, + ) { + dependencies[factory] = meta + } + + public fun provides(target: String, items: Map) { + content.getOrPut(target) { HashMap() }.putAll(items) + } + + public fun provides(target: String, vararg items: Named) { + provides(target, items.associateBy { it.name }) + } + + public fun build(): PluginFactory<*> { + + return object : PluginFactory { + override val tag: PluginTag get() = this@PluginBuilder.tag + + override fun build(context: Context, meta: Meta): Plugin = object : AbstractPlugin() { + override val tag: PluginTag get() = this@PluginBuilder.tag + + override fun content(target: String): Map = this@PluginBuilder.content[target] ?: emptyMap() + + override fun dependsOn(): Map, Meta> = this@PluginBuilder.dependencies + } + + } + } +} + +public fun PluginFactory( + name: String, + group: String = "", + version: String = "", + block: PluginBuilder.() -> Unit, +): PluginFactory<*> = PluginBuilder(name, group, version).apply(block).build() \ No newline at end of file diff --git a/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/context/PluginFactory.kt b/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/context/PluginFactory.kt index b7b0eb3a..308e3bbb 100644 --- a/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/context/PluginFactory.kt +++ b/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/context/PluginFactory.kt @@ -2,12 +2,10 @@ package space.kscience.dataforge.context import space.kscience.dataforge.meta.Meta import space.kscience.dataforge.misc.Type -import kotlin.reflect.KClass @Type(PluginFactory.TYPE) public interface PluginFactory : Factory { public val tag: PluginTag - public val type: KClass public companion object { public const val TYPE: String = "pluginFactory" @@ -20,5 +18,4 @@ public interface PluginFactory : Factory { internal class DeFactoPluginFactory(val plugin: T) : PluginFactory { override fun build(context: Context, meta: Meta): T = plugin override val tag: PluginTag get() = plugin.tag - override val type: KClass get() = plugin::class } diff --git a/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/context/PluginManager.kt b/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/context/PluginManager.kt index 6f36cacd..4a241be6 100644 --- a/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/context/PluginManager.kt +++ b/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/context/PluginManager.kt @@ -1,8 +1,10 @@ package space.kscience.dataforge.context import space.kscience.dataforge.meta.Meta +import space.kscience.dataforge.misc.DFInternal import space.kscience.dataforge.names.plus import kotlin.reflect.KClass +import kotlin.reflect.cast /** @@ -64,15 +66,17 @@ public class PluginManager internal constructor( * @param * @return */ - @Suppress("UNCHECKED_CAST") - public operator fun get(type: KClass, tag: PluginTag? = null, recursive: Boolean = true): T? = - find(recursive) { type.isInstance(it) && (tag == null || tag.matches(it.tag)) } as T? + @DFInternal + public fun getByType(type: KClass, tag: PluginTag? = null, inherit: Boolean = true): T? = + find(inherit) { type.isInstance(it) && (tag == null || tag.matches(it.tag)) }?.let { type.cast(it) } + @OptIn(DFInternal::class) public inline operator fun get(tag: PluginTag? = null, recursive: Boolean = true): T? = - get(T::class, tag, recursive) + getByType(T::class, tag, recursive) + @OptIn(DFInternal::class) public inline operator fun get(factory: PluginFactory, recursive: Boolean = true): T? = - get(factory.type, factory.tag, recursive) + getByType(T::class, factory.tag, recursive) override fun iterator(): Iterator = plugins.iterator() } diff --git a/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/context/PluginTag.kt b/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/context/PluginTag.kt index 146b61ed..b1d18c96 100644 --- a/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/context/PluginTag.kt +++ b/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/context/PluginTag.kt @@ -1,5 +1,6 @@ package space.kscience.dataforge.context +import kotlinx.serialization.Serializable import space.kscience.dataforge.meta.Meta import space.kscience.dataforge.meta.MetaRepr @@ -9,6 +10,7 @@ import space.kscience.dataforge.meta.MetaRepr * * @author Alexander Nozik */ +@Serializable public data class PluginTag( val name: String, val group: String = "", diff --git a/dataforge-context/src/jsMain/kotlin/space/kscience/dataforge/context/loggingJs.kt b/dataforge-context/src/jsMain/kotlin/space/kscience/dataforge/context/loggingJs.kt index 65b74bfa..7942183e 100644 --- a/dataforge-context/src/jsMain/kotlin/space/kscience/dataforge/context/loggingJs.kt +++ b/dataforge-context/src/jsMain/kotlin/space/kscience/dataforge/context/loggingJs.kt @@ -2,7 +2,6 @@ package space.kscience.dataforge.context import space.kscience.dataforge.meta.Meta import space.kscience.dataforge.names.Name -import kotlin.reflect.KClass public class ConsoleLogManager : AbstractPlugin(), LogManager { @@ -27,7 +26,6 @@ public class ConsoleLogManager : AbstractPlugin(), LogManager { override fun build(context: Context, meta: Meta): ConsoleLogManager = ConsoleLogManager() override val tag: PluginTag = PluginTag(group = PluginTag.DATAFORGE_GROUP, name = "log.jsConsole") - override val type: KClass = ConsoleLogManager::class } } diff --git a/dataforge-context/src/jvmMain/kotlin/space/kscience/dataforge/context/loggingJvm.kt b/dataforge-context/src/jvmMain/kotlin/space/kscience/dataforge/context/loggingJvm.kt index 38796a32..4d96aa0b 100644 --- a/dataforge-context/src/jvmMain/kotlin/space/kscience/dataforge/context/loggingJvm.kt +++ b/dataforge-context/src/jvmMain/kotlin/space/kscience/dataforge/context/loggingJvm.kt @@ -3,7 +3,6 @@ package space.kscience.dataforge.context import org.slf4j.LoggerFactory import space.kscience.dataforge.meta.Meta import space.kscience.dataforge.names.Name -import kotlin.reflect.KClass public class SlfLogManager : AbstractPlugin(), LogManager { @@ -27,7 +26,6 @@ public class SlfLogManager : AbstractPlugin(), LogManager { override fun build(context: Context, meta: Meta): SlfLogManager = SlfLogManager() override val tag: PluginTag = PluginTag(group = PluginTag.DATAFORGE_GROUP, name = "log.kotlinLogging") - override val type: KClass = SlfLogManager::class } } diff --git a/dataforge-context/src/jvmMain/kotlin/space/kscience/dataforge/provider/dfType.kt b/dataforge-context/src/jvmMain/kotlin/space/kscience/dataforge/provider/dfType.kt index 98591513..b44e509b 100644 --- a/dataforge-context/src/jvmMain/kotlin/space/kscience/dataforge/provider/dfType.kt +++ b/dataforge-context/src/jvmMain/kotlin/space/kscience/dataforge/provider/dfType.kt @@ -1,8 +1,10 @@ package space.kscience.dataforge.provider import space.kscience.dataforge.context.Context +import space.kscience.dataforge.context.PluginBuilder import space.kscience.dataforge.context.gather import space.kscience.dataforge.misc.DFExperimental +import space.kscience.dataforge.misc.Named import space.kscience.dataforge.misc.Type import space.kscience.dataforge.names.Name import kotlin.reflect.KClass @@ -35,3 +37,13 @@ public inline fun Provider.top(): Map { public inline fun Context.gather(inherit: Boolean = true): Map = gather(T::class.dfType, inherit) + +@DFExperimental +public inline fun PluginBuilder.provides(items: Map) { + provides(T::class.dfType, items) +} + +@DFExperimental +public inline fun PluginBuilder.provides(vararg items: Named) { + provides(T::class.dfType, *items) +} diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataTreeBuilder.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataTreeBuilder.kt index 746c5501..b23d594b 100644 --- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataTreeBuilder.kt +++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataTreeBuilder.kt @@ -6,11 +6,11 @@ import kotlinx.coroutines.flow.MutableSharedFlow import kotlinx.coroutines.launch import space.kscience.dataforge.meta.Meta import space.kscience.dataforge.misc.DFInternal +import space.kscience.dataforge.misc.ThreadSafe import space.kscience.dataforge.names.* import kotlin.collections.set import kotlin.coroutines.CoroutineContext import kotlin.coroutines.coroutineContext -import kotlin.jvm.Synchronized import kotlin.reflect.KType import kotlin.reflect.typeOf @@ -36,7 +36,7 @@ public class DataTreeBuilder( override val updates: MutableSharedFlow = MutableSharedFlow() - @Synchronized + @ThreadSafe private fun remove(token: NameToken) { if (treeItems.remove(token) != null) { launch { @@ -50,12 +50,12 @@ public class DataTreeBuilder( (getItem(name.cutLast()).tree as? DataTreeBuilder)?.remove(name.lastOrNull()!!) } - @Synchronized + @ThreadSafe private fun set(token: NameToken, data: Data) { treeItems[token] = DataTreeItem.Leaf(data) } - @Synchronized + @ThreadSafe private fun set(token: NameToken, node: DataTree) { treeItems[token] = DataTreeItem.Node(node) } @@ -103,7 +103,7 @@ public fun DataSource( block: DataSourceBuilder.() -> Unit, ): DataTreeBuilder = DataTreeBuilder(type, parent.coroutineContext).apply(block) -@Suppress("OPT_IN_USAGE","FunctionName") +@Suppress("OPT_IN_USAGE", "FunctionName") public inline fun DataSource( parent: CoroutineScope, crossinline block: DataSourceBuilder.() -> Unit, diff --git a/dataforge-io/dataforge-io-yaml/src/commonMain/kotlin/space/kscience/dataforge/io/yaml/YamlPlugin.kt b/dataforge-io/dataforge-io-yaml/src/commonMain/kotlin/space/kscience/dataforge/io/yaml/YamlPlugin.kt index c26487a5..f707f513 100644 --- a/dataforge-io/dataforge-io-yaml/src/commonMain/kotlin/space/kscience/dataforge/io/yaml/YamlPlugin.kt +++ b/dataforge-io/dataforge-io-yaml/src/commonMain/kotlin/space/kscience/dataforge/io/yaml/YamlPlugin.kt @@ -8,10 +8,8 @@ import space.kscience.dataforge.io.EnvelopeFormatFactory import space.kscience.dataforge.io.IOPlugin import space.kscience.dataforge.io.MetaFormatFactory import space.kscience.dataforge.meta.Meta -import space.kscience.dataforge.misc.DFExperimental import space.kscience.dataforge.names.Name import space.kscience.dataforge.names.asName -import kotlin.reflect.KClass public class YamlPlugin(meta: Meta) : AbstractPlugin(meta) { public val io: IOPlugin by require(IOPlugin) @@ -27,7 +25,6 @@ public class YamlPlugin(meta: Meta) : AbstractPlugin(meta) { public companion object : PluginFactory { override val tag: PluginTag = PluginTag("io.yaml", group = PluginTag.DATAFORGE_GROUP) - override val type: KClass = YamlPlugin::class override fun build(context: Context, meta: Meta): YamlPlugin = YamlPlugin(meta) } } \ No newline at end of file diff --git a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/IOPlugin.kt b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/IOPlugin.kt index dd890a77..c3248021 100644 --- a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/IOPlugin.kt +++ b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/IOPlugin.kt @@ -9,7 +9,6 @@ import space.kscience.dataforge.meta.get import space.kscience.dataforge.meta.string import space.kscience.dataforge.misc.DFInternal import space.kscience.dataforge.names.Name -import kotlin.reflect.KClass import kotlin.reflect.KType import kotlin.reflect.typeOf @@ -54,8 +53,8 @@ public class IOPlugin(meta: Meta) : AbstractPlugin(meta) { } override fun content(target: String): Map = when (target) { - META_FORMAT_TYPE -> defaultMetaFormats.toMap() - ENVELOPE_FORMAT_TYPE -> defaultEnvelopeFormats.toMap() + META_FORMAT_TYPE -> defaultMetaFormats.associateByName() + ENVELOPE_FORMAT_TYPE -> defaultEnvelopeFormats.associateByName() IO_FORMAT_TYPE -> content(META_FORMAT_TYPE) + content(ENVELOPE_FORMAT_TYPE) else -> super.content(target) } @@ -69,7 +68,6 @@ public class IOPlugin(meta: Meta) : AbstractPlugin(meta) { override val tag: PluginTag = PluginTag("io", group = PluginTag.DATAFORGE_GROUP) - override val type: KClass = IOPlugin::class override fun build(context: Context, meta: Meta): IOPlugin = IOPlugin(meta) public val WORK_DIRECTORY_KEY: Name = Name.of("io", "workDirectory") diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MutableMeta.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MutableMeta.kt index ffd1d6e1..b8e565f7 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MutableMeta.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MutableMeta.kt @@ -2,9 +2,9 @@ package space.kscience.dataforge.meta import kotlinx.serialization.Serializable import space.kscience.dataforge.misc.DFExperimental +import space.kscience.dataforge.misc.ThreadSafe import space.kscience.dataforge.names.* import kotlin.js.JsName -import kotlin.jvm.Synchronized /** @@ -248,10 +248,10 @@ private fun ObservableMeta.adoptBy(parent: MutableMetaImpl, key: NameToken) { */ private class MutableMetaImpl( value: Value?, - children: Map = emptyMap() + children: Map = emptyMap(), ) : AbstractObservableMeta(), ObservableMutableMeta { override var value = value - @Synchronized set(value) { + @ThreadSafe set(value) { val oldValue = field field = value if (oldValue != value) { @@ -292,11 +292,11 @@ private class MutableMetaImpl( override fun getOrCreate(name: Name): ObservableMutableMeta = if (name.isEmpty()) this else get(name) ?: createNode(name) - @Synchronized + @ThreadSafe private fun replaceItem( key: NameToken, oldItem: ObservableMutableMeta?, - newItem: ObservableMutableMeta? + newItem: ObservableMutableMeta?, ) { if (oldItem != newItem) { if (newItem == null) { @@ -318,7 +318,7 @@ private class MutableMetaImpl( } ) - @Synchronized + @ThreadSafe override fun setMeta(name: Name, node: Meta?) { val oldItem: ObservableMutableMeta? = get(name) if (oldItem != node) { @@ -337,6 +337,7 @@ private class MutableMetaImpl( children[token] = newNode } } + else -> { val token = name.firstOrNull()!! //get existing or create new node. @@ -401,7 +402,7 @@ public inline fun Meta.copy(block: MutableMeta.() -> Unit = {}): Meta = private class MutableMetaWithDefault( - val source: MutableMeta, val default: MetaProvider, val rootName: Name + val source: MutableMeta, val default: MetaProvider, val rootName: Name, ) : MutableMeta by source { override val items: Map get() { diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/ObservableMeta.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/ObservableMeta.kt index 8fa432cc..835c7e9a 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/ObservableMeta.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/ObservableMeta.kt @@ -1,7 +1,7 @@ package space.kscience.dataforge.meta +import space.kscience.dataforge.misc.ThreadSafe import space.kscience.dataforge.names.* -import kotlin.jvm.Synchronized import kotlin.reflect.KProperty1 @@ -54,12 +54,12 @@ internal abstract class AbstractObservableMeta : ObservableMeta { listeners.forEach { it.callback(this, name) } } - @Synchronized + @ThreadSafe override fun onChange(owner: Any?, callback: Meta.(name: Name) -> Unit) { listeners.add(MetaListener(owner, callback)) } - @Synchronized + @ThreadSafe override fun removeListener(owner: Any?) { listeners.removeAll { it.owner === owner } } diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/ObservableMetaWrapper.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/ObservableMetaWrapper.kt index 2172f40d..d244e8d3 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/ObservableMetaWrapper.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/ObservableMetaWrapper.kt @@ -1,8 +1,8 @@ package space.kscience.dataforge.meta import space.kscience.dataforge.misc.DFExperimental +import space.kscience.dataforge.misc.ThreadSafe import space.kscience.dataforge.names.* -import kotlin.jvm.Synchronized /** * A class that takes [MutableMeta] provider and adds obsevability on top of that @@ -10,7 +10,7 @@ import kotlin.jvm.Synchronized private class ObservableMetaWrapper( val root: MutableMeta, val absoluteName: Name, - val listeners: MutableSet + val listeners: MutableSet, ) : ObservableMutableMeta { override val items: Map get() = root.items.keys.associateWith { @@ -20,7 +20,7 @@ private class ObservableMetaWrapper( override fun getMeta(name: Name): ObservableMutableMeta? = root.getMeta(name)?.let { ObservableMetaWrapper(root, this.absoluteName + name, listeners) } - @Synchronized + @ThreadSafe override fun onChange(owner: Any?, callback: Meta.(name: Name) -> Unit) { listeners.add( MetaListener(Pair(owner, absoluteName)) { name -> diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/Scheme.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/Scheme.kt index e61b740d..269ec86a 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/Scheme.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/Scheme.kt @@ -5,8 +5,8 @@ import space.kscience.dataforge.meta.descriptors.MetaDescriptor import space.kscience.dataforge.meta.descriptors.get import space.kscience.dataforge.meta.descriptors.validate import space.kscience.dataforge.misc.DFExperimental +import space.kscience.dataforge.misc.ThreadSafe import space.kscience.dataforge.names.* -import kotlin.jvm.Synchronized /** * A base for delegate-based or descriptor-based scheme. [Scheme] has an empty constructor to simplify usage from [Specification]. @@ -92,7 +92,7 @@ public open class Scheme : Described, MetaRepr, MutableMetaProvider, Configurabl listeners.forEach { it.callback(this@Scheme.meta, pathName + name) } } - @Synchronized + @ThreadSafe override fun onChange(owner: Any?, callback: Meta.(name: Name) -> Unit) { listeners.add(MetaListener(owner) { changedName -> if (changedName.startsWith(pathName)) { @@ -101,7 +101,7 @@ public open class Scheme : Described, MetaRepr, MutableMetaProvider, Configurabl }) } - @Synchronized + @ThreadSafe override fun removeListener(owner: Any?) { listeners.removeAll { it.owner === owner } } diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/misc/ThreadSafe.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/misc/ThreadSafe.kt new file mode 100644 index 00000000..856cf5e2 --- /dev/null +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/misc/ThreadSafe.kt @@ -0,0 +1,7 @@ +package space.kscience.dataforge.misc + +@OptIn(ExperimentalMultiplatform::class) +@Target(AnnotationTarget.FUNCTION, AnnotationTarget.PROPERTY_GETTER, AnnotationTarget.PROPERTY_SETTER) +@MustBeDocumented +@OptionalExpectation +public expect annotation class ThreadSafe() diff --git a/dataforge-meta/src/jvmMain/kotlin/space/kscience/dataforge/misc/threadSafeJvm.kt b/dataforge-meta/src/jvmMain/kotlin/space/kscience/dataforge/misc/threadSafeJvm.kt new file mode 100644 index 00000000..22a0f0d3 --- /dev/null +++ b/dataforge-meta/src/jvmMain/kotlin/space/kscience/dataforge/misc/threadSafeJvm.kt @@ -0,0 +1,3 @@ +package space.kscience.dataforge.misc + +public actual typealias ThreadSafe = Synchronized diff --git a/dataforge-scripting/build.gradle.kts b/dataforge-scripting/build.gradle.kts index 31a38f95..be81fe70 100644 --- a/dataforge-scripting/build.gradle.kts +++ b/dataforge-scripting/build.gradle.kts @@ -4,7 +4,6 @@ plugins { kscience{ jvm() - js() dependencies { api(projects.dataforgeWorkspace) implementation(kotlin("scripting-common")) diff --git a/dataforge-workspace/build.gradle.kts b/dataforge-workspace/build.gradle.kts index c784cc68..3e20f40f 100644 --- a/dataforge-workspace/build.gradle.kts +++ b/dataforge-workspace/build.gradle.kts @@ -7,6 +7,9 @@ kscience{ js() native() useCoroutines() + useSerialization{ + protobuf() + } dependencies { api(projects.dataforgeContext) api(projects.dataforgeData) diff --git a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/InMemoryWorkspaceCache.kt b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/InMemoryWorkspaceCache.kt new file mode 100644 index 00000000..53968c21 --- /dev/null +++ b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/InMemoryWorkspaceCache.kt @@ -0,0 +1,38 @@ +package space.kscience.dataforge.workspace + +import space.kscience.dataforge.meta.Meta +import space.kscience.dataforge.names.Name +import kotlin.reflect.KType + +private typealias TaskResultId = Pair + + +public class InMemoryWorkspaceCache : WorkspaceCache { + + // never do that at home! + private val cache = HashMap>>() + + //TODO do actual check + @Suppress("UNUSED_PARAMETER") + private fun TaskData<*>.checkType(taskType: KType): TaskData = this as TaskData + + override suspend fun evaluate(result: TaskResult): TaskResult { + for (d: TaskData in result) { + cache.getOrPut(result.taskName to result.taskMeta) { HashMap() }.getOrPut(d.name) { d } + } + + return object : TaskResult by result { + override fun iterator(): Iterator> = (cache[result.taskName to result.taskMeta] + ?.values?.map { it.checkType(result.dataType) } + ?: emptyList()).iterator() + + override fun get(name: Name): TaskData? { + val cached: TaskData<*> = cache[result.taskName to result.taskMeta]?.get(name) ?: return null + //TODO check types + return cached.checkType(result.dataType) + } + } + } +} + +public fun WorkspaceBuilder.inMemoryCache(): Unit = cache(InMemoryWorkspaceCache()) \ No newline at end of file diff --git a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/WorkspaceBuilder.kt b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/WorkspaceBuilder.kt index 7784bfcc..1538460f 100644 --- a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/WorkspaceBuilder.kt +++ b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/WorkspaceBuilder.kt @@ -124,15 +124,15 @@ public class WorkspaceBuilder(private val parentContext: Context = Global) : Tas tasks[taskName] = task } - public fun useCache() { - cache = InMemoryWorkspaceCache() + public fun cache(cache: WorkspaceCache) { + this.cache = cache } public fun build(): Workspace { val postProcess: suspend (TaskResult<*>) -> TaskResult<*> = { result -> cache?.evaluate(result) ?: result } - return WorkspaceBase(context ?: parentContext, data ?: DataSet.EMPTY, targets, tasks, postProcess) + return WorkspaceImpl(context ?: parentContext, data ?: DataSet.EMPTY, targets, tasks, postProcess) } } diff --git a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/WorkspaceCache.kt b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/WorkspaceCache.kt index af0dc0bf..62df6744 100644 --- a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/WorkspaceCache.kt +++ b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/WorkspaceCache.kt @@ -1,40 +1,5 @@ package space.kscience.dataforge.workspace -import space.kscience.dataforge.meta.Meta -import space.kscience.dataforge.names.Name -import kotlin.reflect.KType - public interface WorkspaceCache { public suspend fun evaluate(result: TaskResult): TaskResult } - -private typealias TaskResultId = Pair - -public class InMemoryWorkspaceCache : WorkspaceCache { - - // never do that at home! - private val cache = HashMap>>() - - //TODO do actual check - @Suppress("UNUSED_PARAMETER") - private fun TaskData<*>.checkType(taskType: KType): TaskData = this as TaskData - - override suspend fun evaluate(result: TaskResult): TaskResult { - for (d: TaskData in result) { - cache.getOrPut(result.taskName to result.taskMeta) { HashMap() }.getOrPut(d.name) { d } - } - - return object : TaskResult by result { - override fun iterator(): Iterator> = (cache[result.taskName to result.taskMeta] - ?.values?.map { it.checkType(result.dataType) } - ?: emptyList()).iterator() - - override fun get(name: Name): TaskData? { - val cached: TaskData<*> = cache[result.taskName to result.taskMeta]?.get(name) ?: return null - //TODO check types - return cached.checkType(result.dataType) - } - } - } -} - diff --git a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/WorkspaceBase.kt b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/WorkspaceImpl.kt similarity index 90% rename from dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/WorkspaceBase.kt rename to dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/WorkspaceImpl.kt index 411605a3..dae9667a 100644 --- a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/WorkspaceBase.kt +++ b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/WorkspaceImpl.kt @@ -7,10 +7,7 @@ import space.kscience.dataforge.meta.Meta import space.kscience.dataforge.names.Name -/** - * A simple workspace without caching - */ -public class WorkspaceBase internal constructor( +internal class WorkspaceImpl internal constructor( override val context: Context, data: DataSet<*>, override val targets: Map, diff --git a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/taskBuilders.kt b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/taskBuilders.kt index baf17ee1..09791d15 100644 --- a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/taskBuilders.kt +++ b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/taskBuilders.kt @@ -4,34 +4,56 @@ import space.kscience.dataforge.context.PluginFactory import space.kscience.dataforge.data.DataSet import space.kscience.dataforge.data.forEach import space.kscience.dataforge.data.map -import space.kscience.dataforge.meta.Meta -import space.kscience.dataforge.meta.MutableMeta -import space.kscience.dataforge.meta.toMutableMeta +import space.kscience.dataforge.meta.* import space.kscience.dataforge.misc.DFExperimental import space.kscience.dataforge.names.Name import space.kscience.dataforge.names.plus +/** + * A task meta without a node corresponding to the task itself (removing a node with name of the task). + */ +public val TaskResultBuilder<*>.defaultDependencyMeta: Meta + get() = taskMeta.copy { + remove(taskName) + } + /** * Select data using given [selector] + * + * @param selector a workspace data selector. Could be either task selector or initial data selector. + * @param dependencyMeta meta used for selector. The same meta is used for caching. By default, uses [defaultDependencyMeta]. */ public suspend fun TaskResultBuilder<*>.from( selector: DataSelector, - meta: Meta = taskMeta -): DataSet = selector.select(workspace, meta) + dependencyMeta: Meta = defaultDependencyMeta, +): DataSet = selector.select(workspace, dependencyMeta) +public suspend inline fun TaskResultBuilder<*>.from( + plugin: P, + dependencyMeta: Meta = defaultDependencyMeta, + selectorBuilder: P.() -> TaskReference, +): DataSet { + require(workspace.context.plugins.contains(plugin)){"Plugin $plugin is not loaded into $workspace"} + val taskReference: TaskReference = plugin.selectorBuilder() + return workspace.produce(plugin.name + taskReference.taskName, dependencyMeta) as TaskResult +} /** * Select data from a [WorkspacePlugin] attached to this [Workspace] context. + * + * @param pluginFactory a plugin which contains the task definition. The plugin must be loaded into Workspace context. + * @param dependencyMeta meta used for selector. The same meta is used for caching. By default, uses [defaultDependencyMeta]. + * @param selectorBuilder a builder of task from the plugin. */ public suspend inline fun TaskResultBuilder<*>.from( pluginFactory: PluginFactory

, - meta: Meta = taskMeta, + dependencyMeta: Meta = defaultDependencyMeta, selectorBuilder: P.() -> TaskReference, ): DataSet { val plugin = workspace.context.plugins[pluginFactory] ?: error("Plugin ${pluginFactory.tag} not loaded into workspace context") val taskReference: TaskReference = plugin.selectorBuilder() - return workspace.produce(plugin.name + taskReference.taskName, meta) as TaskResult + return workspace.produce(plugin.name + taskReference.taskName, dependencyMeta) as TaskResult } public val TaskResultBuilder<*>.allData: DataSelector<*> @@ -42,18 +64,23 @@ public val TaskResultBuilder<*>.allData: DataSelector<*> /** * Perform a lazy mapping task using given [selector] and [action]. The meta of resulting * TODO move selector to receiver with multi-receivers + * + * @param selector a workspace data selector. Could be either task selector or initial data selector. + * @param dependencyMeta meta used for selector. The same meta is used for caching. By default, uses [defaultDependencyMeta]. + * @param dataMetaTransform additional transformation of individual data meta. + * @param action process individual data asynchronously. */ @DFExperimental public suspend inline fun TaskResultBuilder.pipeFrom( selector: DataSelector, - selectorMeta: Meta = taskMeta, - dataMetaTransform: MutableMeta.() -> Unit = {}, + dependencyMeta: Meta = defaultDependencyMeta, + dataMetaTransform: MutableMeta.(name: Name) -> Unit = {}, crossinline action: suspend (arg: T, name: Name, meta: Meta) -> R, ) { - from(selector, selectorMeta).forEach { data -> + from(selector, dependencyMeta).forEach { data -> val meta = data.meta.toMutableMeta().apply { - taskName put taskMeta - dataMetaTransform() + taskMeta[taskName]?.let { taskName.put(it) } + dataMetaTransform(data.name) } val res = data.map(workspace.context.coroutineContext, meta) { diff --git a/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/FileWorkspaceCache.kt b/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/FileWorkspaceCache.kt index 16af238b..5d17baf3 100644 --- a/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/FileWorkspaceCache.kt +++ b/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/FileWorkspaceCache.kt @@ -1,5 +1,15 @@ package space.kscience.dataforge.workspace +import io.ktor.utils.io.core.Input +import io.ktor.utils.io.core.Output +import io.ktor.utils.io.core.readBytes +import io.ktor.utils.io.core.writeFully +import kotlinx.serialization.KSerializer +import kotlinx.serialization.json.Json +import kotlinx.serialization.protobuf.ProtoBuf +import kotlinx.serialization.serializer +import space.kscience.dataforge.context.error +import space.kscience.dataforge.context.logger import space.kscience.dataforge.context.request import space.kscience.dataforge.data.Data import space.kscience.dataforge.data.await @@ -14,9 +24,32 @@ import kotlin.io.path.div import kotlin.io.path.exists import kotlin.reflect.KType +public class JsonIOFormat(override val type: KType) : IOFormat { + + private val serializer: KSerializer = serializer(type) as KSerializer + + override fun readObject(input: Input): T = Json.decodeFromString(serializer, input.readUtf8String()) + + override fun writeObject(output: Output, obj: T) { + output.writeUtf8String(Json.encodeToString(serializer, obj)) + } +} + +public class ProtobufIOFormat(override val type: KType) : IOFormat { + + private val serializer: KSerializer = serializer(type) as KSerializer + + override fun readObject(input: Input): T = ProtoBuf.decodeFromByteArray(serializer, input.readBytes()) + + override fun writeObject(output: Output, obj: T) { + output.writeFully(ProtoBuf.encodeToByteArray(serializer, obj)) + } +} + + public class FileWorkspaceCache(public val cacheDirectory: Path) : WorkspaceCache { - private fun TaskData<*>.checkType(taskType: KType): TaskData = this as TaskData +// private fun TaskData<*>.checkType(taskType: KType): TaskData = this as TaskData @OptIn(DFExperimental::class, DFInternal::class) @@ -24,23 +57,24 @@ public class FileWorkspaceCache(public val cacheDirectory: Path) : WorkspaceCach val io = result.workspace.context.request(IOPlugin) val format: IOFormat = io.resolveIOFormat(result.dataType, result.taskMeta) + ?: ProtobufIOFormat(result.dataType) ?: error("Can't resolve IOFormat for ${result.dataType}") - fun cachedDataPath(dataName: Name): Path = cacheDirectory / - result.taskName.withIndex(result.taskMeta.hashCode().toString(16)).toString() / - dataName.toString() - fun evaluateDatum(data: TaskData): TaskData { - val path = cachedDataPath(data.name) + + val path = cacheDirectory / + result.taskName.withIndex(result.taskMeta.hashCode().toString(16)).toString() / + data.name.toString() + val datum: Data = Data(data.type, meta = data.meta, dependencies = data.dependencies) { // return cached data if it is present if (path.exists()) { try { val envelope: Envelope = io.readEnvelopeFile(path) if (envelope.meta != data.meta) error("Wrong metadata in cached result file") - return@Data envelope.data?.readWith(format) - ?: error("Can't convert envelope without data to Data") + return@Data (envelope.data ?: Binary.EMPTY).readWith(format) } catch (ex: Exception) { + result.workspace.logger.error { "Failed to read data from cache: ${ex.localizedMessage}" } //cleanup cache file path.deleteIfExists() } @@ -63,9 +97,11 @@ public class FileWorkspaceCache(public val cacheDirectory: Path) : WorkspaceCach return object : TaskResult by result { override fun iterator(): Iterator> = - iterator().asSequence().map { evaluateDatum(it) }.iterator() + result.iterator().asSequence().map { evaluateDatum(it) }.iterator() override fun get(name: Name): TaskData? = result[name]?.let { evaluateDatum(it) } } } -} \ No newline at end of file +} + +public fun WorkspaceBuilder.fileCache(cacheDir: Path): Unit = cache(FileWorkspaceCache(cacheDir)) \ No newline at end of file diff --git a/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/workspaceJvm.kt b/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/workspaceJvm.kt index 2d368d09..ea6ffb85 100644 --- a/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/workspaceJvm.kt +++ b/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/workspaceJvm.kt @@ -15,7 +15,7 @@ import space.kscience.dataforge.names.matches * Select the whole data set from the workspace filtered by type. */ @OptIn(DFExperimental::class) -public inline fun TaskResultBuilder<*>.data(namePattern: Name? = null): DataSelector = +public inline fun TaskResultBuilder<*>.dataByType(namePattern: Name? = null): DataSelector = object : DataSelector { override suspend fun select(workspace: Workspace, meta: Meta): DataSet = workspace.data.filterByType { name, _ -> diff --git a/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/CachingWorkspaceTest.kt b/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/CachingWorkspaceTest.kt index 8bcbf736..4e1923bc 100644 --- a/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/CachingWorkspaceTest.kt +++ b/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/CachingWorkspaceTest.kt @@ -1,6 +1,7 @@ package space.kscience.dataforge.workspace import kotlinx.coroutines.ExperimentalCoroutinesApi +import kotlinx.coroutines.coroutineScope import kotlinx.coroutines.test.runTest import org.junit.jupiter.api.Test import space.kscience.dataforge.data.startAll @@ -9,41 +10,58 @@ import space.kscience.dataforge.meta.Meta import space.kscience.dataforge.meta.boolean import space.kscience.dataforge.meta.get import space.kscience.dataforge.misc.DFExperimental +import kotlin.test.assertEquals @OptIn(ExperimentalCoroutinesApi::class, DFExperimental::class) internal class CachingWorkspaceTest { - private val workspace = Workspace { - data { - //statically initialize data - repeat(5) { - static("myData[$it]", it) - } - } - - useCache() - - val doFirst by task { - pipeFrom(data()) { _, name, _ -> - println("Done first on $name with flag=${taskMeta["flag"].boolean ?: false}") - } - } - - @Suppress("UNUSED_VARIABLE") val doSecond by task{ - pipeFrom(doFirst) { _, name, _ -> - println("Done second on $name with flag=${taskMeta["flag"].boolean ?: false}") - } - } - } - @Test fun testMetaPropagation() = runTest { + var firstCounter = 0 + var secondCounter = 0 + + val workspace = Workspace { + data { + //statically initialize data + repeat(5) { + static("myData[$it]", it) + } + } + + inMemoryCache() + + val doFirst by task { + pipeFrom(allData) { _, name, _ -> + firstCounter++ + println("Done first on $name with flag=${taskMeta["flag"].boolean}") + } + } + + @Suppress("UNUSED_VARIABLE") + val doSecond by task { + pipeFrom( + doFirst, + dependencyMeta = if(taskMeta["flag"].boolean == true) taskMeta else Meta.EMPTY + ) { _, name, _ -> + secondCounter++ + println("Done second on $name with flag=${taskMeta["flag"].boolean ?: false}") + } + } + } + val first = workspace.produce("doFirst") val secondA = workspace.produce("doSecond") val secondB = workspace.produce("doSecond", Meta { "flag" put true }) - first.startAll(this) - secondA.startAll(this) - secondB.startAll(this) + val secondC = workspace.produce("doSecond") + coroutineScope { + first.startAll(this) + secondA.startAll(this) + secondB.startAll(this) + //repeat to check caching + secondC.startAll(this) + } + assertEquals(10, firstCounter) + assertEquals(10, secondCounter) } } \ No newline at end of file diff --git a/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/DataPropagationTest.kt b/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/DataPropagationTest.kt index b0c2ebf4..18086902 100644 --- a/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/DataPropagationTest.kt +++ b/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/DataPropagationTest.kt @@ -9,7 +9,6 @@ import space.kscience.dataforge.context.PluginFactory import space.kscience.dataforge.context.PluginTag import space.kscience.dataforge.data.* import space.kscience.dataforge.meta.Meta -import kotlin.reflect.KClass import kotlin.test.Test import kotlin.test.assertEquals @@ -34,7 +33,6 @@ class DataPropagationTestPlugin : WorkspacePlugin() { companion object : PluginFactory { - override val type: KClass = DataPropagationTestPlugin::class override fun build(context: Context, meta: Meta): DataPropagationTestPlugin = DataPropagationTestPlugin() diff --git a/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/FileWorkspaceCacheTest.kt b/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/FileWorkspaceCacheTest.kt new file mode 100644 index 00000000..9d427a76 --- /dev/null +++ b/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/FileWorkspaceCacheTest.kt @@ -0,0 +1,33 @@ +package space.kscience.dataforge.workspace + +import kotlinx.coroutines.ExperimentalCoroutinesApi +import kotlinx.coroutines.test.runTest +import org.junit.jupiter.api.Test +import space.kscience.dataforge.data.startAll +import space.kscience.dataforge.data.static +import space.kscience.dataforge.misc.DFExperimental +import java.nio.file.Files + +@OptIn(ExperimentalCoroutinesApi::class,DFExperimental::class) +class FileWorkspaceCacheTest { + + @Test + fun testCaching() = runTest { + val workspace = Workspace { + data { + //statically initialize data + repeat(5) { + static("myData[$it]", it) + } + } + fileCache(Files.createTempDirectory("dataforge-temporary-cache")) + + task { + pipeFrom(dataByType()) { arg, _, _ -> arg } + } + } + + workspace.produce("echo").startAll(this) + + } +} \ No newline at end of file diff --git a/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/SimpleWorkspaceTest.kt b/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/SimpleWorkspaceTest.kt index 5e06f45c..7bfe0927 100644 --- a/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/SimpleWorkspaceTest.kt +++ b/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/SimpleWorkspaceTest.kt @@ -13,7 +13,6 @@ import space.kscience.dataforge.meta.* import space.kscience.dataforge.misc.DFExperimental import space.kscience.dataforge.names.get import space.kscience.dataforge.names.plus -import kotlin.reflect.KClass import kotlin.test.Test import kotlin.test.assertEquals import kotlin.test.assertTrue @@ -22,11 +21,10 @@ import kotlin.test.assertTrue /** * Make a fake-factory for a one single plugin. Useful for unique or test plugins */ -public inline fun P.toFactory(): PluginFactory

= object : PluginFactory

{ +public fun

P.toFactory(): PluginFactory

= object : PluginFactory

{ override fun build(context: Context, meta: Meta): P = this@toFactory override val tag: PluginTag = this@toFactory.tag - override val type: KClass = P::class } public fun Workspace.produceBlocking(task: String, block: MutableMeta.() -> Unit = {}): DataSet = runBlocking { @@ -39,7 +37,7 @@ internal object TestPlugin : WorkspacePlugin() { val test by task { // type is inferred - pipeFrom(data()) { arg, _, _ -> + pipeFrom(dataByType()) { arg, _, _ -> logger.info { "Test: $arg" } arg } @@ -76,7 +74,7 @@ internal class SimpleWorkspaceTest { } val square by task { - pipeFrom(data()) { arg, name, meta -> + pipeFrom(dataByType()) { arg, name, meta -> if (meta["testFlag"].boolean == true) { println("Side effect") } @@ -86,7 +84,7 @@ internal class SimpleWorkspaceTest { } val linear by task { - pipeFrom(data()) { arg, name, _ -> + pipeFrom(dataByType()) { arg, name, _ -> workspace.logger.info { "Starting linear on $name" } arg * 2 + 1 } diff --git a/gradle.properties b/gradle.properties index cf8696b2..8792120f 100644 --- a/gradle.properties +++ b/gradle.properties @@ -6,4 +6,4 @@ kotlin.mpp.stability.nowarn=true kotlin.incremental.js.ir=true kotlin.native.ignoreDisabledTargets=true -toolsVersion=0.14.3-kotlin-1.8.20-RC +toolsVersion=0.14.4-kotlin-1.8.20-RC diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 070cb702..e1bef7e8 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.6-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.0.2-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/settings.gradle.kts b/settings.gradle.kts index a183e7ac..e9fb1b81 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -1,7 +1,7 @@ rootProject.name = "dataforge-core" enableFeaturePreview("TYPESAFE_PROJECT_ACCESSORS") -enableFeaturePreview("VERSION_CATALOGS") +//enableFeaturePreview("VERSION_CATALOGS") pluginManagement {