diff --git a/build.gradle.kts b/build.gradle.kts index 016a350e..04cce594 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -1,9 +1,10 @@ plugins { - id("scientifik.mpp") version "0.1.4" apply false - id("scientifik.publish") version "0.1.4" apply false + id("scientifik.mpp") version "0.2.1" apply false + id("scientifik.jvm") version "0.2.1" apply false + id("scientifik.publish") version "0.2.1" apply false } -val dataforgeVersion by extra("0.1.3") +val dataforgeVersion by extra("0.1.4") val bintrayRepo by extra("dataforge") val githubProject by extra("dataforge-core") diff --git a/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/AbstractPlugin.kt b/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/AbstractPlugin.kt index c9268790..232c7cd4 100644 --- a/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/AbstractPlugin.kt +++ b/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/AbstractPlugin.kt @@ -3,10 +3,13 @@ package hep.dataforge.context import hep.dataforge.meta.EmptyMeta import hep.dataforge.meta.Meta import hep.dataforge.names.Name -import hep.dataforge.names.toName +import kotlin.properties.ReadOnlyProperty +import kotlin.reflect.KClass +import kotlin.reflect.KProperty abstract class AbstractPlugin(override val meta: Meta = EmptyMeta) : Plugin { private var _context: Context? = null + private val dependencies = ArrayList>() override val context: Context get() = _context ?: error("Plugin $tag is not attached") @@ -19,9 +22,23 @@ abstract class AbstractPlugin(override val meta: Meta = EmptyMeta) : Plugin { this._context = null } - override fun provideTop(target: String): Map = emptyMap() + final override fun dependsOn(): List> = dependencies - companion object{ - fun Collection.toMap(): Map = associate { it.name.toName() to it } + /** + * Register plugin dependency and return a delegate which provides lazily initialized reference to dependent plugin + */ + protected fun

require(factory: PluginFactory

): ReadOnlyProperty { + dependencies.add(factory) + return PluginDependencyDelegate(factory.type) + } + + override fun provideTop(target: String): Map = emptyMap() +} + +fun Collection.toMap(): Map = associate { it.name to it } + +private class PluginDependencyDelegate

(val type: KClass) : ReadOnlyProperty { + override fun getValue(thisRef: AbstractPlugin, property: KProperty<*>): P { + return thisRef.context.plugins[type] ?: error("Plugin with type $type not found") } } \ No newline at end of file diff --git a/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/Context.kt b/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/Context.kt index 80746456..ffc5b197 100644 --- a/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/Context.kt +++ b/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/Context.kt @@ -2,8 +2,8 @@ package hep.dataforge.context import hep.dataforge.meta.* import hep.dataforge.names.Name -import hep.dataforge.names.appendLeft -import hep.dataforge.names.toName +import hep.dataforge.names.asName +import hep.dataforge.names.plus import hep.dataforge.provider.Provider import hep.dataforge.provider.top import hep.dataforge.values.Value @@ -27,7 +27,7 @@ import kotlin.jvm.JvmName * @author Alexander Nozik */ open class Context( - final override val name: String, + final override val name: Name, val parent: Context? = Global ) : Named, MetaRepr, Provider, CoroutineScope { @@ -45,7 +45,7 @@ open class Context( /** * Context logger */ - val logger: KLogger = KotlinLogging.logger(name) + val logger: KLogger = KotlinLogging.logger(name.toString()) /** * A [PluginManager] for current context @@ -64,7 +64,7 @@ open class Context( override fun provideTop(target: String): Map { return when (target) { Value.TYPE -> properties.sequence().toMap() - Plugin.PLUGIN_TARGET -> plugins.sequence(true).associateBy { it.name.toName() } + Plugin.PLUGIN_TARGET -> plugins.sequence(true).associateBy { it.name } else -> emptyMap() } } @@ -105,8 +105,8 @@ open class Context( override fun toMeta(): Meta = buildMeta { "parent" to parent?.name - "properties" to properties.seal() - "plugins" to plugins.map { it.toMeta() } + "properties" put properties.seal() + "plugins" put plugins.map { it.toMeta() } } } @@ -118,14 +118,14 @@ fun Context.content(target: String): Map = content(target) @JvmName("typedContent") inline fun Context.content(target: String): Map = plugins.flatMap { plugin -> - plugin.top(target).entries.map { (it.key.appendLeft(plugin.name)) to it.value } + plugin.top(target).entries.map { (plugin.name + it.key) to it.value } }.associate { it } /** * A global root context. Closing [Global] terminates the framework. */ -object Global : Context("GLOBAL", null) { +object Global : Context("GLOBAL".asName(), null) { /** * Closing all contexts * @@ -173,7 +173,7 @@ interface ContextAware { val logger: KLogger get() = if (this is Named) { - KotlinLogging.logger(context.name + "." + (this as Named).name) + KotlinLogging.logger((context.name + this.name).toString()) } else { context.logger } diff --git a/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/ContextBuilder.kt b/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/ContextBuilder.kt index 92840862..58a03554 100644 --- a/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/ContextBuilder.kt +++ b/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/ContextBuilder.kt @@ -2,11 +2,12 @@ package hep.dataforge.context import hep.dataforge.meta.MetaBuilder import hep.dataforge.meta.buildMeta +import hep.dataforge.names.toName /** * A convenience builder for context */ -class ContextBuilder(var name: String = "@anonimous", val parent: Context = Global) { +class ContextBuilder(var name: String = "@anonymous", val parent: Context = Global) { private val plugins = ArrayList() private var meta = MetaBuilder() @@ -31,7 +32,7 @@ class ContextBuilder(var name: String = "@anonimous", val parent: Context = Glob } fun build(): Context { - return Context(name, parent).apply { + return Context(name.toName(), parent).apply { this@ContextBuilder.plugins.forEach { plugins.load(it) } diff --git a/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/Factory.kt b/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/Factory.kt new file mode 100644 index 00000000..b20672c2 --- /dev/null +++ b/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/Factory.kt @@ -0,0 +1,8 @@ +package hep.dataforge.context + +import hep.dataforge.meta.EmptyMeta +import hep.dataforge.meta.Meta + +interface Factory { + operator fun invoke(meta: Meta = EmptyMeta, context: Context = Global): T +} \ No newline at end of file diff --git a/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/Named.kt b/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/Named.kt index 1ac31702..9e9db17f 100644 --- a/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/Named.kt +++ b/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/Named.kt @@ -15,6 +15,10 @@ */ package hep.dataforge.context +import hep.dataforge.names.Name +import hep.dataforge.names.asName +import hep.dataforge.names.isEmpty + /** * Any object that have name * @@ -27,10 +31,9 @@ interface Named { * * @return */ - val name: String + val name: Name companion object { - const val ANONYMOUS = "" /** * Get the name of given object. If object is Named its name is used, @@ -39,11 +42,11 @@ interface Named { * @param obj * @return */ - fun nameOf(obj: Any): String { + fun nameOf(obj: Any): Name { return if (obj is Named) { obj.name } else { - obj.toString() + obj.toString().asName() } } } @@ -54,4 +57,4 @@ interface Named { * @return */ val Named.isAnonymous: Boolean - get() = this.name == Named.ANONYMOUS + get() = this.name.isEmpty() diff --git a/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/Plugin.kt b/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/Plugin.kt index e06bf3f9..90d669e0 100644 --- a/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/Plugin.kt +++ b/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/Plugin.kt @@ -3,6 +3,8 @@ package hep.dataforge.context import hep.dataforge.meta.Meta import hep.dataforge.meta.MetaRepr import hep.dataforge.meta.buildMeta +import hep.dataforge.names.Name +import hep.dataforge.names.toName import hep.dataforge.provider.Provider /** @@ -37,7 +39,7 @@ interface Plugin : Named, ContextAware, Provider, MetaRepr { * * @return */ - override val name: String get() = tag.name + override val name: Name get() = tag.name.toName() /** * Plugin dependencies which are required to attach this plugin. Plugin @@ -46,7 +48,7 @@ interface Plugin : Named, ContextAware, Provider, MetaRepr { * * @return */ - fun dependsOn(): List> = emptyList() + fun dependsOn(): Collection> /** * Start this plugin and attach registration info to the context. This method @@ -64,10 +66,10 @@ interface Plugin : Named, ContextAware, Provider, MetaRepr { fun detach() override fun toMeta(): Meta = buildMeta { - "context" to context.name + "context" put context.name.toString() "type" to this::class.simpleName - "tag" to tag - "meta" to meta + "tag" put tag + "meta" put meta } companion object { diff --git a/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/PluginManager.kt b/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/PluginManager.kt index dad483a8..dd114f79 100644 --- a/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/PluginManager.kt +++ b/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/PluginManager.kt @@ -38,7 +38,7 @@ class PluginManager(override val context: Context) : ContextAware, Iterable Boolean): Plugin? = sequence(recursive).find(predicate) + fun find(recursive: Boolean = true, predicate: (Plugin) -> Boolean): Plugin? = sequence(recursive).find(predicate) /** @@ -47,7 +47,7 @@ class PluginManager(override val context: Context) : ContextAware, Iterable get(type: KClass, tag: PluginTag? = null, recursive: Boolean = true): T? = - get(recursive) { type.isInstance(it) && (tag == null || tag.matches(it.tag)) } as T? + find(recursive) { type.isInstance(it) && (tag == null || tag.matches(it.tag)) } as T? - inline fun get(tag: PluginTag? = null, recursive: Boolean = true): T? = + inline operator fun get(tag: PluginTag? = null, recursive: Boolean = true): T? = get(T::class, tag, recursive) + inline operator fun get(factory: PluginFactory, recursive: Boolean = true): T? = + get(factory.type, factory.tag, recursive) /** * Load given plugin into this manager and return loaded instance. @@ -97,7 +99,7 @@ class PluginManager(override val context: Context) : ContextAware, Iterable load(factory: PluginFactory, meta: Meta = EmptyMeta): T = - load(factory(meta)) + load(factory(meta, context)) fun load(factory: PluginFactory, metaBuilder: MetaBuilder.() -> Unit): T = load(factory, buildMeta(metaBuilder)) @@ -122,7 +124,7 @@ class PluginManager(override val context: Context) : ContextAware, Iterable fetch(factory: PluginFactory, recursive: Boolean = true, meta: Meta = EmptyMeta): T { val loaded = get(factory.type, factory.tag, recursive) return when { - loaded == null -> load(factory(meta)) + loaded == null -> load(factory(meta, context)) loaded.meta == meta -> loaded // if meta is the same, return existing plugin else -> throw RuntimeException("Can't load plugin with tag ${factory.tag}. Plugin with this tag and different configuration already exists in context.") } diff --git a/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/PluginRepository.kt b/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/PluginRepository.kt index eb715f80..06c1132b 100644 --- a/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/PluginRepository.kt +++ b/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/PluginRepository.kt @@ -4,10 +4,9 @@ import hep.dataforge.meta.EmptyMeta import hep.dataforge.meta.Meta import kotlin.reflect.KClass -interface PluginFactory { +interface PluginFactory : Factory { val tag: PluginTag val type: KClass - operator fun invoke(meta: Meta = EmptyMeta): T } expect object PluginRepository { @@ -25,25 +24,26 @@ expect object PluginRepository { * Fetch specific plugin and instantiate it with given meta */ fun PluginRepository.fetch(tag: PluginTag, meta: Meta = EmptyMeta): Plugin = - list().find { it.tag.matches(tag) }?.invoke(meta) ?: error("Plugin with tag $tag not found in the repository") + list().find { it.tag.matches(tag) }?.invoke(meta = meta) + ?: error("Plugin with tag $tag not found in the repository") fun PluginRepository.register( tag: PluginTag, type: KClass, - constructor: (Meta) -> T + constructor: (Context, Meta) -> T ): PluginFactory { val factory = object : PluginFactory { override val tag: PluginTag = tag override val type: KClass = type - override fun invoke(meta: Meta): T = constructor(meta) + override fun invoke(meta: Meta, context: Context): T = constructor(context, meta) } register(factory) return factory } -inline fun PluginRepository.register(tag: PluginTag, noinline constructor: (Meta) -> T) = +inline fun PluginRepository.register(tag: PluginTag, noinline constructor: (Context, Meta) -> T) = register(tag, T::class, constructor) -fun PluginRepository.register(plugin: Plugin) = register(plugin.tag, plugin::class) { plugin } \ No newline at end of file +fun PluginRepository.register(plugin: Plugin) = register(plugin.tag, plugin::class) { _, _ -> plugin } \ No newline at end of file diff --git a/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/PluginTag.kt b/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/PluginTag.kt index 25164c72..390de7bc 100644 --- a/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/PluginTag.kt +++ b/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/PluginTag.kt @@ -37,9 +37,9 @@ data class PluginTag( override fun toString(): String = listOf(group, name, version).joinToString(separator = ":") override fun toMeta(): Meta = buildMeta { - "name" to name - "group" to group - "version" to version + "name" put name + "group" put group + "version" put version } companion object { diff --git a/dataforge-context/src/commonTest/kotlin/hep/dataforge/context/ContextTest.kt b/dataforge-context/src/commonTest/kotlin/hep/dataforge/context/ContextTest.kt index c77439d6..584e500c 100644 --- a/dataforge-context/src/commonTest/kotlin/hep/dataforge/context/ContextTest.kt +++ b/dataforge-context/src/commonTest/kotlin/hep/dataforge/context/ContextTest.kt @@ -5,7 +5,6 @@ import hep.dataforge.names.appendLeft import hep.dataforge.names.toName import kotlin.test.Test import kotlin.test.assertEquals -import kotlin.test.assertTrue class ContextTest { @@ -26,7 +25,7 @@ class ContextTest { val members = Global.content("test") assertEquals(3, members.count()) members.forEach { - assertTrue{it.key == it.value.appendLeft("test")} + assertEquals(it.key, it.value.appendLeft("test")) } } diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/Data.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/Data.kt index 9b0d9027..718fb46f 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/Data.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/Data.kt @@ -1,8 +1,6 @@ package hep.dataforge.data -import hep.dataforge.meta.EmptyMeta -import hep.dataforge.meta.Meta -import hep.dataforge.meta.MetaRepr +import hep.dataforge.meta.* import kotlinx.coroutines.CoroutineScope import kotlin.coroutines.CoroutineContext import kotlin.coroutines.EmptyCoroutineContext @@ -11,7 +9,7 @@ import kotlin.reflect.KClass /** * A data element characterized by its meta */ -interface Data : Goal, MetaRepr { +interface Data : Goal, MetaRepr{ /** * Type marker for the data. The type is known before the calculation takes place so it could be checked. */ @@ -21,7 +19,12 @@ interface Data : Goal, MetaRepr { */ val meta: Meta - override fun toMeta(): Meta = meta + override fun toMeta(): Meta = buildMeta { + "type" put (type.simpleName?:"undefined") + if(!meta.isEmpty()) { + "meta" put meta + } + } companion object { const val TYPE = "data" @@ -34,7 +37,7 @@ interface Data : Goal, MetaRepr { block: suspend CoroutineScope.() -> T ): Data = DynamicData(type, meta, context, dependencies, block) - operator inline fun invoke( + inline operator fun invoke( meta: Meta = EmptyMeta, context: CoroutineContext = EmptyCoroutineContext, dependencies: Collection> = emptyList(), @@ -50,7 +53,7 @@ interface Data : Goal, MetaRepr { block: suspend CoroutineScope.() -> T ): Data = NamedData(name, invoke(type, meta, context, dependencies, block)) - operator inline fun invoke( + inline operator fun invoke( name: String, meta: Meta = EmptyMeta, context: CoroutineContext = EmptyCoroutineContext, @@ -65,18 +68,6 @@ interface Data : Goal, MetaRepr { } -fun Data.cast(type: KClass): Data { - return object : Data by this { - override val type: KClass = type - } -} - -/** - * Upcast a [Data] to a supertype - */ -inline fun Data.cast(): Data = cast(R::class) - - class DynamicData( override val type: KClass, override val meta: Meta = EmptyMeta, @@ -94,7 +85,7 @@ class StaticData( class NamedData(val name: String, data: Data) : Data by data -fun Data.pipe( +fun Data.map( outputType: KClass, coroutineContext: CoroutineContext = EmptyCoroutineContext, meta: Meta = this.meta, @@ -107,7 +98,7 @@ fun Data.pipe( /** * Create a data pipe */ -inline fun Data.pipe( +inline fun Data.map( coroutineContext: CoroutineContext = EmptyCoroutineContext, meta: Meta = this.meta, noinline block: suspend CoroutineScope.(T) -> R @@ -118,7 +109,7 @@ inline fun Data.pipe( /** * Create a joined data. */ -inline fun Collection>.join( +inline fun Collection>.reduce( coroutineContext: CoroutineContext = EmptyCoroutineContext, meta: Meta, noinline block: suspend CoroutineScope.(Collection) -> R @@ -128,10 +119,10 @@ inline fun Collection>.join( coroutineContext, this ) { - block(map { this.run { it.await(this) } }) + block(map { run { it.await(this) } }) } -fun Map>.join( +fun Map>.reduce( outputType: KClass, coroutineContext: CoroutineContext = EmptyCoroutineContext, meta: Meta, @@ -152,7 +143,7 @@ fun Map>.join( * @param T type of the input goal * @param R type of the result goal */ -inline fun Map>.join( +inline fun Map>.reduce( coroutineContext: CoroutineContext = EmptyCoroutineContext, meta: Meta, noinline block: suspend CoroutineScope.(Map) -> R diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataFilter.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataFilter.kt index a23b550d..a55aac9d 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataFilter.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataFilter.kt @@ -5,12 +5,23 @@ import hep.dataforge.names.toName class DataFilter(override val config: Config) : Specific { + /** + * A source node for the filter + */ var from by string() + /** + * A target placement for the filtered node + */ var to by string() - var pattern by string("*.") + /** + * A regular expression pattern for the filter + */ + var pattern by string(".*") // val prefix by string() // val suffix by string() + fun isEmpty(): Boolean = config.isEmpty() + companion object : Specification { override fun wrap(config: Config): DataFilter = DataFilter(config) } diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataNode.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataNode.kt index a407b512..12bb06ab 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataNode.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataNode.kt @@ -1,5 +1,6 @@ package hep.dataforge.data +import hep.dataforge.meta.* import hep.dataforge.names.* import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Job @@ -9,22 +10,26 @@ import kotlin.collections.component2 import kotlin.collections.set import kotlin.reflect.KClass -sealed class DataItem { +sealed class DataItem : MetaRepr { abstract val type: KClass class Node(val value: DataNode) : DataItem() { override val type: KClass get() = value.type + + override fun toMeta(): Meta = value.toMeta() } class Leaf(val value: Data) : DataItem() { override val type: KClass get() = value.type + + override fun toMeta(): Meta = value.toMeta() } } /** * A tree-like data structure grouped into the node. All data inside the node must inherit its type */ -interface DataNode { +interface DataNode : MetaRepr { /** * The minimal common ancestor to all data in the node @@ -33,12 +38,24 @@ interface DataNode { val items: Map> + override fun toMeta(): Meta = buildMeta { + "type" put (type.simpleName ?: "undefined") + "items" put { + this@DataNode.items.forEach { + it.key.toString() put it.value.toMeta() + } + } + } + companion object { const val TYPE = "dataNode" - fun build(type: KClass, block: DataTreeBuilder.() -> Unit) = + operator fun invoke(type: KClass, block: DataTreeBuilder.() -> Unit) = DataTreeBuilder(type).apply(block).build() + inline operator fun invoke(noinline block: DataTreeBuilder.() -> Unit) = + DataTreeBuilder(T::class).apply(block).build() + fun builder(type: KClass) = DataTreeBuilder(type) } } @@ -68,10 +85,12 @@ fun DataNode<*>.joinAll(scope: CoroutineScope): Job = scope.launch { operator fun DataNode.get(name: Name): DataItem? = when (name.length) { 0 -> error("Empty name") - 1 -> (items[name.first()] as? DataItem.Leaf) + 1 -> items[name.first()] else -> get(name.first()!!.asName()).node?.get(name.cutFirst()) } +operator fun DataNode.get(name: String): DataItem? = get(name.toName()) + /** * Sequence of all children including nodes */ @@ -108,7 +127,9 @@ class DataTree internal constructor( override val type: KClass, override val items: Map> ) : DataNode { - //TODO add node-level meta? + override fun toString(): String { + return super.toString() + } } private sealed class DataTreeBuilderItem { @@ -119,10 +140,11 @@ private sealed class DataTreeBuilderItem { /** * A builder for a DataTree. */ -class DataTreeBuilder(private val type: KClass) { +@DFBuilder +class DataTreeBuilder(val type: KClass) { private val map = HashMap>() - operator fun set(token: NameToken, node: DataTreeBuilder) { + operator fun set(token: NameToken, node: DataTreeBuilder) { if (map.containsKey(token)) error("Tree entry with name $token is not empty") map[token] = DataTreeBuilderItem.Node(node) } @@ -134,9 +156,9 @@ class DataTreeBuilder(private val type: KClass) { private fun buildNode(token: NameToken): DataTreeBuilder { return if (!map.containsKey(token)) { - DataTreeBuilder(type).also { map[token] = DataTreeBuilderItem.Node(it) } + DataTreeBuilder(type).also { map[token] = DataTreeBuilderItem.Node(it) } } else { - (map[token] as? DataTreeBuilderItem.Node ?: error("The node with name $token is occupied by leaf")).tree + (map[token] as? DataTreeBuilderItem.Node ?: error("The node with name $token is occupied by leaf")).tree } } @@ -156,7 +178,7 @@ class DataTreeBuilder(private val type: KClass) { } } - operator fun set(name: Name, node: DataTreeBuilder) { + operator fun set(name: Name, node: DataTreeBuilder) { when (name.length) { 0 -> error("Can't add data with empty name") 1 -> set(name.first()!!, node) @@ -174,19 +196,19 @@ class DataTreeBuilder(private val type: KClass) { /** * Append data to node */ - infix fun String.to(data: Data) = set(toName(), data) + infix fun String.put(data: Data) = set(toName(), data) /** * Append node */ - infix fun String.to(node: DataNode) = set(toName(), node) + infix fun String.put(node: DataNode) = set(toName(), node) - infix fun String.to(item: DataItem) = set(toName(), item) + infix fun String.put(item: DataItem) = set(toName(), item) /** * Build and append node */ - infix fun String.to(block: DataTreeBuilder.() -> Unit) = set(toName(), DataTreeBuilder(type).apply(block)) + infix fun String.put(block: DataTreeBuilder.() -> Unit) = set(toName(), DataTreeBuilder(type).apply(block)) fun update(node: DataNode) { @@ -207,6 +229,42 @@ class DataTreeBuilder(private val type: KClass) { } } +fun DataTreeBuilder.datum(name: Name, data: Data) { + this[name] = data +} + +fun DataTreeBuilder.datum(name: String, data: Data) { + this[name.toName()] = data +} + +fun DataTreeBuilder.static(name: Name, data: T, meta: Meta = EmptyMeta) { + this[name] = Data.static(data, meta) +} + +fun DataTreeBuilder.static(name: Name, data: T, block: MetaBuilder.() -> Unit = {}) { + this[name] = Data.static(data, buildMeta(block)) +} + +fun DataTreeBuilder.static(name: String, data: T, block: MetaBuilder.() -> Unit = {}) { + this[name.toName()] = Data.static(data, buildMeta(block)) +} + +fun DataTreeBuilder.node(name: Name, node: DataNode) { + this[name] = node +} + +fun DataTreeBuilder.node(name: String, node: DataNode) { + this[name.toName()] = node +} + +inline fun DataTreeBuilder.node(name: Name, noinline block: DataTreeBuilder.() -> Unit) { + this[name] = DataNode(T::class, block) +} + +inline fun DataTreeBuilder.node(name: String, noinline block: DataTreeBuilder.() -> Unit) { + this[name.toName()] = DataNode(T::class, block) +} + /** * Generate a mutable builder from this node. Node content is not changed */ @@ -214,7 +272,7 @@ fun DataNode.builder(): DataTreeBuilder = DataTreeBuilder(type). dataSequence().forEach { (name, data) -> this[name] = data } } -fun DataNode.filter(predicate: (Name, Data) -> Boolean): DataNode = DataNode.build(type) { +fun DataNode.filter(predicate: (Name, Data) -> Boolean): DataNode = DataNode.invoke(type) { dataSequence().forEach { (name, data) -> if (predicate(name, data)) { this[name] = data @@ -222,9 +280,4 @@ fun DataNode.filter(predicate: (Name, Data) -> Boolean): DataNod } } -fun DataNode.first(): Data? = dataSequence().first().second - -/** - * Check that node is compatible with given type meaning that each element could be cast to the type - */ -expect fun DataNode<*>.checkType(type: KClass<*>) \ No newline at end of file +fun DataNode.first(): Data? = dataSequence().first().second \ No newline at end of file diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/Goal.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/Goal.kt index 54bb743e..8275d31e 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/Goal.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/Goal.kt @@ -85,7 +85,7 @@ open class DynamicGoal( /** * Create a one-to-one goal based on existing goal */ -fun Goal.pipe( +fun Goal.map( coroutineContext: CoroutineContext = EmptyCoroutineContext, block: suspend CoroutineScope.(T) -> R ): Goal = DynamicGoal(coroutineContext, listOf(this)) { @@ -95,11 +95,11 @@ fun Goal.pipe( /** * Create a joining goal. */ -fun Collection>.join( +fun Collection>.reduce( coroutineContext: CoroutineContext = EmptyCoroutineContext, block: suspend CoroutineScope.(Collection) -> R ): Goal = DynamicGoal(coroutineContext, this) { - block(map { this.run { it.await(this) } }) + block(map { run { it.await(this) } }) } /** @@ -108,7 +108,7 @@ fun Collection>.join( * @param T type of the input goal * @param R type of the result goal */ -fun Map>.join( +fun Map>.reduce( coroutineContext: CoroutineContext = EmptyCoroutineContext, block: suspend CoroutineScope.(Map) -> R ): Goal = DynamicGoal(coroutineContext, this.values) { diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/MapAction.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/MapAction.kt new file mode 100644 index 00000000..8c543927 --- /dev/null +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/MapAction.kt @@ -0,0 +1,78 @@ +package hep.dataforge.data + +import hep.dataforge.meta.Meta +import hep.dataforge.meta.MetaBuilder +import hep.dataforge.meta.builder +import hep.dataforge.meta.seal +import hep.dataforge.names.Name +import kotlin.reflect.KClass + +/** + * Action environment includes data name, data meta and action configuration meta + */ +data class ActionEnv( + val name: Name, + val meta: Meta, + val actionMeta: Meta +) + + +/** + * Action environment + */ +class MapActionBuilder(var name: Name, var meta: MetaBuilder, val actionMeta: Meta) { + lateinit var result: suspend ActionEnv.(T) -> R + + /** + * Calculate the result of goal + */ + fun result(f: suspend ActionEnv.(T) -> R) { + result = f; + } +} + + +class MapAction( + val inputType: KClass, + val outputType: KClass, + private val block: MapActionBuilder.() -> Unit +) : Action { + + override fun invoke(node: DataNode, meta: Meta): DataNode { + node.ensureType(inputType) + + return DataNode.invoke(outputType) { + node.dataSequence().forEach { (name, data) -> + /* + * Creating a new environment for action using **old** name, old meta and task meta + */ + val env = ActionEnv(name, data.meta, meta) + + //applying transformation from builder + val builder = MapActionBuilder( + name, + data.meta.builder(), // using data meta + meta + ).apply(block) + + //getting new name + val newName = builder.name + + //getting new meta + val newMeta = builder.meta.seal() + + val newData = data.map(outputType, meta = newMeta) { builder.result(env, it) } + //setting the data node + this[newName] = newData + } + } + } +} + +inline fun DataNode.map( + meta: Meta, + noinline action: MapActionBuilder.() -> Unit +): DataNode = MapAction(T::class, R::class, action).invoke(this, meta) + + + diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/PipeAction.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/PipeAction.kt deleted file mode 100644 index c84e5a13..00000000 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/PipeAction.kt +++ /dev/null @@ -1,60 +0,0 @@ -package hep.dataforge.data - -import hep.dataforge.meta.* -import hep.dataforge.names.Name -import kotlin.reflect.KClass - -class ActionEnv(val name: Name, val meta: Meta) - - -/** - * Action environment - */ -class PipeBuilder(var name: Name, var meta: MetaBuilder) { - lateinit var result: suspend ActionEnv.(T) -> R - - /** - * Calculate the result of goal - */ - fun result(f: suspend ActionEnv.(T) -> R) { - result = f; - } -} - - -class PipeAction( - val inputType: KClass, - val outputType: KClass, - private val block: PipeBuilder.() -> Unit -) : Action { - - override fun invoke(node: DataNode, meta: Meta): DataNode { - node.checkType(inputType) - - return DataNode.build(outputType) { - node.dataSequence().forEach { (name, data) -> - //merging data meta with action meta (data meta is primary) - val oldMeta = meta.builder().apply { update(data.meta) } - // creating environment from old meta and name - val env = ActionEnv(name, oldMeta) - //applying transformation from builder - val builder = PipeBuilder(name, oldMeta).apply(block) - //getting new name - val newName = builder.name - //getting new meta - val newMeta = builder.meta.seal() - val newData = data.pipe(outputType, meta = newMeta) { builder.result(env, it) } - //setting the data node - this[newName] = newData - } - } - } -} - -inline fun DataNode.pipe( - meta: Meta, - noinline action: PipeBuilder.() -> Unit -): DataNode = PipeAction(T::class, R::class, action).invoke(this, meta) - - - diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/JoinAction.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/ReduceAction.kt similarity index 73% rename from dataforge-data/src/commonMain/kotlin/hep/dataforge/data/JoinAction.kt rename to dataforge-data/src/commonMain/kotlin/hep/dataforge/data/ReduceAction.kt index 4acae87f..9bb49151 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/JoinAction.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/ReduceAction.kt @@ -1,9 +1,7 @@ package hep.dataforge.data -import hep.dataforge.meta.Laminate import hep.dataforge.meta.Meta import hep.dataforge.meta.MetaBuilder -import hep.dataforge.meta.builder import hep.dataforge.names.Name import hep.dataforge.names.toName import kotlin.reflect.KClass @@ -21,7 +19,7 @@ class JoinGroup(var name: String, internal val node: DataNode< } -class JoinGroupBuilder(val actionMeta: Meta) { +class ReduceGroupBuilder(val actionMeta: Meta) { private val groupRules: MutableList<(DataNode) -> List>> = ArrayList(); /** @@ -73,26 +71,31 @@ class JoinGroupBuilder(val actionMeta: Meta) { /** * The same rules as for KPipe */ -class JoinAction( - val inputType: KClass, - val outputType: KClass, - private val action: JoinGroupBuilder.() -> Unit +class ReduceAction( + val inputType: KClass, + val outputType: KClass, + private val action: ReduceGroupBuilder.() -> Unit ) : Action { override fun invoke(node: DataNode, meta: Meta): DataNode { - node.checkType(inputType) - return DataNode.build(outputType) { - JoinGroupBuilder(meta).apply(action).buildGroups(node).forEach { group -> + node.ensureType(inputType) + return DataNode.invoke(outputType) { + ReduceGroupBuilder(meta).apply(action).buildGroups(node).forEach { group -> - val laminate = Laminate(group.meta, meta) + //val laminate = Laminate(group.meta, meta) val dataMap = group.node.dataSequence().associate { it } - val groupName: String = group.name; + val groupName: String = group.name - val env = ActionEnv(groupName.toName(), laminate.builder()) + val groupMeta = group.meta - val res: DynamicData = dataMap.join(outputType, meta = laminate) { group.result.invoke(env, it) } + val env = ActionEnv(groupName.toName(), groupMeta, meta) + + val res: DynamicData = dataMap.reduce( + outputType, + meta = groupMeta + ) { group.result.invoke(env, it) } set(env.name, res) } diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/SplitAction.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/SplitAction.kt index be9764a6..a4f82931 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/SplitAction.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/SplitAction.kt @@ -33,15 +33,15 @@ class SplitBuilder(val name: Name, val meta: Meta) { } class SplitAction( - val inputType: KClass, - val outputType: KClass, + val inputType: KClass, + val outputType: KClass, private val action: SplitBuilder.() -> Unit ) : Action { override fun invoke(node: DataNode, meta: Meta): DataNode { - node.checkType(inputType) + node.ensureType(inputType) - return DataNode.build(outputType) { + return DataNode.invoke(outputType) { node.dataSequence().forEach { (name, data) -> val laminate = Laminate(data.meta, meta) @@ -55,7 +55,7 @@ class SplitAction( rule(env) - val res = data.pipe(outputType, meta = env.meta) { env.result(it) } + val res = data.map(outputType, meta = env.meta) { env.result(it) } set(env.name, res) } } diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/dataCast.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/dataCast.kt new file mode 100644 index 00000000..556b77fc --- /dev/null +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/dataCast.kt @@ -0,0 +1,72 @@ +package hep.dataforge.data + +import hep.dataforge.meta.Meta +import hep.dataforge.names.NameToken +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Deferred +import kotlin.reflect.KClass + +fun Data.upcast(type: KClass): Data { + return object : Data by this { + override val type: KClass = type + } +} + +/** + * Safe upcast a [Data] to a supertype + */ +inline fun Data.upcast(): Data = upcast(R::class) + +/** + * Check if node could be safely cast to given class + */ +expect fun DataNode<*>.canCast(type: KClass): Boolean + +/** + * Check if data could be safely cast to given class + */ +expect fun Data<*>.canCast(type: KClass): Boolean + +fun DataItem<*>.canCast(type: KClass): Boolean = when (this) { + is DataItem.Node -> value.canCast(type) + is DataItem.Leaf -> value.canCast(type) +} + +/** + * Unsafe cast of data node + */ +@Suppress("UNCHECKED_CAST") +fun Data<*>.cast(type: KClass): Data { + return object : Data { + override val meta: Meta get() = this@cast.meta + override val dependencies: Collection> get() = this@cast.dependencies + override val result: Deferred? get() = this@cast.result as Deferred + override fun startAsync(scope: CoroutineScope): Deferred = this@cast.startAsync(scope) as Deferred + override fun reset() = this@cast.reset() + override val type: KClass = type + } +} + +inline fun Data<*>.cast(): Data = cast(R::class) + +@Suppress("UNCHECKED_CAST") +fun DataNode<*>.cast(type: KClass): DataNode { + return object : DataNode { + override val type: KClass = type + override val items: Map> get() = this@cast.items as Map> + } +} + +inline fun DataNode<*>.cast(): DataNode = cast(R::class) + +/** + * Check that node is compatible with given type meaning that each element could be cast to the type + */ +fun DataNode<*>.ensureType(type: KClass) { + if (!canCast(type)) { + error("$type expected, but $type received") + } +} + + +//expect fun DataNode.cast(type: KClass): DataNode \ No newline at end of file diff --git a/dataforge-data/src/commonTest/kotlin/hep/dataforge/data/DataTreeBuilderTest.kt b/dataforge-data/src/commonTest/kotlin/hep/dataforge/data/DataTreeBuilderTest.kt new file mode 100644 index 00000000..32bf1760 --- /dev/null +++ b/dataforge-data/src/commonTest/kotlin/hep/dataforge/data/DataTreeBuilderTest.kt @@ -0,0 +1,32 @@ +package hep.dataforge.data + +import kotlin.test.Test +import kotlin.test.assertTrue + + +internal class DataTreeBuilderTest{ + @Test + fun testDataUpdate(){ + val updateData = DataNode{ + "update" put { + "a" put Data.static("a") + "b" put Data.static("b") + } + } + + val node = DataNode{ + node("primary"){ + static("a","a") + static("b","b") + } + static("root","root") + update(updateData) + } + + println(node.toMeta()) + + assertTrue { node["update.a"] != null } + assertTrue { node["primary.a"] != null } + + } +} \ No newline at end of file diff --git a/dataforge-data/src/jsMain/kotlin/hep/dataforge/data/checkType.kt b/dataforge-data/src/jsMain/kotlin/hep/dataforge/data/checkType.kt deleted file mode 100644 index 56d1d0c9..00000000 --- a/dataforge-data/src/jsMain/kotlin/hep/dataforge/data/checkType.kt +++ /dev/null @@ -1,10 +0,0 @@ -package hep.dataforge.data - -import kotlin.reflect.KClass - -/** - * Check that node is compatible with given type meaning that each element could be cast to the type - */ -actual fun DataNode<*>.checkType(type: KClass<*>) { - //Not supported in js yet -} \ No newline at end of file diff --git a/dataforge-data/src/jsMain/kotlin/hep/dataforge/data/dataJS.kt b/dataforge-data/src/jsMain/kotlin/hep/dataforge/data/dataJS.kt new file mode 100644 index 00000000..54148bc5 --- /dev/null +++ b/dataforge-data/src/jsMain/kotlin/hep/dataforge/data/dataJS.kt @@ -0,0 +1,16 @@ +package hep.dataforge.data + +import kotlin.reflect.KClass + +/** + * Check that node is compatible with given type meaning that each element could be cast to the type + */ +actual fun DataNode<*>.canCast(type: KClass): Boolean { + //Not supported in js yet + return true +} + +actual fun Data<*>.canCast(type: KClass): Boolean { + //Not supported in js yet + return true +} diff --git a/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/CastDataNode.kt b/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/CastDataNode.kt deleted file mode 100644 index 324864fc..00000000 --- a/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/CastDataNode.kt +++ /dev/null @@ -1,55 +0,0 @@ -package hep.dataforge.data - -import hep.dataforge.meta.Meta -import hep.dataforge.names.NameToken -import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.Deferred -import kotlin.reflect.KClass -import kotlin.reflect.full.isSubclassOf - -@Suppress("UNCHECKED_CAST") -fun Data.safeCast(type: KClass): Data? { - return if (this.type.isSubclassOf(type)) { - return object : Data { - override val meta: Meta get() = this@safeCast.meta - override val dependencies: Collection> get() = this@safeCast.dependencies - override val result: Deferred? get() = this@safeCast.result as Deferred - override fun startAsync(scope: CoroutineScope): Deferred = this@safeCast.startAsync(scope) as Deferred - override fun reset() = this@safeCast.reset() - override val type: KClass = type - } - } else { - null - } -} - -/** - * Filter a node by data and node type. Resulting node and its subnodes is guaranteed to have border type [type], - * but could contain empty nodes - */ -fun DataNode.cast(type: KClass): DataNode { - return if (this is CastDataNode) { - origin.cast(type) - } else { - CastDataNode(this, type) - } -} - -inline fun DataNode.cast(): DataNode = cast(R::class) - -class CastDataNode(val origin: DataNode, override val type: KClass) : DataNode { - override val items: Map> by lazy { - origin.items.mapNotNull { (key, item) -> - when (item) { - is DataItem.Leaf -> { - (item.value.safeCast(type))?.let { - key to DataItem.Leaf(it) - } - } - is DataItem.Node -> { - key to DataItem.Node(item.value.cast(type)) - } - } - }.associate { it } - } -} \ No newline at end of file diff --git a/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/TypeFilteredDataNode.kt b/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/TypeFilteredDataNode.kt new file mode 100644 index 00000000..d24de964 --- /dev/null +++ b/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/TypeFilteredDataNode.kt @@ -0,0 +1,25 @@ +package hep.dataforge.data + +import hep.dataforge.names.NameToken +import kotlin.reflect.KClass + + +/** + * A zero-copy data node wrapper that returns only children with appropriate type. + */ +class TypeFilteredDataNode(val origin: DataNode<*>, override val type: KClass) : DataNode { + override val items: Map> by lazy { + origin.items.mapNotNull { (key, item) -> + when (item) { + is DataItem.Leaf -> { + (item.value.filterIsInstance(type))?.let { + key to DataItem.Leaf(it) + } + } + is DataItem.Node -> { + key to DataItem.Node(item.value.filterIsInstance(type)) + } + } + }.associate { it } + } +} \ No newline at end of file diff --git a/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/dataJVM.kt b/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/dataJVM.kt index f87c4155..5b5507b2 100644 --- a/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/dataJVM.kt +++ b/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/dataJVM.kt @@ -2,6 +2,7 @@ package hep.dataforge.data import kotlinx.coroutines.runBlocking import kotlin.reflect.KClass +import kotlin.reflect.full.isSubclassOf import kotlin.reflect.full.isSuperclassOf /** @@ -12,8 +13,39 @@ fun Data.get(): T = runBlocking { await() } /** * Check that node is compatible with given type meaning that each element could be cast to the type */ -actual fun DataNode<*>.checkType(type: KClass<*>) { - if (!type.isSuperclassOf(type)) { - error("$type expected, but $type received") +actual fun DataNode<*>.canCast(type: KClass): Boolean = + type.isSuperclassOf(type) + +actual fun Data<*>.canCast(type: KClass): Boolean = + this.type.isSubclassOf(type) + +/** + * Cast the node to given type if the cast is possible or return null + */ +fun Data<*>.filterIsInstance(type: KClass): Data? = + if (canCast(type)) cast(type) else null + +/** + * Filter a node by data and node type. Resulting node and its subnodes is guaranteed to have border type [type], + * but could contain empty nodes + */ +fun DataNode<*>.filterIsInstance(type: KClass): DataNode { + return if (canCast(type)) { + cast(type) + } else if (this is TypeFilteredDataNode) { + origin.filterIsInstance(type) + } else { + TypeFilteredDataNode(this, type) } -} \ No newline at end of file +} + +/** + * Filter all elements of given data item that could be cast to given type. If no elements are available, return null. + */ +fun DataItem<*>?.filterIsInstance(type: KClass): DataItem? = when (this) { + null -> null + is DataItem.Node -> DataItem.Node(this.value.filterIsInstance(type)) + is DataItem.Leaf -> this.value.filterIsInstance(type)?.let { DataItem.Leaf(it) } +} + +inline fun DataItem<*>?.filterIsInstance(): DataItem? = this@filterIsInstance.filterIsInstance(R::class) \ No newline at end of file diff --git a/dataforge-io/build.gradle.kts b/dataforge-io/build.gradle.kts index b4306efc..083e9d53 100644 --- a/dataforge-io/build.gradle.kts +++ b/dataforge-io/build.gradle.kts @@ -5,8 +5,8 @@ plugins { description = "IO module" scientifik{ - serialization = true - io = true + withSerialization() + withIO() } @@ -17,6 +17,11 @@ kotlin { api(project(":dataforge-context")) } } + jvmMain{ + dependencies { + + } + } jsMain{ dependencies{ api(npm("text-encoding")) diff --git a/dataforge-io/dataforge-io-yaml/build.gradle.kts b/dataforge-io/dataforge-io-yaml/build.gradle.kts new file mode 100644 index 00000000..74ba43cf --- /dev/null +++ b/dataforge-io/dataforge-io-yaml/build.gradle.kts @@ -0,0 +1,12 @@ +plugins { + id("scientifik.jvm") +} + +description = "YAML meta IO" + +dependencies { + api(project(":dataforge-io")) + api("org.yaml:snakeyaml:1.25") + testImplementation(kotlin("test")) + testImplementation(kotlin("test-junit")) +} diff --git a/dataforge-io/dataforge-io-yaml/src/main/kotlin/hep/dataforge/io/yaml/FrontMatterEnvelopeFormat.kt b/dataforge-io/dataforge-io-yaml/src/main/kotlin/hep/dataforge/io/yaml/FrontMatterEnvelopeFormat.kt new file mode 100644 index 00000000..db701625 --- /dev/null +++ b/dataforge-io/dataforge-io-yaml/src/main/kotlin/hep/dataforge/io/yaml/FrontMatterEnvelopeFormat.kt @@ -0,0 +1,88 @@ +package hep.dataforge.io.yaml + +import hep.dataforge.context.Context +import hep.dataforge.io.* +import hep.dataforge.meta.DFExperimental +import hep.dataforge.meta.EmptyMeta +import hep.dataforge.meta.Meta +import kotlinx.io.core.* +import kotlinx.serialization.toUtf8Bytes + +@DFExperimental +class FrontMatterEnvelopeFormat( + val io: IOPlugin, + meta: Meta = EmptyMeta +) : EnvelopeFormat { + + override fun Input.readPartial(): PartialEnvelope { + var line: String = "" + var offset = 0u + do { + line = readUTF8Line() ?: error("Input does not contain front matter separator") + offset += line.toUtf8Bytes().size.toUInt() + } while (!line.startsWith(SEPARATOR)) + + val readMetaFormat = + metaTypeRegex.matchEntire(line)?.groupValues?.first() + ?.let { io.metaFormat(it) } ?: YamlMetaFormat.default + + val metaBlock = buildPacket { + do { + line = readUTF8Line() ?: error("Input does not contain closing front matter separator") + appendln(line) + offset += line.toUtf8Bytes().size.toUInt() + } while (!line.startsWith(SEPARATOR)) + } + val meta = readMetaFormat.fromBytes(metaBlock) + return PartialEnvelope(meta, offset, null) + } + + override fun Input.readObject(): Envelope { + var line: String = "" + do { + line = readUTF8Line() ?: error("Input does not contain front matter separator") + } while (!line.startsWith(SEPARATOR)) + + val readMetaFormat = + metaTypeRegex.matchEntire(line)?.groupValues?.first() + ?.let { io.metaFormat(it) } ?: YamlMetaFormat.default + + val metaBlock = buildPacket { + do { + appendln(readUTF8Line() ?: error("Input does not contain closing front matter separator")) + } while (!line.startsWith(SEPARATOR)) + } + val meta = readMetaFormat.fromBytes(metaBlock) + val bytes = readBytes() + val data = bytes.asBinary() + return SimpleEnvelope(meta, data) + } + + override fun Output.writeEnvelope(envelope: Envelope, metaFormatFactory: MetaFormatFactory, formatMeta: Meta) { + val metaFormat = metaFormatFactory(formatMeta, io.context) + writeText("$SEPARATOR\r\n") + metaFormat.run { writeObject(envelope.meta) } + writeText("$SEPARATOR\r\n") + envelope.data?.read { copyTo(this@writeEnvelope) } + } + + companion object : EnvelopeFormatFactory { + const val SEPARATOR = "---" + + private val metaTypeRegex = "---(\\w*)\\s*".toRegex() + + override fun invoke(meta: Meta, context: Context): EnvelopeFormat { + return FrontMatterEnvelopeFormat(context.io, meta) + } + + override fun peekFormat(io: IOPlugin, input: Input): EnvelopeFormat? { + val line = input.readUTF8Line(3, 30) + return if (line != null && line.startsWith("---")) { + invoke() + } else { + null + } + } + + } +} \ No newline at end of file diff --git a/dataforge-io/dataforge-io-yaml/src/main/kotlin/hep/dataforge/io/yaml/YamlMetaFormat.kt b/dataforge-io/dataforge-io-yaml/src/main/kotlin/hep/dataforge/io/yaml/YamlMetaFormat.kt new file mode 100644 index 00000000..24ea44ec --- /dev/null +++ b/dataforge-io/dataforge-io-yaml/src/main/kotlin/hep/dataforge/io/yaml/YamlMetaFormat.kt @@ -0,0 +1,56 @@ +package hep.dataforge.io.yaml + +import hep.dataforge.context.Context +import hep.dataforge.descriptors.NodeDescriptor +import hep.dataforge.io.MetaFormat +import hep.dataforge.io.MetaFormatFactory +import hep.dataforge.meta.DFExperimental +import hep.dataforge.meta.Meta +import hep.dataforge.meta.toMap +import hep.dataforge.meta.toMeta +import hep.dataforge.names.Name +import hep.dataforge.names.plus +import kotlinx.io.core.Input +import kotlinx.io.core.Output +import kotlinx.io.core.readUByte +import kotlinx.io.core.writeText +import org.yaml.snakeyaml.Yaml +import java.io.InputStream + +private class InputAsStream(val input: Input) : InputStream() { + override fun read(): Int { + if (input.endOfInput) return -1 + return input.readUByte().toInt() + } + + override fun close() { + input.close() + } +} + +private fun Input.asStream() = InputAsStream(this) + +@DFExperimental +class YamlMetaFormat(val meta: Meta) : MetaFormat { + private val yaml = Yaml() + + override fun Output.writeMeta(meta: Meta, descriptor: NodeDescriptor?) { + val string = yaml.dump(meta.toMap(descriptor)) + writeText(string) + } + + override fun Input.readMeta(descriptor: NodeDescriptor?): Meta { + val map: Map = yaml.load(asStream()) + return map.toMeta(descriptor) + } + + companion object : MetaFormatFactory { + val default = YamlMetaFormat() + + override fun invoke(meta: Meta, context: Context): MetaFormat = YamlMetaFormat(meta) + + override val name: Name = super.name + "yaml" + + override val key: Short = 0x594d //YM + } +} \ No newline at end of file diff --git a/dataforge-io/dataforge-io-yaml/src/test/kotlin/hep/dataforge/io/yaml/YamlMetaFormatTest.kt b/dataforge-io/dataforge-io-yaml/src/test/kotlin/hep/dataforge/io/yaml/YamlMetaFormatTest.kt new file mode 100644 index 00000000..414162f7 --- /dev/null +++ b/dataforge-io/dataforge-io-yaml/src/test/kotlin/hep/dataforge/io/yaml/YamlMetaFormatTest.kt @@ -0,0 +1,43 @@ +package hep.dataforge.io.yaml + +import hep.dataforge.io.parse +import hep.dataforge.io.toString +import hep.dataforge.meta.Meta +import hep.dataforge.meta.buildMeta +import hep.dataforge.meta.get +import hep.dataforge.meta.seal +import org.junit.Test +import kotlin.test.assertEquals + + +class YamlMetaFormatTest{ + @Test + fun testYamlMetaFormat(){ + val meta = buildMeta { + "a" put 22 + "node" put { + "b" put "DDD" + "c" put 11.1 + "d" put { + "d1" put { + "d11" put "aaa" + "d12" put "bbb" + } + "d2" put 2 + } + "array" put doubleArrayOf(1.0, 2.0, 3.0) + } + } + val string = meta.toString(YamlMetaFormat) + println(string) + val result = YamlMetaFormat.parse(string) + + assertEquals(meta, meta.seal()) + + meta.items.keys.forEach { + if (meta[it] != result[it]) error("${meta[it]} != ${result[it]}") + } + + assertEquals(meta, result) + } +} \ No newline at end of file diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Binary.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Binary.kt index 67c962dd..ca05de4d 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Binary.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Binary.kt @@ -1,9 +1,7 @@ package hep.dataforge.io -import kotlinx.io.core.ByteReadPacket -import kotlinx.io.core.Input -import kotlinx.io.core.buildPacket -import kotlinx.io.core.readBytes +import kotlinx.io.core.* +import kotlin.math.min /** * A source of binary data @@ -30,19 +28,21 @@ interface RandomAccessBinary : Binary { /** * Read at most [size] of bytes starting at [from] offset from the beginning of the binary. * This method could be called multiple times simultaneously. + * + * If size */ fun read(from: UInt, size: UInt = UInt.MAX_VALUE, block: Input.() -> R): R override fun read(block: Input.() -> R): R = read(0.toUInt(), UInt.MAX_VALUE, block) } -fun Binary.readAll(): ByteReadPacket = read { - ByteReadPacket(this.readBytes()) +fun Binary.toBytes(): ByteArray = read { + this.readBytes() } @ExperimentalUnsignedTypes fun RandomAccessBinary.readPacket(from: UInt, size: UInt): ByteReadPacket = read(from, size) { - ByteReadPacket(this.readBytes()) + buildPacket { copyTo(this) } } @ExperimentalUnsignedTypes @@ -53,33 +53,35 @@ object EmptyBinary : RandomAccessBinary { override fun read(from: UInt, size: UInt, block: Input.() -> R): R { error("The binary is empty") } + } @ExperimentalUnsignedTypes -class ArrayBinary(val array: ByteArray) : RandomAccessBinary { +inline class ArrayBinary(val array: ByteArray) : RandomAccessBinary { override val size: ULong get() = array.size.toULong() override fun read(from: UInt, size: UInt, block: Input.() -> R): R { - return ByteReadPacket(array, from.toInt(), size.toInt()).block() + val theSize = min(size, array.size.toUInt() - from) + return buildPacket { + writeFully(array, from.toInt(), theSize.toInt()) + }.block() } } +fun ByteArray.asBinary() = ArrayBinary(this) + /** * Read given binary as object using given format */ fun Binary.readWith(format: IOFormat): T = format.run { read { - readThis() + readObject() } } -/** - * Write this object to a binary - * TODO make a lazy binary that does not use intermediate array - */ -fun T.writeWith(format: IOFormat): Binary = format.run{ +fun IOFormat.writeBinary(obj: T): Binary { val packet = buildPacket { - writeThis(this@writeWith) + writeObject(obj) } - return@run ArrayBinary(packet.readBytes()) + return ArrayBinary(packet.readBytes()) } \ No newline at end of file diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/BinaryMetaFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/BinaryMetaFormat.kt index daf08756..ba9886d8 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/BinaryMetaFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/BinaryMetaFormat.kt @@ -1,17 +1,22 @@ package hep.dataforge.io +import hep.dataforge.context.Context import hep.dataforge.descriptors.NodeDescriptor import hep.dataforge.meta.* +import hep.dataforge.names.Name +import hep.dataforge.names.plus import hep.dataforge.values.* import kotlinx.io.core.Input import kotlinx.io.core.Output import kotlinx.io.core.readText import kotlinx.io.core.writeText -object BinaryMetaFormat : MetaFormat { - override val name: String = "bin" +object BinaryMetaFormat : MetaFormat, MetaFormatFactory { + override val name: Name = super.name + "bin" override val key: Short = 0x4249//BI + override fun invoke(meta: Meta, context: Context): MetaFormat = this + override fun Input.readMeta(descriptor: NodeDescriptor?): Meta { return (readMetaItem() as MetaItem.NodeItem).node } @@ -23,7 +28,7 @@ object BinaryMetaFormat : MetaFormat { writeText(str) } - private fun Output.writeValue(value: Value) { + fun Output.writeValue(value: Value) { if (value.isList()) { writeChar('L') writeInt(value.list.size) @@ -80,7 +85,7 @@ object BinaryMetaFormat : MetaFormat { writeValue(item.value) } is MetaItem.NodeItem -> { - writeThis(item.node) + writeObject(item.node) } } } @@ -92,7 +97,7 @@ object BinaryMetaFormat : MetaFormat { } @Suppress("UNCHECKED_CAST") - private fun Input.readMetaItem(): MetaItem { + fun Input.readMetaItem(): MetaItem { return when (val keyChar = readByte().toChar()) { 'S' -> MetaItem.ValueItem(StringValue(readString())) 'N' -> MetaItem.ValueItem(Null) diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Envelope.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Envelope.kt index c2abca21..80e07b56 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Envelope.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Envelope.kt @@ -1,9 +1,11 @@ package hep.dataforge.io -import hep.dataforge.meta.Laminate -import hep.dataforge.meta.Meta -import hep.dataforge.meta.get -import hep.dataforge.meta.string +import hep.dataforge.meta.* +import hep.dataforge.names.asName +import hep.dataforge.names.plus +import kotlinx.io.core.Output +import kotlinx.io.core.buildPacket +import kotlinx.io.core.readBytes interface Envelope { val meta: Meta @@ -14,12 +16,17 @@ interface Envelope { /** * meta keys */ - const val ENVELOPE_NODE = "@envelope" - const val ENVELOPE_TYPE_KEY = "$ENVELOPE_NODE.type" - const val ENVELOPE_DATA_TYPE_KEY = "$ENVELOPE_NODE.dataType" - const val ENVELOPE_DESCRIPTION_KEY = "$ENVELOPE_NODE.description" + val ENVELOPE_NODE_KEY = "@envelope".asName() + val ENVELOPE_TYPE_KEY = ENVELOPE_NODE_KEY + "type" + val ENVELOPE_DATA_TYPE_KEY = ENVELOPE_NODE_KEY + "dataType" + val ENVELOPE_DATA_ID_KEY = ENVELOPE_NODE_KEY + "dataID" + val ENVELOPE_DESCRIPTION_KEY = ENVELOPE_NODE_KEY + "description" //const val ENVELOPE_TIME_KEY = "@envelope.time" + /** + * Build a static envelope using provided builder + */ + operator fun invoke(block: EnvelopeBuilder.() -> Unit) = EnvelopeBuilder().apply(block).build() } } @@ -28,24 +35,35 @@ class SimpleEnvelope(override val meta: Meta, override val data: Binary?) : Enve /** * The purpose of the envelope * - * @return */ val Envelope.type: String? get() = meta[Envelope.ENVELOPE_TYPE_KEY].string /** * The type of data encoding * - * @return */ val Envelope.dataType: String? get() = meta[Envelope.ENVELOPE_DATA_TYPE_KEY].string /** * Textual user friendly description * - * @return */ val Envelope.description: String? get() = meta[Envelope.ENVELOPE_DESCRIPTION_KEY].string +/** + * An optional unique identifier that is used for data comparison. Data without identifier could not be compared to another data. + */ +val Envelope.dataID: String? get() = meta[Envelope.ENVELOPE_DATA_ID_KEY].string + +fun Envelope.metaEquals(other: Envelope): Boolean = this.meta == other.meta + +fun Envelope.dataEquals(other: Envelope): Boolean = this.dataID != null && this.dataID == other.dataID + +fun Envelope.contentEquals(other: Envelope): Boolean { + return (this === other || (metaEquals(other) && dataEquals(other))) +} + + /** * An envelope, which wraps existing envelope and adds one or several additional layers of meta */ @@ -55,7 +73,7 @@ class ProxyEnvelope(val source: Envelope, vararg meta: Meta) : Envelope { } /** - * Add few meta layers to existing envelope + * Add few meta layers to existing envelope (on top of existing meta) */ fun Envelope.withMetaLayers(vararg layers: Meta): Envelope { return when { @@ -63,4 +81,34 @@ fun Envelope.withMetaLayers(vararg layers: Meta): Envelope { this is ProxyEnvelope -> ProxyEnvelope(source, *layers, *this.meta.layers.toTypedArray()) else -> ProxyEnvelope(this, *layers) } +} + +class EnvelopeBuilder { + private val metaBuilder = MetaBuilder() + var data: Binary? = null + + fun meta(block: MetaBuilder.() -> Unit) { + metaBuilder.apply(block) + } + + fun meta(meta: Meta) { + metaBuilder.update(meta) + } + + var type by metaBuilder.string(key = Envelope.ENVELOPE_TYPE_KEY) + var dataType by metaBuilder.string(key = Envelope.ENVELOPE_DATA_TYPE_KEY) + var dataID by metaBuilder.string(key = Envelope.ENVELOPE_DATA_ID_KEY) + var description by metaBuilder.string(key = Envelope.ENVELOPE_DESCRIPTION_KEY) + + /** + * Construct a binary and transform it into byte-array based buffer + */ + fun data(block: Output.() -> Unit) { + val bytes = buildPacket { + block() + } + data = ArrayBinary(bytes.readBytes()) + } + + internal fun build() = SimpleEnvelope(metaBuilder.seal(), data) } \ No newline at end of file diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/EnvelopeFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/EnvelopeFormat.kt index 24217e14..c52b9e1d 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/EnvelopeFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/EnvelopeFormat.kt @@ -1,11 +1,15 @@ package hep.dataforge.io -import hep.dataforge.context.Named -import hep.dataforge.io.EnvelopeFormat.Companion.ENVELOPE_FORMAT_TYPE +import hep.dataforge.context.Context +import hep.dataforge.io.EnvelopeFormatFactory.Companion.ENVELOPE_FORMAT_TYPE +import hep.dataforge.meta.EmptyMeta import hep.dataforge.meta.Meta +import hep.dataforge.names.Name +import hep.dataforge.names.asName import hep.dataforge.provider.Type import kotlinx.io.core.Input import kotlinx.io.core.Output +import kotlin.reflect.KClass /** * A partially read envelope with meta, but without data @@ -13,19 +17,33 @@ import kotlinx.io.core.Output @ExperimentalUnsignedTypes data class PartialEnvelope(val meta: Meta, val dataOffset: UInt, val dataSize: ULong?) + +interface EnvelopeFormat : IOFormat { + val defaultMetaFormat: MetaFormatFactory get() = JsonMetaFormat + + fun Input.readPartial(): PartialEnvelope + + fun Output.writeEnvelope(envelope: Envelope, metaFormatFactory: MetaFormatFactory, formatMeta: Meta = EmptyMeta) + + override fun Input.readObject(): Envelope + + override fun Output.writeObject(obj: Envelope): Unit = writeEnvelope(obj, defaultMetaFormat) +} + @Type(ENVELOPE_FORMAT_TYPE) -interface EnvelopeFormat : IOFormat, Named { - fun Input.readPartial(formats: Collection = IOPlugin.defaultMetaFormats): PartialEnvelope +interface EnvelopeFormatFactory : IOFormatFactory { + override val name: Name get() = "envelope".asName() + override val type: KClass get() = Envelope::class - fun Input.readEnvelope(formats: Collection = IOPlugin.defaultMetaFormats): Envelope + override fun invoke(meta: Meta, context: Context): EnvelopeFormat - override fun Input.readThis(): Envelope = readEnvelope() - - fun Output.writeEnvelope(envelope: Envelope, format: MetaFormat = JsonMetaFormat) - - override fun Output.writeThis(obj: Envelope) = writeEnvelope(obj) + /** + * Try to infer specific format from input and return null if the attempt is failed. + * This method does **not** return Input into initial state. + */ + fun peekFormat(io: IOPlugin, input: Input): EnvelopeFormat? companion object { - const val ENVELOPE_FORMAT_TYPE = "envelopeFormat" + const val ENVELOPE_FORMAT_TYPE = "io.format.envelope" } } \ No newline at end of file diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/FunctionServer.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/FunctionServer.kt deleted file mode 100644 index 2b254716..00000000 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/FunctionServer.kt +++ /dev/null @@ -1,38 +0,0 @@ -package hep.dataforge.io - -import kotlin.reflect.KClass - -/** - * A descriptor for specific type of functions - */ -interface FunctionSpec { - val inputType: KClass - val outputType: KClass -} - -/** - * A server that could produce asynchronous function values - */ -interface FunctionServer { - /** - * Call a function with given name and descriptor - */ - suspend fun > call(name: String, descriptor: D, arg: T): R - - /** - * Resolve a function descriptor for given types - */ - fun resolveType(inputType: KClass, outputType: KClass): FunctionSpec - - /** - * Get a generic suspended function with given name and descriptor - */ - operator fun > get(name: String, descriptor: D): (suspend (T) -> R) = - { call(name, descriptor, it) } -} - -suspend inline fun FunctionServer.call(name: String, arg: T): R = - call(name, resolveType(T::class, R::class), arg) - -inline operator fun FunctionServer.get(name: String): (suspend (T) -> R) = - get(name, resolveType(T::class, R::class)) diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOFormat.kt index 9cc9a584..093ffbc8 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOFormat.kt @@ -1,14 +1,164 @@ package hep.dataforge.io +import hep.dataforge.context.Context +import hep.dataforge.context.Factory +import hep.dataforge.context.Named +import hep.dataforge.io.IOFormatFactory.Companion.IO_FORMAT_TYPE +import hep.dataforge.meta.Meta +import hep.dataforge.meta.MetaItem +import hep.dataforge.names.Name +import hep.dataforge.names.asName +import hep.dataforge.provider.Type +import hep.dataforge.values.Value import kotlinx.io.core.* +import kotlinx.io.pool.ObjectPool +import kotlinx.serialization.ImplicitReflectionSerializer +import kotlinx.serialization.KSerializer +import kotlinx.serialization.cbor.Cbor +import kotlinx.serialization.serializer +import kotlin.math.min +import kotlin.reflect.KClass /** - * And interface for serialization facilities + * And interface for reading and writing objects into with IO streams */ interface IOFormat { - fun Output.writeThis(obj: T) - fun Input.readThis(): T + fun Output.writeObject(obj: T) + fun Input.readObject(): T } -fun IOFormat.writePacket(obj: T): ByteReadPacket = buildPacket { writeThis(obj) } -fun IOFormat.writeBytes(obj: T): ByteArray = buildPacket { writeThis(obj) }.readBytes() \ No newline at end of file +fun Input.readWith(format: IOFormat): T = format.run { readObject() } +fun Output.writeWith(format: IOFormat, obj: T) = format.run { writeObject(obj) } + +class ListIOFormat(val format: IOFormat) : IOFormat> { + override fun Output.writeObject(obj: List) { + writeInt(obj.size) + format.run { + obj.forEach { + writeObject(it) + } + } + } + + override fun Input.readObject(): List { + val size = readInt() + return format.run { + List(size) { readObject() } + } + } +} + +val IOFormat.list get() = ListIOFormat(this) + +fun ObjectPool.fill(block: IoBuffer.() -> Unit): IoBuffer { + val buffer = borrow() + return try { + buffer.apply(block) + } catch (ex: Exception) { + //recycle(buffer) + throw ex + } +} + +@Type(IO_FORMAT_TYPE) +interface IOFormatFactory : Factory>, Named { + /** + * Explicit type for dynamic type checks + */ + val type: KClass + + companion object { + const val IO_FORMAT_TYPE = "io.format" + } +} + +@Deprecated("To be removed in io-2") +inline fun buildPacketWithoutPool(headerSizeHint: Int = 0, block: BytePacketBuilder.() -> Unit): ByteReadPacket { + val builder = BytePacketBuilder(headerSizeHint, IoBuffer.NoPool) + block(builder) + return builder.build() +} + +fun IOFormat.writePacket(obj: T): ByteReadPacket = buildPacket { writeObject(obj) } +fun IOFormat.writeBytes(obj: T): ByteArray = buildPacket { writeObject(obj) }.readBytes() +fun IOFormat.readBytes(array: ByteArray): T { + //= ByteReadPacket(array).readThis() + val byteArrayInput: Input = object : AbstractInput( + IoBuffer.Pool.borrow(), + remaining = array.size.toLong(), + pool = IoBuffer.Pool + ) { + var written = 0 + override fun closeSource() { + // do nothing + } + + override fun fill(): IoBuffer? { + if (array.size - written <= 0) return null + + return IoBuffer.Pool.fill { + reserveEndGap(IoBuffer.ReservedSize) + val toWrite = min(capacity, array.size - written) + writeFully(array, written, toWrite) + written += toWrite + } + } + + } + return byteArrayInput.readObject() +} + +object DoubleIOFormat : IOFormat, IOFormatFactory { + override fun invoke(meta: Meta, context: Context): IOFormat = this + + override val name: Name = "double".asName() + + override val type: KClass get() = Double::class + + override fun Output.writeObject(obj: Double) { + writeDouble(obj) + } + + override fun Input.readObject(): Double = readDouble() +} + +object ValueIOFormat : IOFormat, IOFormatFactory { + override fun invoke(meta: Meta, context: Context): IOFormat = this + + override val name: Name = "value".asName() + + override val type: KClass get() = Value::class + + override fun Output.writeObject(obj: Value) { + BinaryMetaFormat.run { writeValue(obj) } + } + + override fun Input.readObject(): Value { + return (BinaryMetaFormat.run { readMetaItem() } as? MetaItem.ValueItem)?.value + ?: error("The item is not a value") + } +} + +/** + * Experimental + */ +@ImplicitReflectionSerializer +class SerializerIOFormat( + type: KClass, + val serializer: KSerializer = type.serializer() +) : IOFormat { + + //override val name: Name = type.simpleName?.toName() ?: EmptyName + + + override fun Output.writeObject(obj: T) { + val bytes = Cbor.plain.dump(serializer, obj) + writeFully(bytes) + } + + override fun Input.readObject(): T { + //FIXME reads the whole input + val bytes = readBytes() + return Cbor.plain.load(serializer, bytes) + } +} \ No newline at end of file diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOPlugin.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOPlugin.kt index b65e4982..7e61924f 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOPlugin.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOPlugin.kt @@ -1,37 +1,61 @@ package hep.dataforge.io -import hep.dataforge.context.AbstractPlugin -import hep.dataforge.context.PluginFactory -import hep.dataforge.context.PluginTag -import hep.dataforge.context.content -import hep.dataforge.meta.Meta +import hep.dataforge.context.* +import hep.dataforge.io.EnvelopeFormatFactory.Companion.ENVELOPE_FORMAT_TYPE +import hep.dataforge.io.IOFormatFactory.Companion.IO_FORMAT_TYPE +import hep.dataforge.io.MetaFormatFactory.Companion.META_FORMAT_TYPE +import hep.dataforge.meta.* import hep.dataforge.names.Name +import hep.dataforge.names.get import kotlin.reflect.KClass class IOPlugin(meta: Meta) : AbstractPlugin(meta) { override val tag: PluginTag get() = Companion.tag - val metaFormats by lazy { - context.content(MetaFormat.META_FORMAT_TYPE).values + val metaFormatFactories by lazy { + context.content(META_FORMAT_TYPE).values } - fun metaFormat(key: Short): MetaFormat? = metaFormats.find { it.key == key } - fun metaFormat(name: String): MetaFormat? = metaFormats.find { it.name == name } + fun metaFormat(key: Short, meta: Meta = EmptyMeta): MetaFormat? = + metaFormatFactories.find { it.key == key }?.invoke(meta) + + fun metaFormat(name: String, meta: Meta = EmptyMeta): MetaFormat? = + metaFormatFactories.find { it.name.toString() == name }?.invoke(meta) + + val envelopeFormatFactories by lazy { + context.content(ENVELOPE_FORMAT_TYPE).values + } override fun provideTop(target: String): Map { return when (target) { - MetaFormat.META_FORMAT_TYPE -> defaultMetaFormats.toMap() - EnvelopeFormat.ENVELOPE_FORMAT_TYPE -> defaultEnvelopeFormats.toMap() + META_FORMAT_TYPE -> defaultMetaFormats.toMap() + ENVELOPE_FORMAT_TYPE -> defaultEnvelopeFormats.toMap() else -> super.provideTop(target) } } + val ioFormats: Map> by lazy { + context.content>(IO_FORMAT_TYPE) + } + + fun resolveIOFormat(item: MetaItem<*>, type: KClass): IOFormat? { + val key = item.string ?: item.node["name"]?.string ?: error("Format name not defined") + return ioFormats[key]?.let { + @Suppress("UNCHECKED_CAST") + if (it.type != type) error("Format type ${it.type} is not the same as requested type $type") + else it.invoke(item.node["meta"].node ?: EmptyMeta, context) as IOFormat + } + } + companion object : PluginFactory { - val defaultMetaFormats: List = listOf(JsonMetaFormat, BinaryMetaFormat) + val defaultMetaFormats: List = listOf(JsonMetaFormat, BinaryMetaFormat) val defaultEnvelopeFormats = listOf(TaggedEnvelopeFormat) override val tag: PluginTag = PluginTag("io", group = PluginTag.DATAFORGE_GROUP) + override val type: KClass = IOPlugin::class - override fun invoke(meta: Meta): IOPlugin = IOPlugin(meta) + override fun invoke(meta: Meta, context: Context): IOPlugin = IOPlugin(meta) } -} \ No newline at end of file +} + +val Context.io: IOPlugin get() = plugins.fetch(IOPlugin) \ No newline at end of file diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/JsonMetaFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/JsonMetaFormat.kt index 64b368e2..a95cdec4 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/JsonMetaFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/JsonMetaFormat.kt @@ -1,12 +1,17 @@ +@file:Suppress("UNUSED_PARAMETER") + package hep.dataforge.io +import hep.dataforge.context.Context import hep.dataforge.descriptors.ItemDescriptor import hep.dataforge.descriptors.NodeDescriptor import hep.dataforge.descriptors.ValueDescriptor import hep.dataforge.meta.Meta import hep.dataforge.meta.MetaBase import hep.dataforge.meta.MetaItem +import hep.dataforge.names.Name import hep.dataforge.names.NameToken +import hep.dataforge.names.plus import hep.dataforge.names.toName import hep.dataforge.values.* import kotlinx.io.core.Input @@ -19,28 +24,32 @@ import kotlin.collections.component2 import kotlin.collections.set -object JsonMetaFormat : MetaFormat { - - override val name: String = "json" - override val key: Short = 0x4a53//"JS" +class JsonMetaFormat(private val json: Json = Json.indented) : MetaFormat { override fun Output.writeMeta(meta: Meta, descriptor: NodeDescriptor?) { - val json = meta.toJson(descriptor) - writeText(json.toString()) + val jsonObject = meta.toJson(descriptor) + writeText(json.stringify(JsonObjectSerializer, jsonObject)) } override fun Input.readMeta(descriptor: NodeDescriptor?): Meta { val str = readText() - val json = Json.plain.parseJson(str) + val jsonElement = json.parseJson(str) + return jsonElement.toMeta() + } - if (json is JsonObject) { - return json.toMeta() - } else { - TODO("Non-object root not supported") - } + companion object : MetaFormatFactory { + val default = JsonMetaFormat() + + override fun invoke(meta: Meta, context: Context): MetaFormat = default + + override val name: Name = super.name + "json" + override val key: Short = 0x4a53//"JS" } } +/** + * @param descriptor reserved for custom serialization in future + */ fun Value.toJson(descriptor: ValueDescriptor? = null): JsonElement { return if (isList()) { JsonArray(list.map { it.toJson() }) @@ -54,7 +63,7 @@ fun Value.toJson(descriptor: ValueDescriptor? = null): JsonElement { } } -//Use theese methods to customize JSON key mapping +//Use these methods to customize JSON key mapping private fun NameToken.toJsonKey(descriptor: ItemDescriptor?) = toString() private fun NodeDescriptor?.getDescriptor(key: String) = this?.items?.get(key) @@ -78,7 +87,12 @@ fun Meta.toJson(descriptor: NodeDescriptor? = null): JsonObject { return JsonObject(map) } -fun JsonObject.toMeta(descriptor: NodeDescriptor? = null): JsonMeta = JsonMeta(this, descriptor) +fun JsonElement.toMeta(descriptor: NodeDescriptor? = null): Meta { + return when (val item = toMetaItem(descriptor)) { + is MetaItem.NodeItem<*> -> item.node + is MetaItem.ValueItem ->item.value.toMeta() + } +} fun JsonPrimitive.toValue(descriptor: ValueDescriptor?): Value { return when (this) { @@ -93,7 +107,7 @@ fun JsonElement.toMetaItem(descriptor: ItemDescriptor? = null): MetaItem { - val meta = toMeta(descriptor as? NodeDescriptor) + val meta = JsonMeta(this, descriptor as? NodeDescriptor) MetaItem.NodeItem(meta) } is JsonArray -> { @@ -129,7 +143,7 @@ class JsonMeta(val json: JsonObject, val descriptor: NodeDescriptor? = null) : M this[name] = MetaItem.ValueItem(value.toValue(itemDescriptor as? ValueDescriptor)) as MetaItem } is JsonObject -> { - this[name] = MetaItem.NodeItem(value.toMeta(itemDescriptor as? NodeDescriptor)) + this[name] = MetaItem.NodeItem(JsonMeta(value, itemDescriptor as? NodeDescriptor)) } is JsonArray -> { when { diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/MetaFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/MetaFormat.kt index 3185e29e..ca9a53a2 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/MetaFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/MetaFormat.kt @@ -1,49 +1,64 @@ package hep.dataforge.io -import hep.dataforge.context.Named +import hep.dataforge.context.Context import hep.dataforge.descriptors.NodeDescriptor -import hep.dataforge.io.MetaFormat.Companion.META_FORMAT_TYPE +import hep.dataforge.io.MetaFormatFactory.Companion.META_FORMAT_TYPE import hep.dataforge.meta.Meta +import hep.dataforge.names.Name +import hep.dataforge.names.asName import hep.dataforge.provider.Type import kotlinx.io.core.* +import kotlin.reflect.KClass /** * A format for meta serialization */ -@Type(META_FORMAT_TYPE) -interface MetaFormat : IOFormat, Named { - override val name: String - val key: Short - override fun Output.writeThis(obj: Meta) { +interface MetaFormat : IOFormat { + + override fun Output.writeObject(obj: Meta) { writeMeta(obj, null) } - override fun Input.readThis(): Meta = readMeta(null) + override fun Input.readObject(): Meta = readMeta() fun Output.writeMeta(meta: Meta, descriptor: NodeDescriptor? = null) fun Input.readMeta(descriptor: NodeDescriptor? = null): Meta +} - companion object{ - const val META_FORMAT_TYPE = "metaFormat" +@Type(META_FORMAT_TYPE) +interface MetaFormatFactory : IOFormatFactory { + override val name: Name get() = "meta".asName() + + override val type: KClass get() = Meta::class + + val key: Short + + override operator fun invoke(meta: Meta, context: Context): MetaFormat + + companion object { + const val META_FORMAT_TYPE = "io.format.meta" } } fun Meta.toString(format: MetaFormat): String = buildPacket { - format.run { writeThis(this@toString) } + format.run { writeObject(this@toString) } }.readText() -fun Meta.toBytes(format: MetaFormat = JsonMetaFormat): ByteReadPacket = buildPacket { - format.run { writeThis(this@toBytes) } -} +fun Meta.toString(formatFactory: MetaFormatFactory): String = toString(formatFactory()) +fun Meta.toBytes(format: MetaFormat = JsonMetaFormat.default): ByteReadPacket = buildPacket { + format.run { writeObject(this@toBytes) } +} fun MetaFormat.parse(str: String): Meta { - return ByteReadPacket(str.toByteArray()).readThis() + return buildPacket { writeText(str) }.readObject() } +fun MetaFormatFactory.parse(str: String): Meta = invoke().parse(str) + fun MetaFormat.fromBytes(packet: ByteReadPacket): Meta { - return packet.readThis() + return packet.readObject() } diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/MetaSerializer.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/MetaSerializer.kt deleted file mode 100644 index e1a754eb..00000000 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/MetaSerializer.kt +++ /dev/null @@ -1,53 +0,0 @@ -package hep.dataforge.io - -import hep.dataforge.meta.Config -import hep.dataforge.meta.Meta -import hep.dataforge.meta.toConfig -import hep.dataforge.names.Name -import hep.dataforge.names.toName -import kotlinx.serialization.* -import kotlinx.serialization.internal.StringDescriptor -import kotlinx.serialization.json.JsonObjectSerializer - -@Serializer(Name::class) -object NameSerializer : KSerializer { - override val descriptor: SerialDescriptor = StringDescriptor - - override fun deserialize(decoder: Decoder): Name { - return decoder.decodeString().toName() - } - - override fun serialize(encoder: Encoder, obj: Name) { - encoder.encodeString(obj.toString()) - } -} - -/** - * Serialized for meta - */ -@Serializer(Meta::class) -object MetaSerializer : KSerializer { - override val descriptor: SerialDescriptor = JsonObjectSerializer.descriptor - - override fun deserialize(decoder: Decoder): Meta { - //currently just delegates serialization to json serializer - return JsonObjectSerializer.deserialize(decoder).toMeta() - } - - override fun serialize(encoder: Encoder, obj: Meta) { - JsonObjectSerializer.serialize(encoder, obj.toJson()) - } -} - -@Serializer(Config::class) -object ConfigSerializer : KSerializer { - override val descriptor: SerialDescriptor = JsonObjectSerializer.descriptor - - override fun deserialize(decoder: Decoder): Config { - return JsonObjectSerializer.deserialize(decoder).toMeta().toConfig() - } - - override fun serialize(encoder: Encoder, obj: Config) { - JsonObjectSerializer.serialize(encoder, obj.toJson()) - } -} \ No newline at end of file diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Responder.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Responder.kt new file mode 100644 index 00000000..e3545782 --- /dev/null +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Responder.kt @@ -0,0 +1,8 @@ +package hep.dataforge.io + +interface Responder { + /** + * Send a request and wait for response for this specific request + */ + suspend fun respond(request: Envelope): Envelope +} \ No newline at end of file diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt index 291f539d..cce3eade 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt @@ -1,43 +1,50 @@ package hep.dataforge.io +import hep.dataforge.context.Context +import hep.dataforge.meta.Meta +import hep.dataforge.meta.get +import hep.dataforge.meta.string +import hep.dataforge.names.Name +import hep.dataforge.names.plus +import hep.dataforge.names.toName +import kotlinx.io.charsets.Charsets import kotlinx.io.core.* - @ExperimentalUnsignedTypes -object TaggedEnvelopeFormat : EnvelopeFormat { - const val VERSION = "DF03" - private const val START_SEQUENCE = "#~" - private const val END_SEQUENCE = "~#\r\n" - private const val TAG_SIZE = 26u +class TaggedEnvelopeFormat( + val io: IOPlugin, + val version: VERSION = TaggedEnvelopeFormat.VERSION.DF02 +) : EnvelopeFormat { + +// private val metaFormat = io.metaFormat(metaFormatKey) +// ?: error("Meta format with key $metaFormatKey could not be resolved in $io") - override val name: String get() = VERSION private fun Tag.toBytes(): ByteReadPacket = buildPacket(24) { writeText(START_SEQUENCE) - writeText(VERSION) + writeText(version.name) writeShort(metaFormatKey) writeUInt(metaSize) - writeULong(dataSize) + when (version) { + TaggedEnvelopeFormat.VERSION.DF02 -> { + writeUInt(dataSize.toUInt()) + } + TaggedEnvelopeFormat.VERSION.DF03 -> { + writeULong(dataSize) + } + } writeText(END_SEQUENCE) } - private fun Input.readTag(): Tag { - val start = readTextExactBytes(2) - if (start != START_SEQUENCE) error("The input is not an envelope") - val version = readTextExactBytes(4) - if (version != VERSION) error("Wrong version of DataForge: expected $VERSION but found $version") - val metaFormatKey = readShort() - val metaLength = readUInt() - val dataLength = readULong() - return Tag(metaFormatKey, metaLength, dataLength) - } - - override fun Output.writeEnvelope(envelope: Envelope, format: MetaFormat) { - val metaBytes = format.writeBytes(envelope.meta) - val tag = Tag(format.key, metaBytes.size.toUInt(), envelope.data?.size ?: 0.toULong()) + override fun Output.writeEnvelope(envelope: Envelope, metaFormatFactory: MetaFormatFactory, formatMeta: Meta) { + val metaFormat = metaFormatFactory.invoke(formatMeta, io.context) + val metaBytes = metaFormat.writeBytes(envelope.meta) + val tag = Tag(metaFormatFactory.key, metaBytes.size.toUInt() + 2u, envelope.data?.size ?: 0.toULong()) writePacket(tag.toBytes()) writeFully(metaBytes) + writeText("\r\n") envelope.data?.read { copyTo(this@writeEnvelope) } + flush() } /** @@ -46,30 +53,29 @@ object TaggedEnvelopeFormat : EnvelopeFormat { * @param input an input to read from * @param formats a collection of meta formats to resolve */ - override fun Input.readEnvelope(formats: Collection): Envelope { - val tag = readTag() + override fun Input.readObject(): Envelope { + val tag = readTag(version) - val metaFormat = formats.find { it.key == tag.metaFormatKey } + val metaFormat = io.metaFormat(tag.metaFormatKey) ?: error("Meta format with key ${tag.metaFormatKey} not found") val metaPacket = ByteReadPacket(readBytes(tag.metaSize.toInt())) - val meta = metaFormat.run { metaPacket.readThis() } - val dataBytes = readBytes(tag.dataSize.toInt()) + val meta = metaFormat.run { metaPacket.readObject() } return SimpleEnvelope(meta, ArrayBinary(dataBytes)) } - override fun Input.readPartial(formats: Collection): PartialEnvelope { - val tag = readTag() + override fun Input.readPartial(): PartialEnvelope { + val tag = readTag(version) - val metaFormat = formats.find { it.key == tag.metaFormatKey } + val metaFormat = io.metaFormat(tag.metaFormatKey) ?: error("Meta format with key ${tag.metaFormatKey} not found") val metaPacket = ByteReadPacket(readBytes(tag.metaSize.toInt())) - val meta = metaFormat.run { metaPacket.readThis() } + val meta = metaFormat.run { metaPacket.readObject() } - return PartialEnvelope(meta, TAG_SIZE + tag.metaSize, tag.dataSize) + return PartialEnvelope(meta, version.tagSize + tag.metaSize, tag.dataSize) } private data class Tag( @@ -78,4 +84,57 @@ object TaggedEnvelopeFormat : EnvelopeFormat { val dataSize: ULong ) + enum class VERSION(val tagSize: UInt) { + DF02(20u), + DF03(24u) + } + + companion object : EnvelopeFormatFactory { + private const val START_SEQUENCE = "#~" + private const val END_SEQUENCE = "~#\r\n" + + override val name: Name = super.name + "tagged" + + override fun invoke(meta: Meta, context: Context): EnvelopeFormat { + val io = context.io + + val metaFormatName = meta["name"].string?.toName() ?: JsonMetaFormat.name + val metaFormatFactory = io.metaFormatFactories.find { it.name == metaFormatName } + ?: error("Meta format could not be resolved") + + return TaggedEnvelopeFormat(io) + } + + private fun Input.readTag(version: VERSION): Tag { + val start = readTextExactBytes(2, charset = Charsets.ISO_8859_1) + if (start != START_SEQUENCE) error("The input is not an envelope") + val versionString = readTextExactBytes(4, charset = Charsets.ISO_8859_1) + if (version.name != versionString) error("Wrong version of DataForge: expected $version but found $versionString") + val metaFormatKey = readShort() + val metaLength = readUInt() + val dataLength: ULong = when (version) { + VERSION.DF02 -> readUInt().toULong() + VERSION.DF03 -> readULong() + } + val end = readTextExactBytes(4, charset = Charsets.ISO_8859_1) + if (end != END_SEQUENCE) error("The input is not an envelope") + return Tag(metaFormatKey, metaLength, dataLength) + } + + override fun peekFormat(io: IOPlugin, input: Input): EnvelopeFormat? { + return try { + val header = input.readTextExactBytes(6) + when (header.substring(2..5)) { + VERSION.DF02.name -> TaggedEnvelopeFormat(io, VERSION.DF02) + VERSION.DF03.name -> TaggedEnvelopeFormat(io, VERSION.DF03) + else -> null + } + } catch (ex: Exception) { + null + } + } + + val default by lazy { invoke() } + } + } \ No newline at end of file diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaglessEnvelopeFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaglessEnvelopeFormat.kt new file mode 100644 index 00000000..14d871db --- /dev/null +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaglessEnvelopeFormat.kt @@ -0,0 +1,189 @@ +package hep.dataforge.io + +import hep.dataforge.context.Context +import hep.dataforge.meta.* +import hep.dataforge.names.asName +import kotlinx.io.core.* +import kotlinx.serialization.toUtf8Bytes + +class TaglessEnvelopeFormat( + val io: IOPlugin, + meta: Meta = EmptyMeta +) : EnvelopeFormat { + + private val metaStart = meta[META_START_PROPERTY].string ?: DEFAULT_META_START + private val dataStart = meta[DATA_START_PROPERTY].string ?: DEFAULT_DATA_START + + private fun Output.writeProperty(key: String, value: Any) { + writeText("#? $key: $value;\r\n") + } + + override fun Output.writeEnvelope(envelope: Envelope, metaFormatFactory: MetaFormatFactory, formatMeta: Meta) { + val metaFormat = metaFormatFactory(formatMeta, io.context) + + //printing header + writeText(TAGLESS_ENVELOPE_HEADER + "\r\n") + + //printing all properties + writeProperty(META_TYPE_PROPERTY, metaFormatFactory.type) + //TODO add optional metaFormat properties + writeProperty(DATA_LENGTH_PROPERTY, envelope.data?.size ?: 0) + + //Printing meta + if (!envelope.meta.isEmpty()) { + val metaBytes = metaFormat.writeBytes(envelope.meta) + writeProperty(META_LENGTH_PROPERTY, metaBytes.size) + writeText(metaStart + "\r\n") + writeFully(metaBytes) + writeText("\r\n") + } + + //Printing data + envelope.data?.let { data -> + writeText(dataStart + "\r\n") + writeFully(data.toBytes()) + } + } + + override fun Input.readObject(): Envelope { + var line: String = "" + do { + line = readUTF8Line() ?: error("Input does not contain tagless envelope header") + } while (!line.startsWith(TAGLESS_ENVELOPE_HEADER)) + val properties = HashMap() + + line = "" + while (line.isBlank() || line.startsWith("#?")) { + if (line.startsWith("#?")) { + val match = propertyPattern.find(line) + ?: error("Line $line does not match property declaration pattern") + val (key, value) = match.destructured + properties[key] = value + } + line = readUTF8Line() ?: return SimpleEnvelope(Meta.empty, null) + } + + var meta: Meta = EmptyMeta + + if (line.startsWith(metaStart)) { + val metaFormat = properties[META_TYPE_PROPERTY]?.let { io.metaFormat(it) } ?: JsonMetaFormat.default + val metaSize = properties.get(META_LENGTH_PROPERTY)?.toInt() + meta = if (metaSize != null) { + val metaPacket = buildPacket { + writeFully(readBytes(metaSize)) + } + metaFormat.run { metaPacket.readObject() } + } else { + metaFormat.run { + readObject() + } + } + } + + do { + line = readUTF8Line() ?: return SimpleEnvelope(meta, null) + //returning an Envelope without data if end of input is reached + } while (!line.startsWith(dataStart)) + + val data: Binary? = if (properties.containsKey(DATA_LENGTH_PROPERTY)) { + val bytes = ByteArray(properties[DATA_LENGTH_PROPERTY]!!.toInt()) + readFully(bytes) + bytes.asBinary() + } else { + val bytes = readBytes() + bytes.asBinary() + } + + return SimpleEnvelope(meta, data) + } + + override fun Input.readPartial(): PartialEnvelope { + var offset = 0u + var line: String = "" + do { + line = readUTF8Line() ?: error("Input does not contain tagless envelope header") + offset += line.toUtf8Bytes().size.toUInt() + } while (!line.startsWith(TAGLESS_ENVELOPE_HEADER)) + val properties = HashMap() + + line = "" + while (line.isBlank() || line.startsWith("#?")) { + if (line.startsWith("#?")) { + val match = propertyPattern.find(line) + ?: error("Line $line does not match property declaration pattern") + val (key, value) = match.destructured + properties[key] = value + } + line = readUTF8Line() ?: return PartialEnvelope(Meta.empty, offset.toUInt(), 0.toULong()) + offset += line.toUtf8Bytes().size.toUInt() + } + + var meta: Meta = EmptyMeta + + if (line.startsWith(metaStart)) { + val metaFormat = properties[META_TYPE_PROPERTY]?.let { io.metaFormat(it) } ?: JsonMetaFormat.default + + val metaSize = properties.get(META_LENGTH_PROPERTY)?.toInt() + meta = if (metaSize != null) { + val metaPacket = buildPacket { + writeFully(readBytes(metaSize)) + } + offset += metaSize.toUInt() + metaFormat.run { metaPacket.readObject() } + } else { + error("Can't partially read an envelope with undefined meta size") + } + } + + do { + line = readUTF8Line() ?: return PartialEnvelope(Meta.empty, offset.toUInt(), 0.toULong()) + offset += line.toUtf8Bytes().size.toUInt() + //returning an Envelope without data if end of input is reached + } while (!line.startsWith(dataStart)) + + val dataSize = properties[DATA_LENGTH_PROPERTY]?.toULong() + return PartialEnvelope(meta, offset, dataSize) + } + + companion object : EnvelopeFormatFactory { + + private val propertyPattern = "#\\?\\s*(?[\\w.]*)\\s*:\\s*(?[^;]*);?".toRegex() + + const val META_TYPE_PROPERTY = "metaType" + const val META_LENGTH_PROPERTY = "metaLength" + const val DATA_LENGTH_PROPERTY = "dataLength" + + + const val TAGLESS_ENVELOPE_TYPE = "tagless" + + const val TAGLESS_ENVELOPE_HEADER = "#~DFTL~#" + const val META_START_PROPERTY = "metaSeparator" + const val DEFAULT_META_START = "#~META~#" + const val DATA_START_PROPERTY = "dataSeparator" + const val DEFAULT_DATA_START = "#~DATA~#" + + const val code: Int = 0x4446544c //DFTL + + override val name = TAGLESS_ENVELOPE_TYPE.asName() + + override fun invoke(meta: Meta, context: Context): EnvelopeFormat { + return TaglessEnvelopeFormat(context.io, meta) + } + + val default by lazy { invoke() } + + override fun peekFormat(io: IOPlugin, input: Input): EnvelopeFormat? { + return try { + val buffer = ByteArray(TAGLESS_ENVELOPE_HEADER.length) + input.readFully(buffer) + return if (buffer.toString() == TAGLESS_ENVELOPE_HEADER) { + TaglessEnvelopeFormat(io) + } else { + null + } + } catch (ex: Exception) { + null + } + } + } +} \ No newline at end of file diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/functions/FunctionServer.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/functions/FunctionServer.kt new file mode 100644 index 00000000..2eca605c --- /dev/null +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/functions/FunctionServer.kt @@ -0,0 +1,75 @@ +package hep.dataforge.io.functions + +import hep.dataforge.context.ContextAware +import hep.dataforge.io.IOFormat +import hep.dataforge.io.IOPlugin +import hep.dataforge.meta.Meta +import hep.dataforge.meta.get +import hep.dataforge.names.asName +import hep.dataforge.names.plus +import kotlin.reflect.KClass + + +/** + * A server that could produce asynchronous function values + */ +interface FunctionServer : ContextAware { + /** + * Call a function with given name and descriptor + */ + suspend fun call(meta: Meta, arg: T, inputType: KClass, outputType: KClass): R + + suspend fun callMany( + meta: Meta, + arg: List, + inputType: KClass, + outputType: KClass + ): List = List(arg.size) { + call(meta, arg[it], inputType, outputType) + } + + /** + * Get a generic suspended function with given name and descriptor + */ + fun function( + meta: Meta, + inputType: KClass, + outputType: KClass + ): (suspend (T) -> R) = { call(meta, it, inputType, outputType) } + + companion object { + const val FUNCTION_NAME_KEY = "function" + val FORMAT_KEY = "format".asName() + val INPUT_FORMAT_KEY = FORMAT_KEY + "input" + val OUTPUT_FORMAT_KEY = FORMAT_KEY + "output" + } +} + +suspend inline fun FunctionServer.call(meta: Meta, arg: T) = + call(meta, arg, T::class, R::class) + +suspend inline fun FunctionServer.callMany(meta: Meta, arg: List) = + callMany(meta, arg, T::class, R::class) + +inline fun FunctionServer.function(meta: Meta) = + function(meta, T::class, R::class) + +fun IOPlugin.getInputFormat(meta: Meta, type: KClass): IOFormat { + return meta[FunctionServer.INPUT_FORMAT_KEY]?.let { + resolveIOFormat(it, type) + } ?: error("Input format not resolved") +} + +fun IOPlugin.getOutputFormat(meta: Meta, type: KClass): IOFormat { + return meta[FunctionServer.OUTPUT_FORMAT_KEY]?.let { + resolveIOFormat(it, type) + } ?: error("Input format not resolved") +} + +inline fun IOPlugin.getInputFormat(meta: Meta): IOFormat = + getInputFormat(meta, T::class) + +inline fun IOPlugin.getOutputFormat(meta: Meta): IOFormat = + getOutputFormat(meta, R::class) + + diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/functions/RemoteFunctionClient.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/functions/RemoteFunctionClient.kt new file mode 100644 index 00000000..7c294891 --- /dev/null +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/functions/RemoteFunctionClient.kt @@ -0,0 +1,98 @@ +package hep.dataforge.io.functions + +import hep.dataforge.context.Context +import hep.dataforge.context.ContextAware +import hep.dataforge.io.* +import hep.dataforge.meta.Meta +import hep.dataforge.meta.get +import hep.dataforge.meta.int +import kotlin.reflect.KClass + +class RemoteFunctionClient(override val context: Context, val responder: Responder) : FunctionServer, ContextAware { + + private fun IOPlugin.encodeOne( + meta: Meta, + value: T, + valueType: KClass = value::class + ): Envelope = Envelope.invoke { + meta(meta) + type = REQUEST_TYPE + data { + val inputFormat: IOFormat = getInputFormat(meta, valueType) + inputFormat.run { + writeObject(value) + } + } + } + + private fun IOPlugin.encodeMany( + meta: Meta, + values: List, + valueType: KClass + ): Envelope = Envelope.invoke { + meta(meta) + type = REQUEST_TYPE + meta { + SIZE_KEY put values.size + } + data { + val inputFormat: IOFormat = getInputFormat(meta, valueType) + inputFormat.run { + values.forEach { + writeObject(it) + } + } + } + } + + private fun IOPlugin.decode(envelope: Envelope, valueType: KClass): List { + require(envelope.type == RESPONSE_TYPE) { "Unexpected message type: ${envelope.type}" } + val size = envelope.meta[SIZE_KEY].int ?: 1 + + return if (size == 0) { + emptyList() + } else { + val outputFormat: IOFormat = getOutputFormat(envelope.meta, valueType) + envelope.data?.read { + List(size) { + outputFormat.run { + readObject() + } + } + } ?: error("Message does not contain data") + } + } + + private val plugin by lazy { + context.plugins.load(IOPlugin) + } + + override suspend fun call( + meta: Meta, + arg: T, + inputType: KClass, + outputType: KClass + ): R = plugin.run { + val request = encodeOne(meta, arg) + val response = responder.respond(request) + return decode(response, outputType).first() + } + + override suspend fun callMany( + meta: Meta, + arg: List, + inputType: KClass, + outputType: KClass + ): List = plugin.run { + val request = encodeMany(meta, arg, inputType) + val response = responder.respond(request) + return decode(response, outputType) + } + + companion object { + const val REQUEST_TYPE = "function.request" + const val RESPONSE_TYPE = "function.response" + + const val SIZE_KEY = "size" + } +} \ No newline at end of file diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/functions/RemoteFunctionServer.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/functions/RemoteFunctionServer.kt new file mode 100644 index 00000000..8252b1d3 --- /dev/null +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/functions/RemoteFunctionServer.kt @@ -0,0 +1,58 @@ +package hep.dataforge.io.functions + +import hep.dataforge.context.Context +import hep.dataforge.context.ContextAware +import hep.dataforge.io.Envelope +import hep.dataforge.io.IOPlugin +import hep.dataforge.io.Responder +import hep.dataforge.io.type +import hep.dataforge.meta.get +import hep.dataforge.meta.int + +class RemoteFunctionServer( + override val context: Context, + val functionServer: FunctionServer +) : ContextAware, Responder { + + private val plugin by lazy { + context.plugins.load(IOPlugin) + } + + + override suspend fun respond(request: Envelope): Envelope { + require(request.type == RemoteFunctionClient.REQUEST_TYPE) { "Unexpected message type: ${request.type}" } + + val inputFormat = plugin.getInputFormat(request.meta) + val outputFormat = plugin.getOutputFormat(request.meta) + + val size = request.meta[RemoteFunctionClient.SIZE_KEY].int ?: 1 + + val input = request.data?.read { + inputFormat.run { + List(size) { + readObject() + } + } + } ?: error("Input is empty") + + val output = functionServer.callMany( + request.meta, + input + ) + + return Envelope.invoke { + meta { + meta(request.meta) + } + type = RemoteFunctionClient.RESPONSE_TYPE + data { + outputFormat.run { + output.forEach { + writeObject(it) + } + } + } + + } + } +} \ No newline at end of file diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/serialization/MetaSerializer.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/serialization/MetaSerializer.kt new file mode 100644 index 00000000..b22fed4a --- /dev/null +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/serialization/MetaSerializer.kt @@ -0,0 +1,141 @@ +package hep.dataforge.io.serialization + +import hep.dataforge.io.toJson +import hep.dataforge.io.toMeta +import hep.dataforge.meta.* +import hep.dataforge.names.NameToken +import hep.dataforge.values.* +import kotlinx.serialization.* +import kotlinx.serialization.internal.* +import kotlinx.serialization.json.JsonInput +import kotlinx.serialization.json.JsonObjectSerializer +import kotlinx.serialization.json.JsonOutput + + +@Serializer(Value::class) +object ValueSerializer : KSerializer { + private val valueTypeSerializer = EnumSerializer(ValueType::class) + private val listSerializer by lazy { ArrayListSerializer(ValueSerializer) } + + override val descriptor: SerialDescriptor = descriptor("hep.dataforge.values.Value") { + boolean("isList") + enum("valueType") + element("value", null) + } + + private fun Decoder.decodeValue(): Value { + return when (decode(valueTypeSerializer)) { + ValueType.NULL -> Null + ValueType.NUMBER -> decodeDouble().asValue() //TODO differentiate? + ValueType.BOOLEAN -> decodeBoolean().asValue() + ValueType.STRING -> decodeString().asValue() + else -> decodeString().parseValue() + } + } + + + override fun deserialize(decoder: Decoder): Value { + val isList = decoder.decodeBoolean() + return if (isList) { + listSerializer.deserialize(decoder).asValue() + } else { + decoder.decodeValue() + } + } + + private fun Encoder.encodeValue(value: Value) { + encode(valueTypeSerializer, value.type) + when (value.type) { + ValueType.NULL -> { + // do nothing + } + ValueType.NUMBER -> encodeDouble(value.double) + ValueType.BOOLEAN -> encodeBoolean(value.boolean) + ValueType.STRING -> encodeString(value.string) + else -> encodeString(value.string) + } + } + + override fun serialize(encoder: Encoder, obj: Value) { + encoder.encodeBoolean(obj.isList()) + if (obj.isList()) { + listSerializer.serialize(encoder, obj.list) + } else { + encoder.encodeValue(obj) + } + } +} + +@Serializer(MetaItem::class) +object MetaItemSerializer : KSerializer> { + override val descriptor: SerialDescriptor = descriptor("MetaItem") { + boolean("isNode") + element("value", null) + } + + + override fun deserialize(decoder: Decoder): MetaItem<*> { + val isNode = decoder.decodeBoolean() + return if (isNode) { + MetaItem.NodeItem(decoder.decode(MetaSerializer)) + } else { + MetaItem.ValueItem(decoder.decode(ValueSerializer)) + } + } + + override fun serialize(encoder: Encoder, obj: MetaItem<*>) { + encoder.encodeBoolean(obj is MetaItem.NodeItem) + when (obj) { + is MetaItem.NodeItem -> MetaSerializer.serialize(encoder, obj.node) + is MetaItem.ValueItem -> ValueSerializer.serialize(encoder, obj.value) + } + } +} + +private class DeserializedMeta(override val items: Map>) : MetaBase() + +/** + * Serialized for meta + */ +@Serializer(Meta::class) +object MetaSerializer : KSerializer { + private val mapSerializer = HashMapSerializer( + StringSerializer, + MetaItemSerializer + ) + + override val descriptor: SerialDescriptor = NamedMapClassDescriptor( + "hep.dataforge.meta.Meta", + StringSerializer.descriptor, + MetaItemSerializer.descriptor + ) + + override fun deserialize(decoder: Decoder): Meta { + return if (decoder is JsonInput) { + JsonObjectSerializer.deserialize(decoder).toMeta() + } else { + DeserializedMeta(mapSerializer.deserialize(decoder).mapKeys { NameToken(it.key) }) + } + } + + override fun serialize(encoder: Encoder, obj: Meta) { + if (encoder is JsonOutput) { + JsonObjectSerializer.serialize(encoder, obj.toJson()) + } else { + mapSerializer.serialize(encoder, obj.items.mapKeys { it.key.toString() }) + } + } +} + +@Serializer(Config::class) +object ConfigSerializer : KSerializer { + override val descriptor: SerialDescriptor = MetaSerializer.descriptor + + override fun deserialize(decoder: Decoder): Config { + return MetaSerializer.deserialize(decoder).toConfig() + } + + override fun serialize(encoder: Encoder, obj: Config) { + MetaSerializer.serialize(encoder, obj) + } +} \ No newline at end of file diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/serialization/nameSerializers.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/serialization/nameSerializers.kt new file mode 100644 index 00000000..c12a6c19 --- /dev/null +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/serialization/nameSerializers.kt @@ -0,0 +1,33 @@ +package hep.dataforge.io.serialization + +import hep.dataforge.names.Name +import hep.dataforge.names.NameToken +import hep.dataforge.names.toName +import kotlinx.serialization.* +import kotlinx.serialization.internal.StringDescriptor + +@Serializer(Name::class) +object NameSerializer : KSerializer { + override val descriptor: SerialDescriptor = StringDescriptor.withName("Name") + + override fun deserialize(decoder: Decoder): Name { + return decoder.decodeString().toName() + } + + override fun serialize(encoder: Encoder, obj: Name) { + encoder.encodeString(obj.toString()) + } +} + +@Serializer(NameToken::class) +object NameTokenSerializer : KSerializer { + override val descriptor: SerialDescriptor = StringDescriptor.withName("NameToken") + + override fun deserialize(decoder: Decoder): NameToken { + return decoder.decodeString().toName().first()!! + } + + override fun serialize(encoder: Encoder, obj: NameToken) { + encoder.encodeString(obj.toString()) + } +} \ No newline at end of file diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/serialization/serializationUtils.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/serialization/serializationUtils.kt new file mode 100644 index 00000000..09d17054 --- /dev/null +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/serialization/serializationUtils.kt @@ -0,0 +1,80 @@ +package hep.dataforge.io.serialization + +import kotlinx.serialization.CompositeDecoder +import kotlinx.serialization.Decoder +import kotlinx.serialization.KSerializer +import kotlinx.serialization.SerialDescriptor +import kotlinx.serialization.internal.* + +/** + * A convenience builder for serial descriptors + */ +inline class SerialDescriptorBuilder(private val impl: SerialClassDescImpl) { + fun element( + name: String, + descriptor: SerialDescriptor?, + isOptional: Boolean = false, + vararg annotations: Annotation + ) { + impl.addElement(name, isOptional) + descriptor?.let { impl.pushDescriptor(descriptor) } + annotations.forEach { + impl.pushAnnotation(it) + } + } + + fun element( + name: String, + isOptional: Boolean = false, + vararg annotations: Annotation, + block: SerialDescriptorBuilder.() -> Unit + ) { + impl.addElement(name, isOptional) + impl.pushDescriptor(SerialDescriptorBuilder(SerialClassDescImpl(name)).apply(block).build()) + annotations.forEach { + impl.pushAnnotation(it) + } + } + + fun boolean(name: String, isOptional: Boolean = false, vararg annotations: Annotation) = + element(name, BooleanDescriptor, isOptional, *annotations) + + fun string(name: String, isOptional: Boolean = false, vararg annotations: Annotation) = + element(name, StringDescriptor, isOptional, *annotations) + + fun int(name: String, isOptional: Boolean = false, vararg annotations: Annotation) = + element(name, IntDescriptor, isOptional, *annotations) + + fun double(name: String, isOptional: Boolean = false, vararg annotations: Annotation) = + element(name, DoubleDescriptor, isOptional, *annotations) + + fun float(name: String, isOptional: Boolean = false, vararg annotations: Annotation) = + element(name, FloatDescriptor, isOptional, *annotations) + + fun long(name: String, isOptional: Boolean = false, vararg annotations: Annotation) = + element(name, LongDescriptor, isOptional, *annotations) + + fun doubleArray(name: String, isOptional: Boolean = false, vararg annotations: Annotation) = + element(name, DoubleArraySerializer.descriptor, isOptional, *annotations) + + inline fun > enum(name: String, isOptional: Boolean = false, vararg annotations: Annotation) = + element(name, EnumSerializer(E::class).descriptor, isOptional, *annotations) + + fun classAnnotation(a: Annotation) = impl.pushClassAnnotation(a) + + fun build(): SerialDescriptor = impl +} + +inline fun KSerializer.descriptor( + name: String, + block: SerialDescriptorBuilder.() -> Unit +): SerialDescriptor = + SerialDescriptorBuilder(SerialClassDescImpl(name)).apply(block).build() + +fun Decoder.decodeStructure( + desc: SerialDescriptor, + vararg typeParams: KSerializer<*> = emptyArray(), + block: CompositeDecoder.() -> Unit +) { + beginStructure(desc, *typeParams).apply(block).endStructure(desc) +} \ No newline at end of file diff --git a/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/EnvelopeFormatTest.kt b/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/EnvelopeFormatTest.kt new file mode 100644 index 00000000..29e60f2f --- /dev/null +++ b/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/EnvelopeFormatTest.kt @@ -0,0 +1,46 @@ +package hep.dataforge.io + +import kotlin.test.Test +import kotlin.test.assertEquals + + +class EnvelopeFormatTest { + val envelope = Envelope.invoke { + type = "test.format" + meta{ + "d" put 22.2 + } + data{ + writeDouble(22.2) + } + } + + @ExperimentalStdlibApi + @Test + fun testTaggedFormat(){ + TaggedEnvelopeFormat.default.run { + val bytes = writeBytes(envelope) + println(bytes.decodeToString()) + val res = readBytes(bytes) + assertEquals(envelope.meta,res.meta) + val double = res.data?.read { + readDouble() + } + assertEquals(22.2, double) + } + } + + @Test + fun testTaglessFormat(){ + TaglessEnvelopeFormat.default.run { + val bytes = writeBytes(envelope) + println(bytes.decodeToString()) + val res = readBytes(bytes) + assertEquals(envelope.meta,res.meta) + val double = res.data?.read { + readDouble() + } + assertEquals(22.2, double) + } + } +} \ No newline at end of file diff --git a/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/MetaFormatTest.kt b/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/MetaFormatTest.kt index 16d946e3..02180dbc 100644 --- a/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/MetaFormatTest.kt +++ b/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/MetaFormatTest.kt @@ -11,11 +11,11 @@ class MetaFormatTest { @Test fun testBinaryMetaFormat() { val meta = buildMeta { - "a" to 22 - "node" to { - "b" to "DDD" - "c" to 11.1 - "array" to doubleArrayOf(1.0, 2.0, 3.0) + "a" put 22 + "node" put { + "b" put "DDD" + "c" put 11.1 + "array" put doubleArrayOf(1.0, 2.0, 3.0) } } val bytes = meta.toBytes(BinaryMetaFormat) @@ -26,11 +26,11 @@ class MetaFormatTest { @Test fun testJsonMetaFormat() { val meta = buildMeta { - "a" to 22 - "node" to { - "b" to "DDD" - "c" to 11.1 - "array" to doubleArrayOf(1.0, 2.0, 3.0) + "a" put 22 + "node" put { + "b" put "DDD" + "c" put 11.1 + "array" put doubleArrayOf(1.0, 2.0, 3.0) } } val string = meta.toString(JsonMetaFormat) diff --git a/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/MetaSerializerTest.kt b/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/MetaSerializerTest.kt index ea5854d2..7a8447c0 100644 --- a/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/MetaSerializerTest.kt +++ b/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/MetaSerializerTest.kt @@ -1,7 +1,13 @@ package hep.dataforge.io +import hep.dataforge.io.serialization.MetaItemSerializer +import hep.dataforge.io.serialization.MetaSerializer +import hep.dataforge.io.serialization.NameSerializer import hep.dataforge.meta.buildMeta import hep.dataforge.names.toName +import kotlinx.io.charsets.Charsets +import kotlinx.io.core.String +import kotlinx.serialization.cbor.Cbor import kotlinx.serialization.json.Json import kotlin.test.Test import kotlin.test.assertEquals @@ -10,11 +16,11 @@ class MetaSerializerTest { @Test fun testMetaSerialization() { val meta = buildMeta { - "a" to 22 - "node" to { - "b" to "DDD" - "c" to 11.1 - "array" to doubleArrayOf(1.0, 2.0, 3.0) + "a" put 22 + "node" put { + "b" put "DDD" + "c" put 11.1 + "array" put doubleArrayOf(1.0, 2.0, 3.0) } } @@ -23,6 +29,23 @@ class MetaSerializerTest { assertEquals(restored, meta) } + @Test + fun testCborSerialization() { + val meta = buildMeta { + "a" put 22 + "node" put { + "b" put "DDD" + "c" put 11.1 + "array" put doubleArrayOf(1.0, 2.0, 3.0) + } + } + + val bytes = Cbor.dump(MetaSerializer, meta) + println(String(bytes, charset = Charsets.ISO_8859_1)) + val restored = Cbor.load(MetaSerializer, bytes) + assertEquals(restored, meta) + } + @Test fun testNameSerialization() { val name = "a.b.c".toName() @@ -30,4 +53,9 @@ class MetaSerializerTest { val restored = Json.plain.parse(NameSerializer, string) assertEquals(restored, name) } + + @Test + fun testMetaItemDescriptor(){ + val descriptor = MetaItemSerializer.descriptor.getElementDescriptor(0) + } } \ No newline at end of file diff --git a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileBinary.kt b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileBinary.kt index 038281d4..aa90a638 100644 --- a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileBinary.kt +++ b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileBinary.kt @@ -1,21 +1,31 @@ package hep.dataforge.io -import kotlinx.io.core.ByteReadPacket import kotlinx.io.core.Input +import kotlinx.io.core.buildPacket import java.nio.channels.FileChannel import java.nio.file.Files import java.nio.file.Path import java.nio.file.StandardOpenOption +import kotlin.math.min @ExperimentalUnsignedTypes class FileBinary(val path: Path, private val offset: UInt = 0u, size: ULong? = null) : RandomAccessBinary { override val size: ULong = size ?: (Files.size(path).toULong() - offset).toULong() - override fun read(from: UInt, size: UInt, block: Input.() -> R): R { - FileChannel.open(path, StandardOpenOption.READ).use { - val buffer = it.map(FileChannel.MapMode.READ_ONLY, (from + offset).toLong(), size.toLong()) - return ByteReadPacket(buffer).block() + init { + if( size != null && Files.size(path) < offset.toLong() + size.toLong()){ + error("Can't read binary from file. File is to short.") } } -} \ No newline at end of file + + override fun read(from: UInt, size: UInt, block: Input.() -> R): R { + FileChannel.open(path, StandardOpenOption.READ).use { + val theSize: UInt = min(size, Files.size(path).toUInt() - offset) + val buffer = it.map(FileChannel.MapMode.READ_ONLY, (from + offset).toLong(), theSize.toLong()) + return buildPacket { writeFully(buffer) }.block() + } + } +} + +fun Path.asBinary(offset: UInt = 0u, size: ULong? = null): FileBinary = FileBinary(this, offset, size) \ No newline at end of file diff --git a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileEnvelope.kt b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileEnvelope.kt index 0c54012c..3187cd54 100644 --- a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileEnvelope.kt +++ b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileEnvelope.kt @@ -1,5 +1,6 @@ package hep.dataforge.io +import hep.dataforge.meta.EmptyMeta import hep.dataforge.meta.Meta import kotlinx.io.nio.asInput import kotlinx.io.nio.asOutput @@ -14,7 +15,7 @@ class FileEnvelope internal constructor(val path: Path, val format: EnvelopeForm init { val input = Files.newByteChannel(path, StandardOpenOption.READ).asInput() - partialEnvelope = format.run { input.readPartial() } + partialEnvelope = format.run { input.use { it.readPartial()} } } override val meta: Meta get() = partialEnvelope.meta @@ -22,22 +23,30 @@ class FileEnvelope internal constructor(val path: Path, val format: EnvelopeForm override val data: Binary? = FileBinary(path, partialEnvelope.dataOffset, partialEnvelope.dataSize) } -fun Path.readEnvelope(format: EnvelopeFormat) = FileEnvelope(this, format) +fun IOPlugin.readEnvelopeFile( + path: Path, + formatFactory: EnvelopeFormatFactory = TaggedEnvelopeFormat, + formatMeta: Meta = EmptyMeta +): FileEnvelope { + val format = formatFactory(formatMeta, context) + return FileEnvelope(path, format) +} -fun Path.writeEnvelope( +fun IOPlugin.writeEnvelopeFile( + path: Path, envelope: Envelope, - format: EnvelopeFormat = TaggedEnvelopeFormat, - metaFormat: MetaFormat = JsonMetaFormat + formatFactory: EnvelopeFormatFactory = TaggedEnvelopeFormat, + formatMeta: Meta = EmptyMeta ) { val output = Files.newByteChannel( - this, + path, StandardOpenOption.WRITE, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING ).asOutput() - with(format) { - output.writeEnvelope(envelope, metaFormat) + with(formatFactory(formatMeta, context)) { + output.writeObject(envelope) } } diff --git a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/ioFormatsJVM.kt b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/ioFormatsJVM.kt new file mode 100644 index 00000000..c926d07a --- /dev/null +++ b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/ioFormatsJVM.kt @@ -0,0 +1,51 @@ +package hep.dataforge.io + +import hep.dataforge.descriptors.NodeDescriptor +import hep.dataforge.io.functions.FunctionServer +import hep.dataforge.io.functions.FunctionServer.Companion.FUNCTION_NAME_KEY +import hep.dataforge.io.functions.FunctionServer.Companion.INPUT_FORMAT_KEY +import hep.dataforge.io.functions.FunctionServer.Companion.OUTPUT_FORMAT_KEY +import hep.dataforge.io.functions.function +import hep.dataforge.meta.Meta +import hep.dataforge.meta.buildMeta +import hep.dataforge.names.Name +import kotlinx.io.nio.asOutput +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.StandardOpenOption +import kotlin.reflect.KClass +import kotlin.reflect.full.isSuperclassOf + +inline fun IOPlugin.resolveIOFormat(): IOFormat? { + return ioFormats.values.find { it.type.isSuperclassOf(T::class) } as IOFormat? +} + +fun IOPlugin.resolveIOFormatName(type: KClass<*>): Name { + return ioFormats.entries.find { it.value.type.isSuperclassOf(type) }?.key + ?: error("Can't resolve IOFormat for type $type") +} + +inline fun IOPlugin.generateFunctionMeta(functionName: String): Meta = buildMeta { + FUNCTION_NAME_KEY put functionName + INPUT_FORMAT_KEY put resolveIOFormatName(T::class).toString() + OUTPUT_FORMAT_KEY put resolveIOFormatName(R::class).toString() +} + +inline fun FunctionServer.function( + functionName: String +): (suspend (T) -> R) { + val plugin = context.plugins.get() ?: error("IO plugin not loaded") + val meta = plugin.generateFunctionMeta(functionName) + return function(meta) +} + +/** + * Write meta to file in a given [format] + */ +fun Meta.write(path: Path, format: MetaFormat, descriptor: NodeDescriptor? = null) { + format.run { + Files.newByteChannel(path, StandardOpenOption.WRITE, StandardOpenOption.CREATE_NEW) + .asOutput() + .writeMeta(this@write, descriptor) + } +} \ No newline at end of file diff --git a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/tcp/EnvelopeClient.kt b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/tcp/EnvelopeClient.kt new file mode 100644 index 00000000..b6b85101 --- /dev/null +++ b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/tcp/EnvelopeClient.kt @@ -0,0 +1,67 @@ +package hep.dataforge.io.tcp + +import hep.dataforge.context.Context +import hep.dataforge.context.ContextAware +import hep.dataforge.io.* +import hep.dataforge.meta.EmptyMeta +import hep.dataforge.meta.Meta +import kotlinx.coroutines.asCoroutineDispatcher +import kotlinx.coroutines.withContext +import kotlinx.io.streams.writePacket +import java.net.Socket +import java.util.concurrent.Executors +import kotlin.time.ExperimentalTime + +@ExperimentalTime +class EnvelopeClient( + override val context: Context, + val host: String, + val port: Int, + formatFactory: EnvelopeFormatFactory = TaggedEnvelopeFormat, + formatMeta: Meta = EmptyMeta +) : Responder, ContextAware { + + private val dispatcher = Executors.newSingleThreadExecutor().asCoroutineDispatcher() + + private val format = formatFactory(formatMeta, context = context) + +// private var socket: SocketChannel? = null +// +// private fun getSocket(): Socket { +// val socket = socket ?: Socket(host, port).also { this.socket = it } +// return if (socket.isConnected) { +// socket +// } else { +// Socket(host, port).also { this.socket = it } +// } +// } + + suspend fun close() { + try { + respond( + Envelope.invoke { + type = EnvelopeServer.SHUTDOWN_ENVELOPE_TYPE + } + ) + } catch (ex: Exception) { + logger.error { ex } + } + } + + + override suspend fun respond(request: Envelope): Envelope = withContext(dispatcher) { + //val address = InetSocketAddress(host,port) + val socket = Socket(host, port) + val input = socket.getInputStream().asInput() + val output = socket.getOutputStream() + format.run { + output.writePacket { + writeObject(request) + } + logger.debug { "Sent request with type ${request.type} to ${socket.remoteSocketAddress}" } + val res = input.readObject() + logger.debug { "Received response with type ${res.type} from ${socket.remoteSocketAddress}" } + return@withContext res + } + } +} \ No newline at end of file diff --git a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/tcp/EnvelopeServer.kt b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/tcp/EnvelopeServer.kt new file mode 100644 index 00000000..b733aedd --- /dev/null +++ b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/tcp/EnvelopeServer.kt @@ -0,0 +1,102 @@ +package hep.dataforge.io.tcp + +import hep.dataforge.context.Context +import hep.dataforge.context.ContextAware +import hep.dataforge.io.EnvelopeFormatFactory +import hep.dataforge.io.Responder +import hep.dataforge.io.TaggedEnvelopeFormat +import hep.dataforge.io.type +import hep.dataforge.meta.EmptyMeta +import hep.dataforge.meta.Meta +import kotlinx.coroutines.* +import kotlinx.io.streams.writePacket +import java.net.ServerSocket +import java.net.Socket +import kotlin.concurrent.thread + +class EnvelopeServer( + override val context: Context, + val port: Int, + val responder: Responder, + val scope: CoroutineScope, + formatFactory: EnvelopeFormatFactory = TaggedEnvelopeFormat, + formatMeta: Meta = EmptyMeta +) : ContextAware { + + private var job: Job? = null + + private val format = formatFactory(formatMeta, context = context) + + fun start() { + if (job == null) { + logger.info { "Starting envelope server on port $port" } + val job = scope.launch(Dispatchers.IO) { + val serverSocket = ServerSocket(port) + //TODO add handshake and format negotiation + while (isActive && !serverSocket.isClosed) { + val socket = serverSocket.accept() + logger.info { "Accepted connection from ${socket.remoteSocketAddress}" } + readSocket(socket) + } + } + } + } + + fun stop() { + logger.info { "Stopping envelope server on port $port" } + job?.cancel() + job = null + } + +// private fun CoroutineScope.readSocket(socket: Socket) { +// launch(Dispatchers.IO) { +// val input = socket.getInputStream().asInput() +// val output = socket.getOutputStream().asOutput() +// format.run { +// while (isActive && socket.isConnected) { +// val request = input.readThis() +// logger.debug { "Accepted request with type ${request.type} from ${socket.remoteSocketAddress}" } +// if (request.type == SHUTDOWN_ENVELOPE_TYPE) { +// //Echo shutdown command +// logger.info { "Accepted graceful shutdown signal from ${socket.inetAddress}" } +// socket.close() +// cancel("Graceful connection shutdown requested by client") +// } +// val response = responder.respond(request) +// output.writeThis(response) +// } +// } +// } +// } + + private fun readSocket(socket: Socket) { + thread { + val input = socket.getInputStream().asInput() + val outputStream = socket.getOutputStream() + format.run { + while (socket.isConnected) { + val request = input.readObject() + logger.debug { "Accepted request with type ${request.type} from ${socket.remoteSocketAddress}" } + if (request.type == SHUTDOWN_ENVELOPE_TYPE) { + //Echo shutdown command + logger.info { "Accepted graceful shutdown signal from ${socket.inetAddress}" } + socket.close() + return@thread +// cancel("Graceful connection shutdown requested by client") + } + runBlocking { + val response = responder.respond(request) + outputStream.writePacket { + writeObject(response) + } + logger.debug { "Sent response with type ${response.type} to ${socket.remoteSocketAddress}" } + } + } + } + } + } + + companion object { + const val SHUTDOWN_ENVELOPE_TYPE = "@shutdown" + } +} \ No newline at end of file diff --git a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/tcp/InputStreamAsInput.kt b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/tcp/InputStreamAsInput.kt new file mode 100644 index 00000000..1c711be0 --- /dev/null +++ b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/tcp/InputStreamAsInput.kt @@ -0,0 +1,33 @@ +package hep.dataforge.io.tcp + +import kotlinx.io.core.AbstractInput +import kotlinx.io.core.Input +import kotlinx.io.core.IoBuffer +import kotlinx.io.core.IoBuffer.Companion.NoPool +import kotlinx.io.core.writePacket +import kotlinx.io.streams.readPacketAtMost +import java.io.InputStream + +/** + * Modified version of InputStream to Input converter that supports waiting for input + */ +internal class InputStreamAsInput( + private val stream: InputStream +) : AbstractInput(pool = NoPool) { + + + override fun fill(): IoBuffer? { + val packet = stream.readPacketAtMost(4096) + return pool.borrow().apply { + resetForWrite(4096) + writePacket(packet) + } + } + + override fun closeSource() { + stream.close() + } +} + +fun InputStream.asInput(): Input = + InputStreamAsInput(this) diff --git a/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/FileBinaryTest.kt b/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/FileBinaryTest.kt new file mode 100644 index 00000000..94403dcd --- /dev/null +++ b/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/FileBinaryTest.kt @@ -0,0 +1,56 @@ +package hep.dataforge.io + +import hep.dataforge.context.Global +import java.nio.file.Files +import kotlin.test.Test +import kotlin.test.assertEquals + +class FileBinaryTest { + val envelope = Envelope { + meta { + "a" put "AAA" + "b" put 22.2 + } + dataType = "hep.dataforge.test" + dataID = "myData" // добавил только что + data { + writeDouble(16.7) + } + } + + @Test + fun testSize() { + val binary = envelope.data + assertEquals(binary?.size?.toInt(), binary?.toBytes()?.size) + } + + @Test + fun testFileData(){ + val dataFile = Files.createTempFile("dataforge_test_bin", ".bin") + dataFile.toFile().writeText("This is my binary") + val envelopeFromFile = Envelope { + meta { + "a" put "AAA" + "b" put 22.2 + } + dataType = "hep.dataforge.satellite" + dataID = "cellDepositTest" // добавил только что + data = dataFile.asBinary() + } + val binary = envelopeFromFile.data!! + println(binary.toBytes().size) + assertEquals(binary.size.toInt(), binary.toBytes().size) + + } + + @Test + fun testFileDataSizeRewriting() { + println(System.getProperty("user.dir")) + val tmpPath = Files.createTempFile("dataforge_test", ".df") + Global.io.writeEnvelopeFile(tmpPath, envelope) + + val binary = Global.io.readEnvelopeFile(tmpPath).data!! + assertEquals(binary.size.toInt(), binary.toBytes().size) + } + +} \ No newline at end of file diff --git a/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/FileEnvelopeTest.kt b/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/FileEnvelopeTest.kt new file mode 100644 index 00000000..ba7f7cc5 --- /dev/null +++ b/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/FileEnvelopeTest.kt @@ -0,0 +1,31 @@ +package hep.dataforge.io + +import hep.dataforge.context.Global +import java.nio.file.Files +import kotlin.test.Test +import kotlin.test.assertTrue + + +class FileEnvelopeTest { + val envelope = Envelope { + meta { + "a" put "AAA" + "b" put 22.2 + } + dataType = "hep.dataforge.test" + dataID = "myData" // добавил только что + data { + writeDouble(16.7) + + } + } + + @Test + fun testFileWriteRead() { + val tmpPath = Files.createTempFile("dataforge_test", ".df") + Global.io.writeEnvelopeFile(tmpPath,envelope) + println(tmpPath.toUri()) + val restored: Envelope = Global.io.readEnvelopeFile(tmpPath) + assertTrue { envelope.contentEquals(restored) } + } +} \ No newline at end of file diff --git a/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/tcp/EnvelopeServerTest.kt b/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/tcp/EnvelopeServerTest.kt new file mode 100644 index 00000000..37c35efc --- /dev/null +++ b/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/tcp/EnvelopeServerTest.kt @@ -0,0 +1,67 @@ +package hep.dataforge.io.tcp + +import hep.dataforge.context.Global +import hep.dataforge.io.Envelope +import hep.dataforge.io.Responder +import hep.dataforge.io.TaggedEnvelopeFormat +import hep.dataforge.io.writeBytes +import kotlinx.coroutines.GlobalScope +import kotlinx.coroutines.runBlocking +import org.junit.AfterClass +import org.junit.BeforeClass +import org.junit.Test +import kotlin.test.assertEquals +import kotlin.time.ExperimentalTime + +@ExperimentalStdlibApi +object EchoResponder : Responder { + override suspend fun respond(request: Envelope): Envelope { + val string = TaggedEnvelopeFormat().run { writeBytes(request).decodeToString() } + println("ECHO:") + println(string) + return request + } +} + +@ExperimentalTime +@ExperimentalStdlibApi +class EnvelopeServerTest { + companion object { + @JvmStatic + val echoEnvelopeServer = EnvelopeServer(Global, 7778, EchoResponder, GlobalScope) + + @BeforeClass + @JvmStatic + fun start() { + echoEnvelopeServer.start() + } + + @AfterClass + @JvmStatic + fun close() { + echoEnvelopeServer.stop() + } + } + + @Test + fun doEchoTest() { + val request = Envelope.invoke { + type = "test.echo" + meta { + "test.value" put 22 + } + data { + writeDouble(22.7) + } + } + val client = EnvelopeClient(Global, host = "localhost", port = 7778) + runBlocking { + val response = client.respond(request) + + + assertEquals(request.meta, response.meta) +// assertEquals(request.data?.toBytes()?.decodeToString(), response.data?.toBytes()?.decodeToString()) + client.close() + } + } +} \ No newline at end of file diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/descriptors/ItemDescriptor.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/descriptors/ItemDescriptor.kt index 09e4d11e..0658dfd8 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/descriptors/ItemDescriptor.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/descriptors/ItemDescriptor.kt @@ -32,13 +32,11 @@ sealed class ItemDescriptor(override val config: Config) : Specific { var info: String? by string() /** - * A list of tags for this item. Tags used to customize item usage + * Additional attributes of an item. For example validation and widget parameters * * @return */ - var tags: List by value { value -> - value?.list?.map { it.string } ?: emptyList() - } + var attributes by node() /** * True if the item is required @@ -54,7 +52,7 @@ sealed class ItemDescriptor(override val config: Config) : Specific { * * @author Alexander Nozik */ -class NodeDescriptor(config: Config) : ItemDescriptor(config){ +class NodeDescriptor(config: Config) : ItemDescriptor(config) { /** * True if the node is required @@ -68,18 +66,18 @@ class NodeDescriptor(config: Config) : ItemDescriptor(config){ * * @return */ - var default: Meta? by node() + var default: Config? by node() /** * The list of value descriptors */ val values: Map - get() = config.getAll(VALUE_KEY.toName()).entries.associate { (name, node) -> + get() = config.getIndexed(VALUE_KEY.toName()).entries.associate { (name, node) -> name to ValueDescriptor.wrap(node.node ?: error("Value descriptor must be a node")) } fun value(name: String, descriptor: ValueDescriptor) { - if(items.keys.contains(name)) error("The key $name already exists in descriptor") + if (items.keys.contains(name)) error("The key $name already exists in descriptor") val token = NameToken(VALUE_KEY, name) config[token] = descriptor.config } @@ -95,13 +93,13 @@ class NodeDescriptor(config: Config) : ItemDescriptor(config){ * The map of children node descriptors */ val nodes: Map - get() = config.getAll(NODE_KEY.toName()).entries.associate { (name, node) -> + get() = config.getIndexed(NODE_KEY.toName()).entries.associate { (name, node) -> name to wrap(node.node ?: error("Node descriptor must be a node")) } fun node(name: String, descriptor: NodeDescriptor) { - if(items.keys.contains(name)) error("The key $name already exists in descriptor") + if (items.keys.contains(name)) error("The key $name already exists in descriptor") val token = NameToken(NODE_KEY, name) config[token] = descriptor.config } @@ -117,7 +115,7 @@ class NodeDescriptor(config: Config) : ItemDescriptor(config){ companion object : Specification { -// const val ITEM_KEY = "item" + // const val ITEM_KEY = "item" const val NODE_KEY = "node" const val VALUE_KEY = "value" @@ -135,7 +133,7 @@ class NodeDescriptor(config: Config) : ItemDescriptor(config){ * * @author Alexander Nozik */ -class ValueDescriptor(config: Config) : ItemDescriptor(config){ +class ValueDescriptor(config: Config) : ItemDescriptor(config) { /** diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Laminate.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Laminate.kt index b76fd46f..b403544c 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Laminate.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Laminate.kt @@ -4,8 +4,6 @@ import hep.dataforge.names.NameToken /** * A meta laminate consisting of multiple immutable meta layers. For mutable front layer, use [Styled]. - * - * */ class Laminate(layers: List) : MetaBase() { @@ -63,7 +61,7 @@ class Laminate(layers: List) : MetaBase() { } else -> map { when (it) { - is MetaItem.ValueItem -> MetaItem.NodeItem(buildMeta { Meta.VALUE_KEY to it.value }) + is MetaItem.ValueItem -> MetaItem.NodeItem(buildMeta { Meta.VALUE_KEY put it.value }) is MetaItem.NodeItem -> it } }.merge() diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Meta.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Meta.kt index 9e0dedf9..e5aa0c45 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Meta.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Meta.kt @@ -59,6 +59,8 @@ interface Meta : MetaRepr { * A key for single value node */ const val VALUE_KEY = "@value" + + val empty: EmptyMeta = EmptyMeta } } @@ -78,27 +80,6 @@ operator fun Meta?.get(name: Name): MetaItem<*>? { operator fun Meta?.get(token: NameToken): MetaItem<*>? = this?.items?.get(token) operator fun Meta?.get(key: String): MetaItem<*>? = get(key.toName()) -/** - * Get all items matching given name. - */ -fun Meta.getAll(name: Name): Map> { - val root = when (name.length) { - 0 -> error("Can't use empty name for that") - 1 -> this - else -> (this[name.cutLast()] as? NodeItem<*>)?.node - } - - val (body, index) = name.last()!! - val regex = index.toRegex() - - return root?.items - ?.filter { it.key.body == body && (index.isEmpty() || regex.matches(it.key.index)) } - ?.mapKeys { it.key.index } - ?: emptyMap() -} - -fun Meta.getAll(name: String): Map> = getAll(name.toName()) - /** * Get a sequence of [Name]-[Value] pairs */ @@ -136,27 +117,6 @@ interface MetaNode> : Meta { override val items: Map> } -/** - * Get all items matching given name. - */ -fun > M.getAll(name: Name): Map> { - val root: MetaNode? = when (name.length) { - 0 -> error("Can't use empty name for that") - 1 -> this - else -> (this[name.cutLast()] as? NodeItem)?.node - } - - val (body, index) = name.last()!! - val regex = index.toRegex() - - return root?.items - ?.filter { it.key.body == body && (index.isEmpty() || regex.matches(it.key.index)) } - ?.mapKeys { it.key.index } - ?: emptyMap() -} - -fun > M.getAll(name: String): Map> = getAll(name.toName()) - operator fun > MetaNode?.get(name: Name): MetaItem? { if (this == null) return null return name.first()?.let { token -> @@ -183,20 +143,15 @@ operator fun > MetaNode?.get(key: NameToken): MetaItem? = /** * Equals, hashcode and to string for any meta */ -abstract class MetaBase: Meta{ +abstract class MetaBase : Meta { - override fun equals(other: Any?): Boolean = if(other is Meta) { + override fun equals(other: Any?): Boolean = if (other is Meta) { this.items == other.items -// val items = items -// val otherItems = other.items -// (items.keys == otherItems.keys) && items.keys.all { -// items[it] == otherItems[it] -// } } else { false } - override fun hashCode(): Int = items.hashCode() + override fun hashCode(): Int = items.hashCode() override fun toString(): String = items.toString() } @@ -232,8 +187,7 @@ object EmptyMeta : MetaBase() { /** * Unsafe methods to access values and nodes directly from [MetaItem] */ - -val MetaItem<*>?.value +val MetaItem<*>?.value: Value? get() = (this as? ValueItem)?.value ?: (this?.node?.get(VALUE_KEY) as? ValueItem)?.value @@ -247,7 +201,7 @@ val MetaItem<*>?.long get() = number?.toLong() val MetaItem<*>?.short get() = number?.toShort() inline fun > MetaItem<*>?.enum() = if (this is ValueItem && this.value is EnumValue<*>) { - this.value as E + this.value.value as E } else { string?.let { enumValueOf(it) } } @@ -257,17 +211,8 @@ val MetaItem<*>?.stringList get() = value?.list?.map { it.string } ?: emptyList( val MetaItem?.node: M? get() = when (this) { null -> null - is ValueItem -> error("Trying to interpret value meta item as node item") + is ValueItem -> null//error("Trying to interpret value meta item as node item") is NodeItem -> node } -/** - * Generic meta-holder object - */ -interface Metoid { - val meta: Meta -} - -fun Value.toMeta() = buildMeta { Meta.VALUE_KEY to this } - -fun Meta.isEmpty() = this === EmptyMeta || this.items.isEmpty() +fun Meta.isEmpty() = this === EmptyMeta || this.items.isEmpty() \ No newline at end of file diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MetaBuilder.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MetaBuilder.kt index 322b660a..c5ad3831 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MetaBuilder.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MetaBuilder.kt @@ -2,50 +2,118 @@ package hep.dataforge.meta import hep.dataforge.names.Name import hep.dataforge.names.asName +import hep.dataforge.values.EnumValue import hep.dataforge.values.Value +import hep.dataforge.values.asValue +import kotlin.jvm.JvmName /** * DSL builder for meta. Is not intended to store mutable state */ +@DFBuilder class MetaBuilder : AbstractMutableMeta() { override fun wrapNode(meta: Meta): MetaBuilder = if (meta is MetaBuilder) meta else meta.builder() override fun empty(): MetaBuilder = MetaBuilder() - infix fun String.to(value: Any) { - if (value is Meta) { - this@MetaBuilder[this] = value - } - this@MetaBuilder[this] = Value.of(value) + infix fun String.put(value: Value?) { + set(this, value) } - infix fun String.to(meta: Meta) { + infix fun String.put(string: String?) { + set(this, string?.asValue()) + } + + infix fun String.put(number: Number?) { + set(this, number?.asValue()) + } + + infix fun String.put(boolean: Boolean?) { + set(this, boolean?.asValue()) + } + + infix fun String.put(enum: Enum<*>) { + set(this, EnumValue(enum)) + } + + @JvmName("putValues") + infix fun String.put(iterable: Iterable) { + set(this, iterable.asValue()) + } + + @JvmName("putNumbers") + infix fun String.put(iterable: Iterable) { + set(this, iterable.map { it.asValue() }.asValue()) + } + + @JvmName("putStrings") + infix fun String.put(iterable: Iterable) { + set(this, iterable.map { it.asValue() }.asValue()) + } + + infix fun String.put(array: DoubleArray) { + set(this, array.asValue()) + } + + infix fun String.putValue(any: Any?) { + set(this, Value.of(any)) + } + + infix fun String.put(meta: Meta?) { this@MetaBuilder[this] = meta } - infix fun String.to(value: Iterable) { + infix fun String.put(repr: MetaRepr?) { + set(this, repr?.toMeta()) + } + + @JvmName("putMetas") + infix fun String.put(value: Iterable) { this@MetaBuilder[this] = value.toList() } - infix fun String.to(metaBuilder: MetaBuilder.() -> Unit) { + infix fun String.put(metaBuilder: MetaBuilder.() -> Unit) { this@MetaBuilder[this] = MetaBuilder().apply(metaBuilder) } - infix fun Name.to(value: Any) { - if (value is Meta) { - this@MetaBuilder[this] = value - } - this@MetaBuilder[this] = Value.of(value) + infix fun Name.put(value: Value?) { + set(this, value) } - infix fun Name.to(meta: Meta) { + infix fun Name.put(string: String?) { + set(this, string?.asValue()) + } + + infix fun Name.put(number: Number?) { + set(this, number?.asValue()) + } + + infix fun Name.put(boolean: Boolean?) { + set(this, boolean?.asValue()) + } + + infix fun Name.put(enum: Enum<*>) { + set(this, EnumValue(enum)) + } + + @JvmName("putValues") + infix fun Name.put(iterable: Iterable) { + set(this, iterable.asValue()) + } + + infix fun Name.put(meta: Meta?) { this@MetaBuilder[this] = meta } - infix fun Name.to(value: Iterable) { + infix fun Name.put(repr: MetaRepr?) { + set(this, repr?.toMeta()) + } + + @JvmName("putMetas") + infix fun Name.put(value: Iterable) { this@MetaBuilder[this] = value.toList() } - infix fun Name.to(metaBuilder: MetaBuilder.() -> Unit) { + infix fun Name.put(metaBuilder: MetaBuilder.() -> Unit) { this@MetaBuilder[this] = MetaBuilder().apply(metaBuilder) } } @@ -65,6 +133,11 @@ fun Meta.builder(): MetaBuilder { } } +/** + * Create a deep copy of this meta and apply builder to it + */ +fun Meta.edit(builder: MetaBuilder.() -> Unit): MetaBuilder = builder().apply(builder) + /** * Build a [MetaBuilder] using given transformation */ diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MutableMeta.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MutableMeta.kt index 285edd89..4da08ce4 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MutableMeta.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MutableMeta.kt @@ -164,7 +164,7 @@ fun MutableMeta<*>.append(name: Name, value: Any?) { if (newIndex.isNotEmpty()) { set(name, value) } else { - val index = (getAll(name).keys.mapNotNull { it.toIntOrNull() }.max() ?: -1) + 1 + val index = (getIndexed(name).keys.mapNotNull { it.toIntOrNull() }.max() ?: -1) + 1 set(name.withIndex(index.toString()), value) } } diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Specific.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Specific.kt index a89a44a1..d3ec418a 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Specific.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Specific.kt @@ -1,5 +1,8 @@ package hep.dataforge.meta +import hep.dataforge.names.Name +import kotlin.jvm.JvmName + /** * Marker interface for classes with specifications */ @@ -61,5 +64,10 @@ fun > S.createStyle(action: C.() -> Unit): Me fun Specific.spec( spec: Specification, - key: String? = null -): MutableMorphDelegate = MutableMorphDelegate(config, key) { spec.wrap(it) } \ No newline at end of file + key: Name? = null +): MutableMorphDelegate = MutableMorphDelegate(config, key) { spec.wrap(it) } + +fun MetaItem<*>.spec(spec: Specification): T? = node?.let { spec.wrap(it)} + +@JvmName("configSpec") +fun MetaItem.spec(spec: Specification): T? = node?.let { spec.wrap(it)} \ No newline at end of file diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/annotations.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/annotations.kt new file mode 100644 index 00000000..865ef800 --- /dev/null +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/annotations.kt @@ -0,0 +1,11 @@ +package hep.dataforge.meta + +/** + * General marker for dataforge builders + */ +@DslMarker +annotation class DFBuilder + +@Experimental(level = Experimental.Level.WARNING) +@Retention(AnnotationRetention.BINARY) +annotation class DFExperimental \ No newline at end of file diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/configDelegates.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/configDelegates.kt index 6871b6e3..34fa0e71 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/configDelegates.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/configDelegates.kt @@ -1,5 +1,6 @@ package hep.dataforge.meta +import hep.dataforge.names.Name import hep.dataforge.values.DoubleArrayValue import hep.dataforge.values.Null import hep.dataforge.values.Value @@ -11,126 +12,126 @@ import kotlin.jvm.JvmName /** * A property delegate that uses custom key */ -fun Configurable.value(default: Any = Null, key: String? = null): MutableValueDelegate = +fun Configurable.value(default: Any = Null, key: Name? = null): MutableValueDelegate = MutableValueDelegate(config, key, Value.of(default)) fun Configurable.value( default: T? = null, - key: String? = null, + key: Name? = null, writer: (T) -> Value = { Value.of(it) }, reader: (Value?) -> T ): ReadWriteDelegateWrapper = MutableValueDelegate(config, key, default?.let { Value.of(it) }).transform(reader = reader, writer = writer) -fun Configurable.string(default: String? = null, key: String? = null): MutableStringDelegate = +fun Configurable.string(default: String? = null, key: Name? = null): MutableStringDelegate = MutableStringDelegate(config, key, default) -fun Configurable.boolean(default: Boolean? = null, key: String? = null): MutableBooleanDelegate = +fun Configurable.boolean(default: Boolean? = null, key: Name? = null): MutableBooleanDelegate = MutableBooleanDelegate(config, key, default) -fun Configurable.number(default: Number? = null, key: String? = null): MutableNumberDelegate = +fun Configurable.number(default: Number? = null, key: Name? = null): MutableNumberDelegate = MutableNumberDelegate(config, key, default) /* Number delegates*/ -fun Configurable.int(default: Int? = null, key: String? = null) = +fun Configurable.int(default: Int? = null, key: Name? = null) = number(default, key).int -fun Configurable.double(default: Double? = null, key: String? = null) = +fun Configurable.double(default: Double? = null, key: Name? = null) = number(default, key).double -fun Configurable.long(default: Long? = null, key: String? = null) = +fun Configurable.long(default: Long? = null, key: Name? = null) = number(default, key).long -fun Configurable.short(default: Short? = null, key: String? = null) = +fun Configurable.short(default: Short? = null, key: Name? = null) = number(default, key).short -fun Configurable.float(default: Float? = null, key: String? = null) = +fun Configurable.float(default: Float? = null, key: Name? = null) = number(default, key).float @JvmName("safeString") -fun Configurable.string(default: String, key: String? = null) = +fun Configurable.string(default: String, key: Name? = null) = MutableSafeStringDelegate(config, key) { default } @JvmName("safeBoolean") -fun Configurable.boolean(default: Boolean, key: String? = null) = +fun Configurable.boolean(default: Boolean, key: Name? = null) = MutableSafeBooleanDelegate(config, key) { default } @JvmName("safeNumber") -fun Configurable.number(default: Number, key: String? = null) = +fun Configurable.number(default: Number, key: Name? = null) = MutableSafeNumberDelegate(config, key) { default } @JvmName("safeString") -fun Configurable.string(key: String? = null, default: () -> String) = +fun Configurable.string(key: Name? = null, default: () -> String) = MutableSafeStringDelegate(config, key, default) @JvmName("safeBoolean") -fun Configurable.boolean(key: String? = null, default: () -> Boolean) = +fun Configurable.boolean(key: Name? = null, default: () -> Boolean) = MutableSafeBooleanDelegate(config, key, default) @JvmName("safeNumber") -fun Configurable.number(key: String? = null, default: () -> Number) = +fun Configurable.number(key: Name? = null, default: () -> Number) = MutableSafeNumberDelegate(config, key, default) /* Safe number delegates*/ @JvmName("safeInt") -fun Configurable.int(default: Int, key: String? = null) = +fun Configurable.int(default: Int, key: Name? = null) = number(default, key).int @JvmName("safeDouble") -fun Configurable.double(default: Double, key: String? = null) = +fun Configurable.double(default: Double, key: Name? = null) = number(default, key).double @JvmName("safeLong") -fun Configurable.long(default: Long, key: String? = null) = +fun Configurable.long(default: Long, key: Name? = null) = number(default, key).long @JvmName("safeShort") -fun Configurable.short(default: Short, key: String? = null) = +fun Configurable.short(default: Short, key: Name? = null) = number(default, key).short @JvmName("safeFloat") -fun Configurable.float(default: Float, key: String? = null) = +fun Configurable.float(default: Float, key: Name? = null) = number(default, key).float /** * Enum delegate */ -inline fun > Configurable.enum(default: E, key: String? = null) = +inline fun > Configurable.enum(default: E, key: Name? = null) = MutableSafeEnumvDelegate(config, key, default) { enumValueOf(it) } /* Node delegates */ -fun Configurable.node(key: String? = null) = MutableNodeDelegate(config, key) +fun Configurable.node(key: Name? = null): MutableNodeDelegate = MutableNodeDelegate(config, key) -fun Configurable.spec(spec: Specification, key: String? = null) = +fun Configurable.spec(spec: Specification, key: Name? = null) = MutableMorphDelegate(config, key) { spec.wrap(it) } -fun Configurable.spec(builder: (Config) -> T, key: String? = null) = +fun Configurable.spec(builder: (Config) -> T, key: Name? = null) = MutableMorphDelegate(config, key) { specification(builder).wrap(it) } /* * Extra delegates for special cases */ -fun Configurable.stringList(key: String? = null): ReadWriteDelegateWrapper> = - value(emptyList(), key) { it?.list?.map { value -> value.string } ?: emptyList() } +fun Configurable.stringList(vararg strings: String, key: Name? = null): ReadWriteDelegateWrapper> = + value(strings.asList(), key) { it?.list?.map { value -> value.string } ?: emptyList() } -fun Configurable.numberList(key: String? = null): ReadWriteDelegateWrapper> = - value(emptyList(), key) { it?.list?.map { value -> value.number } ?: emptyList() } +fun Configurable.numberList(vararg numbers: Number, key: Name? = null): ReadWriteDelegateWrapper> = + value(numbers.asList(), key) { it?.list?.map { value -> value.number } ?: emptyList() } /** * A special delegate for double arrays */ -fun Configurable.doubleArray(key: String? = null): ReadWriteDelegateWrapper = +fun Configurable.doubleArray(key: Name? = null): ReadWriteDelegateWrapper = value(doubleArrayOf(), key) { (it as? DoubleArrayValue)?.value ?: it?.list?.map { value -> value.number.toDouble() }?.toDoubleArray() ?: doubleArrayOf() } -fun Configurable.child(key: String? = null, converter: (Meta) -> T) = +fun Configurable.node(key: Name? = null, converter: (Meta) -> T) = MutableMorphDelegate(config, key, converter) diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/mapMeta.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/mapMeta.kt new file mode 100644 index 00000000..b0d8162c --- /dev/null +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/mapMeta.kt @@ -0,0 +1,39 @@ +package hep.dataforge.meta + +import hep.dataforge.descriptors.NodeDescriptor +import hep.dataforge.values.Value + +///** +// * Find all elements with given body +// */ +//private fun Meta.byBody(body: String): Map> = +// items.filter { it.key.body == body }.mapKeys { it.key.index } +// +//private fun Meta.distinctNames() = items.keys.map { it.body }.distinct() + +/** + * Convert meta to map of maps + */ +@DFExperimental +fun Meta.toMap(descriptor: NodeDescriptor? = null): Map { + return items.entries.associate { (token, item) -> + token.toString() to when (item) { + is MetaItem.NodeItem -> item.node.toMap() + is MetaItem.ValueItem -> item.value.value + } + } +} + +/** + * Convert map of maps to meta + */ +@DFExperimental +fun Map.toMeta(descriptor: NodeDescriptor? = null): Meta = buildMeta { + entries.forEach { (key, value) -> + @Suppress("UNCHECKED_CAST") + when (value) { + is Map<*, *> -> setNode(key, (value as Map).toMeta()) + else -> setValue(key, Value.of(value)) + } + } +} \ No newline at end of file diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/metaDelegates.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/metaDelegates.kt index 9d606b6f..65e49da2 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/metaDelegates.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/metaDelegates.kt @@ -1,5 +1,7 @@ package hep.dataforge.meta +import hep.dataforge.names.Name +import hep.dataforge.names.asName import hep.dataforge.values.Null import hep.dataforge.values.Value import hep.dataforge.values.asValue @@ -26,15 +28,21 @@ class StringDelegate(val meta: Meta, private val key: String? = null, private va } } -class BooleanDelegate(val meta: Meta, private val key: String? = null, private val default: Boolean? = null) : - ReadOnlyProperty { - override fun getValue(thisRef: Metoid, property: KProperty<*>): Boolean? { +class BooleanDelegate( + val meta: Meta, + private val key: String? = null, + private val default: Boolean? = null +) : ReadOnlyProperty { + override fun getValue(thisRef: Any?, property: KProperty<*>): Boolean? { return meta[key ?: property.name]?.boolean ?: default } } -class NumberDelegate(val meta: Meta, private val key: String? = null, private val default: Number? = null) : - ReadOnlyProperty { +class NumberDelegate( + val meta: Meta, + private val key: String? = null, + private val default: Number? = null +) : ReadOnlyProperty { override fun getValue(thisRef: Any?, property: KProperty<*>): Number? { return meta[key ?: property.name]?.number ?: default } @@ -136,7 +144,7 @@ fun Meta.boolean(default: Boolean? = null, key: String? = null) = BooleanDelegat fun Meta.number(default: Number? = null, key: String? = null) = NumberDelegate(this, key, default) -fun Meta.child(key: String? = null) = ChildDelegate(this, key) { it } +fun Meta.node(key: String? = null) = ChildDelegate(this, key) { it } @JvmName("safeString") fun Meta.string(default: String, key: String? = null) = @@ -166,22 +174,19 @@ fun Meta.number(key: String? = null, default: () -> Number) = inline fun > Meta.enum(default: E, key: String? = null) = SafeEnumDelegate(this, key, default) { enumValueOf(it) } - -fun Metoid.child(key: String? = null, converter: (Meta) -> T) = ChildDelegate(meta, key, converter) - /* Read-write delegates */ class MutableValueDelegate>( val meta: M, - private val key: String? = null, + private val key: Name? = null, private val default: Value? = null ) : ReadWriteProperty { override fun getValue(thisRef: Any?, property: KProperty<*>): Value? { - return meta[key ?: property.name]?.value ?: default + return meta[key ?: property.name.asName()]?.value ?: default } override fun setValue(thisRef: Any?, property: KProperty<*>, value: Value?) { - val name = key ?: property.name + val name = key ?: property.name.asName() if (value == null) { meta.remove(name) } else { @@ -195,15 +200,15 @@ class MutableValueDelegate>( class MutableStringDelegate>( val meta: M, - private val key: String? = null, + private val key: Name? = null, private val default: String? = null ) : ReadWriteProperty { override fun getValue(thisRef: Any?, property: KProperty<*>): String? { - return meta[key ?: property.name]?.string ?: default + return meta[key ?: property.name.asName()]?.string ?: default } override fun setValue(thisRef: Any?, property: KProperty<*>, value: String?) { - val name = key ?: property.name + val name = key ?: property.name.asName() if (value == null) { meta.remove(name) } else { @@ -214,15 +219,15 @@ class MutableStringDelegate>( class MutableBooleanDelegate>( val meta: M, - private val key: String? = null, + private val key: Name? = null, private val default: Boolean? = null ) : ReadWriteProperty { override fun getValue(thisRef: Any?, property: KProperty<*>): Boolean? { - return meta[key ?: property.name]?.boolean ?: default + return meta[key ?: property.name.asName()]?.boolean ?: default } override fun setValue(thisRef: Any?, property: KProperty<*>, value: Boolean?) { - val name = key ?: property.name + val name = key ?: property.name.asName() if (value == null) { meta.remove(name) } else { @@ -233,15 +238,15 @@ class MutableBooleanDelegate>( class MutableNumberDelegate>( val meta: M, - private val key: String? = null, + private val key: Name? = null, private val default: Number? = null ) : ReadWriteProperty { override fun getValue(thisRef: Any?, property: KProperty<*>): Number? { - return meta[key ?: property.name]?.number ?: default + return meta[key ?: property.name.asName()]?.number ?: default } override fun setValue(thisRef: Any?, property: KProperty<*>, value: Number?) { - val name = key ?: property.name + val name = key ?: property.name.asName() if (value == null) { meta.remove(name) } else { @@ -260,52 +265,52 @@ class MutableNumberDelegate>( class MutableSafeStringDelegate>( val meta: M, - private val key: String? = null, + private val key: Name? = null, default: () -> String ) : ReadWriteProperty { private val default: String by lazy(default) override fun getValue(thisRef: Any?, property: KProperty<*>): String { - return meta[key ?: property.name]?.string ?: default + return meta[key ?: property.name.asName()]?.string ?: default } override fun setValue(thisRef: Any?, property: KProperty<*>, value: String) { - meta.setValue(key ?: property.name, value.asValue()) + meta.setValue(key ?: property.name.asName(), value.asValue()) } } class MutableSafeBooleanDelegate>( val meta: M, - private val key: String? = null, + private val key: Name? = null, default: () -> Boolean ) : ReadWriteProperty { private val default: Boolean by lazy(default) override fun getValue(thisRef: Any?, property: KProperty<*>): Boolean { - return meta[key ?: property.name]?.boolean ?: default + return meta[key ?: property.name.asName()]?.boolean ?: default } override fun setValue(thisRef: Any?, property: KProperty<*>, value: Boolean) { - meta.setValue(key ?: property.name, value.asValue()) + meta.setValue(key ?: property.name.asName(), value.asValue()) } } class MutableSafeNumberDelegate>( val meta: M, - private val key: String? = null, + private val key: Name? = null, default: () -> Number ) : ReadWriteProperty { private val default: Number by lazy(default) override fun getValue(thisRef: Any?, property: KProperty<*>): Number { - return meta[key ?: property.name]?.number ?: default + return meta[key ?: property.name.asName()]?.number ?: default } override fun setValue(thisRef: Any?, property: KProperty<*>, value: Number) { - meta.setValue(key ?: property.name, value.asValue()) + meta.setValue(key ?: property.name.asName(), value.asValue()) } val double get() = ReadWriteDelegateWrapper(this, reader = { it.toDouble() }, writer = { it }) @@ -317,16 +322,16 @@ class MutableSafeNumberDelegate>( class MutableSafeEnumvDelegate, E : Enum>( val meta: M, - private val key: String? = null, + private val key: Name? = null, private val default: E, private val resolver: (String) -> E ) : ReadWriteProperty { override fun getValue(thisRef: Any?, property: KProperty<*>): E { - return (meta[key ?: property.name]?.string)?.let { resolver(it) } ?: default + return (meta[key ?: property.name.asName()]?.string)?.let { resolver(it) } ?: default } override fun setValue(thisRef: Any?, property: KProperty<*>, value: E) { - meta.setValue(key ?: property.name, value.name.asValue()) + meta.setValue(key ?: property.name.asName(), value.name.asValue()) } } @@ -334,31 +339,31 @@ class MutableSafeEnumvDelegate, E : Enum>( class MutableNodeDelegate>( val meta: M, - private val key: String? = null -) : ReadWriteProperty { - override fun getValue(thisRef: Any?, property: KProperty<*>): Meta? { - return meta[key ?: property.name]?.node + private val key: Name? = null +) : ReadWriteProperty { + override fun getValue(thisRef: Any?, property: KProperty<*>): M? { + return meta[key ?: property.name.asName()]?.node } - override fun setValue(thisRef: Any?, property: KProperty<*>, value: Meta?) { - meta[key ?: property.name] = value + override fun setValue(thisRef: Any?, property: KProperty<*>, value: M?) { + meta[key ?: property.name.asName()] = value } } class MutableMorphDelegate, T : Configurable>( val meta: M, - private val key: String? = null, + private val key: Name? = null, private val converter: (Meta) -> T ) : ReadWriteProperty { override fun getValue(thisRef: Any?, property: KProperty<*>): T? { - return meta[key ?: property.name]?.node?.let(converter) + return meta[key ?: property.name.asName()]?.node?.let(converter) } override fun setValue(thisRef: Any?, property: KProperty<*>, value: T?) { if (value == null) { - meta.remove(key ?: property.name) + meta.remove(key ?: property.name.asName()) } else { - meta[key ?: property.name] = value.config + meta[key ?: property.name.asName()] = value.config } } } @@ -383,44 +388,45 @@ class ReadWriteDelegateWrapper( /** * A property delegate that uses custom key */ -fun > M.value(default: Value = Null, key: String? = null) = +fun > M.value(default: Value = Null, key: Name? = null) = MutableValueDelegate(this, key, default) -fun > M.string(default: String? = null, key: String? = null) = +fun > M.string(default: String? = null, key: Name? = null) = MutableStringDelegate(this, key, default) -fun > M.boolean(default: Boolean? = null, key: String? = null) = +fun > M.boolean(default: Boolean? = null, key: Name? = null) = MutableBooleanDelegate(this, key, default) -fun > M.number(default: Number? = null, key: String? = null) = +fun > M.number(default: Number? = null, key: Name? = null) = MutableNumberDelegate(this, key, default) -fun > M.node(key: String? = null) = MutableNodeDelegate(this, key) +fun > M.node(key: Name? = null) = + MutableNodeDelegate(this, key) @JvmName("safeString") -fun > M.string(default: String, key: String? = null) = +fun > M.string(default: String, key: Name? = null) = MutableSafeStringDelegate(this, key) { default } @JvmName("safeBoolean") -fun > M.boolean(default: Boolean, key: String? = null) = +fun > M.boolean(default: Boolean, key: Name? = null) = MutableSafeBooleanDelegate(this, key) { default } @JvmName("safeNumber") -fun > M.number(default: Number, key: String? = null) = +fun > M.number(default: Number, key: Name? = null) = MutableSafeNumberDelegate(this, key) { default } @JvmName("safeString") -fun > M.string(key: String? = null, default: () -> String) = +fun > M.string(key: Name? = null, default: () -> String) = MutableSafeStringDelegate(this, key, default) @JvmName("safeBoolean") -fun > M.boolean(key: String? = null, default: () -> Boolean) = +fun > M.boolean(key: Name? = null, default: () -> Boolean) = MutableSafeBooleanDelegate(this, key, default) @JvmName("safeNumber") -fun > M.number(key: String? = null, default: () -> Number) = +fun > M.number(key: Name? = null, default: () -> Number) = MutableSafeNumberDelegate(this, key, default) -inline fun , reified E : Enum> M.enum(default: E, key: String? = null) = +inline fun , reified E : Enum> M.enum(default: E, key: Name? = null) = MutableSafeEnumvDelegate(this, key, default) { enumValueOf(it) } \ No newline at end of file diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/metaMatcher.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/metaMatcher.kt new file mode 100644 index 00000000..6f16f537 --- /dev/null +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/metaMatcher.kt @@ -0,0 +1,51 @@ +package hep.dataforge.meta + +import hep.dataforge.names.Name +import hep.dataforge.names.toName + +/** + * Get all items matching given name. + */ +@DFExperimental +fun Meta.getIndexed(name: Name): Map> { + val root = when (name.length) { + 0 -> error("Can't use empty name for that") + 1 -> this + else -> (this[name.cutLast()] as? MetaItem.NodeItem<*>)?.node + } + + val (body, index) = name.last()!! + val regex = index.toRegex() + + return root?.items + ?.filter { it.key.body == body && (index.isEmpty() || regex.matches(it.key.index)) } + ?.mapKeys { it.key.index } + ?: emptyMap() +} + +@DFExperimental +fun Meta.getIndexed(name: String): Map> = this@getIndexed.getIndexed(name.toName()) + + +/** + * Get all items matching given name. + */ +@DFExperimental +fun > M.getIndexed(name: Name): Map> { + val root: MetaNode? = when (name.length) { + 0 -> error("Can't use empty name for that") + 1 -> this + else -> (this[name.cutLast()] as? MetaItem.NodeItem)?.node + } + + val (body, index) = name.last()!! + val regex = index.toRegex() + + return root?.items + ?.filter { it.key.body == body && (index.isEmpty() || regex.matches(it.key.index)) } + ?.mapKeys { it.key.index } + ?: emptyMap() +} + +@DFExperimental +fun > M.getIndexed(name: String): Map> = getIndexed(name.toName()) \ No newline at end of file diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/names/Name.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/names/Name.kt index 070a8f75..41680eb0 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/names/Name.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/names/Name.kt @@ -139,9 +139,7 @@ fun String.toName(): Name { * Convert the [String] to a [Name] by simply wrapping it in a single name token without parsing. * The input string could contain dots and braces, but they are just escaped, not parsed. */ -fun String.asName(): Name { - return NameToken(this).asName() -} +fun String.asName(): Name = if (isBlank()) EmptyName else NameToken(this).asName() operator fun NameToken.plus(other: Name): Name = Name(listOf(this) + other.tokens) @@ -149,6 +147,8 @@ operator fun Name.plus(other: Name): Name = Name(this.tokens + other.tokens) operator fun Name.plus(other: String): Name = this + other.toName() +operator fun Name.plus(other: NameToken): Name = Name(tokens + other) + fun Name.appendLeft(other: String): Name = NameToken(other) + this fun NameToken.asName() = Name(listOf(this)) diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/values/Value.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/values/Value.kt index cd68e040..f044b018 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/values/Value.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/values/Value.kt @@ -41,10 +41,12 @@ interface Value { * get this value represented as List */ val list: List - get() = listOf(this) + get() = if(this == Null) emptyList() else listOf(this) override fun equals(other: Any?): Boolean + override fun hashCode(): Int + companion object { const val TYPE = "value" @@ -58,7 +60,14 @@ interface Value { true -> True false -> False is Number -> value.asValue() - is Iterable<*> -> ListValue(value.map { of(it) }) + is Iterable<*> -> { + val list = value.map { of(it) } + if (list.isEmpty()) { + Null + } else { + ListValue(list) + } + } is DoubleArray -> value.asValue() is IntArray -> value.asValue() is FloatArray -> value.asValue() @@ -86,14 +95,9 @@ object Null : Value { override fun toString(): String = value.toString() override fun equals(other: Any?): Boolean = other === Null + override fun hashCode(): Int = 0 } -/** - * Check if value is null - */ -fun Value.isNull(): Boolean = this == Null - - /** * Singleton true value */ @@ -106,7 +110,7 @@ object True : Value { override fun toString(): String = value.toString() override fun equals(other: Any?): Boolean = other === True - + override fun hashCode(): Int = 1 } /** @@ -121,10 +125,9 @@ object False : Value { override fun toString(): String = True.value.toString() override fun equals(other: Any?): Boolean = other === False + override fun hashCode(): Int = -1 } -val Value.boolean get() = this == True || this.list.firstOrNull() == True || (type == ValueType.STRING && string.toBoolean()) - class NumberValue(override val number: Number) : Value { override val value: Any? get() = number override val type: ValueType get() = ValueType.NUMBER @@ -178,23 +181,21 @@ class EnumValue>(override val value: E) : Value { class ListValue(override val list: List) : Value { init { - if (list.isEmpty()) { - throw IllegalArgumentException("Can't create list value from empty list") - } + require(list.isNotEmpty()) { "Can't create list value from empty list" } } - override val value: Any? get() = list + override val value: List get() = list override val type: ValueType get() = list.first().type override val number: Number get() = list.first().number override val string: String get() = list.first().string - override fun toString(): String = list.joinToString (prefix = "[", postfix = "]") + override fun toString(): String = list.joinToString(prefix = "[", postfix = "]") override fun equals(other: Any?): Boolean { if (this === other) return true if (other !is Value) return false - if( other is DoubleArrayValue){ - + if (other is DoubleArrayValue) { + return DoubleArray(list.size) { list[it].number.toDouble() }.contentEquals(other.value) } return list == other.list } @@ -206,29 +207,26 @@ class ListValue(override val list: List) : Value { } -/** - * Check if value is list - */ -fun Value.isList(): Boolean = this.list.size > 1 - - fun Number.asValue(): Value = NumberValue(this) fun Boolean.asValue(): Value = if (this) True else False fun String.asValue(): Value = StringValue(this) -fun Iterable.asValue(): Value = ListValue(this.toList()) +fun Iterable.asValue(): Value { + val list = toList() + return if (list.isEmpty()) Null else ListValue(this.toList()) +} -fun IntArray.asValue(): Value = ListValue(map{NumberValue(it)}) +fun IntArray.asValue(): Value = if (isEmpty()) Null else ListValue(map { NumberValue(it) }) -fun LongArray.asValue(): Value = ListValue(map{NumberValue(it)}) +fun LongArray.asValue(): Value = if (isEmpty()) Null else ListValue(map { NumberValue(it) }) -fun ShortArray.asValue(): Value = ListValue(map{NumberValue(it)}) +fun ShortArray.asValue(): Value = if (isEmpty()) Null else ListValue(map { NumberValue(it) }) -fun FloatArray.asValue(): Value = ListValue(map{NumberValue(it)}) +fun FloatArray.asValue(): Value = if (isEmpty()) Null else ListValue(map { NumberValue(it) }) -fun ByteArray.asValue(): Value = ListValue(map{NumberValue(it)}) +fun ByteArray.asValue(): Value = if (isEmpty()) Null else ListValue(map { NumberValue(it) }) /** diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/values/exoticValues.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/values/exoticValues.kt index f6122dbb..d0fb86e8 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/values/exoticValues.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/values/exoticValues.kt @@ -14,6 +14,8 @@ class LazyParsedValue(override val string: String) : Value { override fun toString(): String = string override fun equals(other: Any?): Boolean = other is Value && this.parsedValue == other + + override fun hashCode(): Int = string.hashCode() } fun String.lazyParseValue(): LazyParsedValue = LazyParsedValue(this) @@ -44,4 +46,4 @@ class DoubleArrayValue(override val value: DoubleArray) : Value { override fun toString(): String = list.joinToString (prefix = "[", postfix = "]") } -fun DoubleArray.asValue(): DoubleArrayValue = DoubleArrayValue(this) +fun DoubleArray.asValue(): Value = if(isEmpty()) Null else DoubleArrayValue(this) diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/values/valueExtensions.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/values/valueExtensions.kt new file mode 100644 index 00000000..a63d5ec1 --- /dev/null +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/values/valueExtensions.kt @@ -0,0 +1,37 @@ +package hep.dataforge.values + +import hep.dataforge.meta.Meta +import hep.dataforge.meta.buildMeta + +/** + * Check if value is null + */ +fun Value.isNull(): Boolean = this == Null + +/** + * Check if value is list + */ +fun Value.isList(): Boolean = this.list.size > 1 + +val Value.boolean + get() = this == True + || this.list.firstOrNull() == True + || (type == ValueType.STRING && string.toBoolean()) + + +val Value.int get() = number.toInt() +val Value.double get() = number.toDouble() +val Value.float get() = number.toFloat() +val Value.long get() = number.toLong() + +val Value.stringList: List get() = list.map { it.string } + +val Value.doubleArray: DoubleArray + get() = if (this is DoubleArrayValue) { + value + } else { + DoubleArray(list.size) { list[it].double } + } + + +fun Value.toMeta() = buildMeta { Meta.VALUE_KEY put this } \ No newline at end of file diff --git a/dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/MetaBuilderTest.kt b/dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/MetaBuilderTest.kt index 849a34fc..3fe10ceb 100644 --- a/dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/MetaBuilderTest.kt +++ b/dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/MetaBuilderTest.kt @@ -9,13 +9,13 @@ class MetaBuilderTest { @Test fun testBuilder() { val meta = buildMeta { - "a" to 22 - "b" to listOf(1, 2, 3) + "a" put 22 + "b" put listOf(1, 2, 3) this["c"] = "myValue".asValue() - "node" to { - "e" to 12.2 - "childNode" to { - "f" to true + "node" put { + "e" put 12.2 + "childNode" put { + "f" put true } } } @@ -27,12 +27,12 @@ class MetaBuilderTest { fun testSNS(){ val meta = buildMeta { repeat(10){ - "b.a[$it]" to it + "b.a[$it]" put it } }.seal() assertEquals(10, meta.values().count()) - val nodes = meta.getAll("b.a") + val nodes = meta.getIndexed("b.a") assertEquals(3, nodes["3"]?.int) } diff --git a/dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/MetaExtensionTest.kt b/dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/MetaExtensionTest.kt new file mode 100644 index 00000000..f2fffd19 --- /dev/null +++ b/dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/MetaExtensionTest.kt @@ -0,0 +1,23 @@ +package hep.dataforge.meta + +import kotlin.test.Test + + +class MetaExtensionTest { + + enum class TestEnum{ + test + } + + @Test + fun testEnum(){ + val meta = buildMeta{"enum" put TestEnum.test} + meta["enum"].enum() + } + @Test + fun testEnumByString(){ + val meta = buildMeta{"enum" put TestEnum.test.name} + println(meta["enum"].enum()) + } + +} \ No newline at end of file diff --git a/dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/MetaTest.kt b/dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/MetaTest.kt index 7f83b0b2..fb424116 100644 --- a/dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/MetaTest.kt +++ b/dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/MetaTest.kt @@ -17,17 +17,36 @@ class MetaTest { @Test fun metaEqualityTest() { val meta1 = buildMeta { - "a" to 22 - "b" to { - "c" to "ddd" + "a" put 22 + "b" put { + "c" put "ddd" } } val meta2 = buildMeta { - "b" to { - "c" to "ddd" + "b" put { + "c" put "ddd" } - "a" to 22 + "a" put 22 }.seal() assertEquals(meta1, meta2) } + + @Test + fun metaToMap(){ + val meta = buildMeta { + "a" put 22 + "b" put { + "c" put "ddd" + } + "list" put (0..4).map { + buildMeta { + "value" put it + } + } + } + val map = meta.toMap() + val reconstructed = map.toMeta() + + assertEquals(meta,reconstructed) + } } \ No newline at end of file diff --git a/dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/MutableMetaTest.kt b/dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/MutableMetaTest.kt index 5ab75fd4..9057782f 100644 --- a/dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/MutableMetaTest.kt +++ b/dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/MutableMetaTest.kt @@ -7,12 +7,12 @@ class MutableMetaTest{ @Test fun testRemove(){ val meta = buildMeta { - "aNode" to { - "innerNode" to { - "innerValue" to true + "aNode" put { + "innerNode" put { + "innerValue" put true } - "b" to 22 - "c" to "StringValue" + "b" put 22 + "c" put "StringValue" } }.toConfig() diff --git a/dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/SpecificationTest.kt b/dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/SpecificationTest.kt new file mode 100644 index 00000000..9098cf18 --- /dev/null +++ b/dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/SpecificationTest.kt @@ -0,0 +1,23 @@ +package hep.dataforge.meta + +import kotlin.test.Test +import kotlin.test.assertEquals + +class SpecificationTest { + class TestSpecific(override val config: Config) : Specific { + var list by numberList(1, 2, 3) + + companion object : Specification { + override fun wrap(config: Config): TestSpecific = TestSpecific(config) + } + } + + + @Test + fun testSpecific(){ + val testObject = TestSpecific.build { + list = emptyList() + } + assertEquals(emptyList(), testObject.list) + } +} \ No newline at end of file diff --git a/dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/StyledTest.kt b/dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/StyledTest.kt index 5a4699f9..a4cbe18e 100644 --- a/dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/StyledTest.kt +++ b/dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/StyledTest.kt @@ -9,8 +9,8 @@ class StyledTest{ fun testSNS(){ val meta = buildMeta { repeat(10){ - "b.a[$it]" to { - "d" to it + "b.a[$it]" put { + "d" put it } } }.seal().withStyle() @@ -18,9 +18,9 @@ class StyledTest{ val bNode = meta["b"].node - val aNodes = bNode?.getAll("a") + val aNodes = bNode?.getIndexed("a") - val allNodes = meta.getAll("b.a") + val allNodes = meta.getIndexed("b.a") assertEquals(3, aNodes?.get("3").node["d"].int) assertEquals(3, allNodes["3"].node["d"].int) diff --git a/dataforge-meta/src/jsMain/kotlin/hep/dataforge/meta/DynamicMeta.kt b/dataforge-meta/src/jsMain/kotlin/hep/dataforge/meta/DynamicMeta.kt index 694e2441..ff23a571 100644 --- a/dataforge-meta/src/jsMain/kotlin/hep/dataforge/meta/DynamicMeta.kt +++ b/dataforge-meta/src/jsMain/kotlin/hep/dataforge/meta/DynamicMeta.kt @@ -3,18 +3,27 @@ package hep.dataforge.meta import hep.dataforge.names.NameToken import hep.dataforge.values.Null import hep.dataforge.values.Value +import hep.dataforge.values.isList //TODO add Meta wrapper for dynamic +fun Value.toDynamic(): dynamic { + return if (isList()) { + list.map { it.toDynamic() }.toTypedArray().asDynamic() + } else { + value.asDynamic() + } +} + /** * Represent or copy this [Meta] to dynamic object to be passed to JS libraries */ fun Meta.toDynamic(): dynamic { - if(this is DynamicMeta) return this.obj + if (this is DynamicMeta) return this.obj fun MetaItem<*>.toDynamic(): dynamic = when (this) { - is MetaItem.ValueItem -> this.value.value.asDynamic() + is MetaItem.ValueItem -> this.value.toDynamic() is MetaItem.NodeItem -> this.node.toDynamic() } @@ -35,15 +44,21 @@ class DynamicMeta(val obj: dynamic) : MetaBase() { private fun isArray(@Suppress("UNUSED_PARAMETER") obj: dynamic): Boolean = js("Array.isArray(obj)") as Boolean + private fun isPrimitive(obj: dynamic): Boolean = + (jsTypeOf(obj) != "object") + @Suppress("UNCHECKED_CAST") private fun asItem(obj: dynamic): MetaItem? { - if (obj == null) return MetaItem.ValueItem(Null) - return when (jsTypeOf(obj as? Any)) { - "boolean" -> MetaItem.ValueItem(Value.of(obj as Boolean)) - "number" -> MetaItem.ValueItem(Value.of(obj as Number)) - "string" -> MetaItem.ValueItem(Value.of(obj as String)) - "object" -> MetaItem.NodeItem(DynamicMeta(obj)) - else -> null + return when { + obj == null -> MetaItem.ValueItem(Null) + isArray(obj) && (obj as Array).all { isPrimitive(it) } -> MetaItem.ValueItem(Value.of(obj as Array)) + else -> when (jsTypeOf(obj)) { + "boolean" -> MetaItem.ValueItem(Value.of(obj as Boolean)) + "number" -> MetaItem.ValueItem(Value.of(obj as Number)) + "string" -> MetaItem.ValueItem(Value.of(obj as String)) + "object" -> MetaItem.NodeItem(DynamicMeta(obj)) + else -> null + } } } @@ -51,11 +66,15 @@ class DynamicMeta(val obj: dynamic) : MetaBase() { get() = keys().flatMap>> { key -> val value = obj[key] ?: return@flatMap emptyList() if (isArray(value)) { - return@flatMap (value as Array) - .mapIndexedNotNull() { index, it -> + val array = value as Array + return@flatMap if (array.all { isPrimitive(it) }) { + listOf(NameToken(key) to MetaItem.ValueItem(Value.of(array))) + } else { + array.mapIndexedNotNull { index, it -> val item = asItem(it) ?: return@mapIndexedNotNull null NameToken(key, index.toString()) to item } + } } else { val item = asItem(value) ?: return@flatMap emptyList() listOf(NameToken(key) to item) diff --git a/dataforge-meta/src/jsTest/kotlin/hep/dataforge/meta/DynamicMetaTest.kt b/dataforge-meta/src/jsTest/kotlin/hep/dataforge/meta/DynamicMetaTest.kt index 56033171..71eca4f5 100644 --- a/dataforge-meta/src/jsTest/kotlin/hep/dataforge/meta/DynamicMetaTest.kt +++ b/dataforge-meta/src/jsTest/kotlin/hep/dataforge/meta/DynamicMetaTest.kt @@ -1,7 +1,9 @@ package hep.dataforge.meta +import hep.dataforge.values.int import kotlin.test.Test import kotlin.test.assertEquals +import kotlin.test.assertTrue class DynamicMetaTest { @@ -10,7 +12,7 @@ class DynamicMetaTest { fun testDynamicMeta() { val d = js("{}") d.a = 22 - d.array = arrayOf(1,2,3) + d.array = arrayOf(1, 2, 3) d.b = "myString" d.ob = js("{}") d.ob.childNode = 18 @@ -18,7 +20,35 @@ class DynamicMetaTest { val meta = DynamicMeta(d) assertEquals(true, meta["ob.booleanNode"].boolean) - assertEquals(2,meta["array[1]"].int) + assertEquals(2, meta["array"].value?.list?.get(1)?.int) + assertEquals(4, meta.items.size) + } + + @Test + fun testMetaToDynamic(){ + val meta = buildMeta { + "a" put 22 + "array" put listOf(1, 2, 3) + "b" put "myString" + "ob" put { + "childNode" put 18 + "booleanNode" put true + } + } + + val dynamic = meta.toDynamic() + + assertEquals(2,dynamic.array[1]) + + assertEquals(22, dynamic.a) + + val keys = js("Object.keys(dynamic)") as Array + + assertTrue { keys.contains("ob") } + + assertEquals(18, dynamic.ob.childNode) + + assertEquals(meta, DynamicMeta(dynamic)) } } \ No newline at end of file diff --git a/dataforge-output/src/commonMain/kotlin/hep/dataforge/output/OutputManager.kt b/dataforge-output/src/commonMain/kotlin/hep/dataforge/output/OutputManager.kt index 448b3adb..d9f7c1b2 100644 --- a/dataforge-output/src/commonMain/kotlin/hep/dataforge/output/OutputManager.kt +++ b/dataforge-output/src/commonMain/kotlin/hep/dataforge/output/OutputManager.kt @@ -70,7 +70,7 @@ class ConsoleOutputManager : AbstractPlugin(), OutputManager { override val type = ConsoleOutputManager::class - override fun invoke(meta:Meta) = ConsoleOutputManager() + override fun invoke(meta: Meta, context: Context) = ConsoleOutputManager() } } diff --git a/dataforge-scripting/src/jvmTest/kotlin/hep/dataforge/scripting/BuildersKtTest.kt b/dataforge-scripting/src/jvmTest/kotlin/hep/dataforge/scripting/BuildersKtTest.kt index bcc357d4..5a9ba56d 100644 --- a/dataforge-scripting/src/jvmTest/kotlin/hep/dataforge/scripting/BuildersKtTest.kt +++ b/dataforge-scripting/src/jvmTest/kotlin/hep/dataforge/scripting/BuildersKtTest.kt @@ -3,7 +3,9 @@ package hep.dataforge.scripting import hep.dataforge.context.Global import hep.dataforge.meta.get import hep.dataforge.meta.int -import hep.dataforge.workspace.* +import hep.dataforge.workspace.SimpleWorkspaceBuilder +import hep.dataforge.workspace.context +import hep.dataforge.workspace.target import org.junit.Test import kotlin.test.assertEquals @@ -17,7 +19,7 @@ class BuildersKtTest { context("test") target("testTarget"){ - "a" to 12 + "a" put 12 } } } @@ -30,7 +32,7 @@ class BuildersKtTest { context("test") target("testTarget"){ - "a" to 12 + "a" put 12 } """.trimIndent() val workspace = Builders.buildWorkspace(script) diff --git a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/Dependency.kt b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/Dependency.kt index 24c012ea..ddb53d5d 100644 --- a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/Dependency.kt +++ b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/Dependency.kt @@ -6,10 +6,7 @@ import hep.dataforge.data.filter import hep.dataforge.meta.Meta import hep.dataforge.meta.MetaRepr import hep.dataforge.meta.buildMeta -import hep.dataforge.names.EmptyName -import hep.dataforge.names.Name -import hep.dataforge.names.get -import hep.dataforge.names.isEmpty +import hep.dataforge.names.* /** * A dependency of the task which allows to lazily create a data tree for single dependency @@ -24,13 +21,13 @@ class DataDependency(val filter: DataFilter, val placement: Name = EmptyName) : return if (placement.isEmpty()) { result } else { - DataNode.build(Any::class) { this[placement] = result } + DataNode.invoke(Any::class) { this[placement] = result } } } override fun toMeta(): Meta = buildMeta { - "data" to filter.config - "to" to placement + "data" put filter.config + "to" put placement.toString() } } @@ -38,31 +35,63 @@ class AllDataDependency(val placement: Name = EmptyName) : Dependency() { override fun apply(workspace: Workspace): DataNode = if (placement.isEmpty()) { workspace.data } else { - DataNode.build(Any::class) { this[placement] = workspace.data } + DataNode.invoke(Any::class) { this[placement] = workspace.data } } override fun toMeta() = buildMeta { - "data" to "*" - "to" to placement + "data" put "@all" + "to" put placement.toString() } - } -class TaskModelDependency(val name: String, val meta: Meta, val placement: Name = EmptyName) : Dependency() { - override fun apply(workspace: Workspace): DataNode { - val task = workspace.tasks[name] ?: error("Task with name $name is not found in the workspace") +abstract class TaskDependency( + val meta: Meta, + val placement: Name = EmptyName +) : Dependency() { + abstract fun resolveTask(workspace: Workspace): Task + + /** + * A name of the dependency for logging and serialization + */ + abstract val name: Name + + override fun apply(workspace: Workspace): DataNode { + val task = resolveTask(workspace) if (task.isTerminal) TODO("Support terminal task") val result = workspace.run(task, meta) return if (placement.isEmpty()) { result } else { - DataNode.build(Any::class) { this[placement] = result } + DataNode(task.type) { this[placement] = result } } } override fun toMeta(): Meta = buildMeta { - "task" to name - "meta" to meta - "to" to placement + "task" put name.toString() + "meta" put meta + "to" put placement.toString() } +} + +class DirectTaskDependency( + val task: Task, + meta: Meta, + placement: Name +) : TaskDependency(meta, placement) { + override fun resolveTask(workspace: Workspace): Task = task + + override val name: Name get() = DIRECT_TASK_NAME + task.name + + companion object { + val DIRECT_TASK_NAME = "@direct".asName() + } +} + +class WorkspaceTaskDependency( + override val name: Name, + meta: Meta, + placement: Name +) : TaskDependency(meta, placement) { + override fun resolveTask(workspace: Workspace): Task<*> = + workspace.tasks[name] ?: error("Task with name $name is not found in the workspace") } \ No newline at end of file diff --git a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/GenericTask.kt b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/GenericTask.kt index 278098d1..ff499888 100644 --- a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/GenericTask.kt +++ b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/GenericTask.kt @@ -1,37 +1,38 @@ package hep.dataforge.workspace -import hep.dataforge.data.* +import hep.dataforge.data.DataNode import hep.dataforge.descriptors.NodeDescriptor import hep.dataforge.meta.Meta import hep.dataforge.meta.get import hep.dataforge.meta.node +import hep.dataforge.names.Name import kotlin.reflect.KClass //data class TaskEnv(val workspace: Workspace, val model: TaskModel) class GenericTask( - override val name: String, + override val name: Name, override val type: KClass, override val descriptor: NodeDescriptor, private val modelTransform: TaskModelBuilder.(Meta) -> Unit, private val dataTransform: Workspace.() -> TaskModel.(DataNode) -> DataNode ) : Task { - private fun gather(workspace: Workspace, model: TaskModel): DataNode { - return DataNode.build(Any::class) { - model.dependencies.forEach { dep -> - update(dep.apply(workspace)) - } - } - } +// private fun gather(workspace: Workspace, model: TaskModel): DataNode { +// return DataNode.invoke(Any::class) { +// model.dependencies.forEach { dep -> +// update(dep.apply(workspace)) +// } +// } +// } override fun run(workspace: Workspace, model: TaskModel): DataNode { //validate model validate(model) // gather data - val input = gather(workspace, model) + val input = model.buildInput(workspace)// gather(workspace, model) //execute workspace.context.logger.info{"Starting task '$name' on ${model.target} with meta: \n${model.meta}"} @@ -54,7 +55,7 @@ class GenericTask( override fun build(workspace: Workspace, taskConfig: Meta): TaskModel { val taskMeta = taskConfig[name]?.node ?: taskConfig val builder = TaskModelBuilder(name, taskMeta) - modelTransform.invoke(builder, taskMeta) + builder.modelTransform(taskMeta) return builder.build() } //TODO add validation diff --git a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/SimpleWorkspace.kt b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/SimpleWorkspace.kt index 68438440..c94fd8ca 100644 --- a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/SimpleWorkspace.kt +++ b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/SimpleWorkspace.kt @@ -1,12 +1,11 @@ package hep.dataforge.workspace import hep.dataforge.context.Context -import hep.dataforge.context.Global import hep.dataforge.context.content +import hep.dataforge.context.toMap import hep.dataforge.data.DataNode import hep.dataforge.meta.Meta import hep.dataforge.names.Name -import hep.dataforge.names.toName /** @@ -20,11 +19,10 @@ class SimpleWorkspace( ) : Workspace { override val tasks: Map> by lazy { - context.content>(Task.TYPE) + tasks.associate { it.name.toName() to it } + context.content>(Task.TYPE) + tasks.toMap() } companion object { - fun build(parent: Context = Global, block: SimpleWorkspaceBuilder.() -> Unit): SimpleWorkspace = - SimpleWorkspaceBuilder(parent).apply(block).build() + } } \ No newline at end of file diff --git a/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/TaskBuilder.kt b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskBuilder.kt similarity index 57% rename from dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/TaskBuilder.kt rename to dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskBuilder.kt index 175c4d87..80d89e24 100644 --- a/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/TaskBuilder.kt +++ b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskBuilder.kt @@ -6,54 +6,78 @@ import hep.dataforge.descriptors.NodeDescriptor import hep.dataforge.meta.Meta import hep.dataforge.meta.get import hep.dataforge.meta.string +import hep.dataforge.names.EmptyName import hep.dataforge.names.Name +import hep.dataforge.names.isEmpty import hep.dataforge.names.toName +import kotlin.jvm.JvmName import kotlin.reflect.KClass @TaskBuildScope -class TaskBuilder(val name: String) { - private var modelTransform: TaskModelBuilder.(Meta) -> Unit = { data("*") } +class TaskBuilder(val name: Name, val type: KClass) { + private var modelTransform: TaskModelBuilder.(Meta) -> Unit = { allData() } +// private val additionalDependencies = HashSet() var descriptor: NodeDescriptor? = null + private val dataTransforms: MutableList = ArrayList() /** * TODO will look better as extension class */ - private class DataTransformation( + private inner class DataTransformation( val from: String = "", val to: String = "", - val transform: (Context, TaskModel, DataNode) -> DataNode + val transform: (Context, TaskModel, DataNode) -> DataNode ) { - operator fun invoke(workspace: Workspace, model: TaskModel, node: DataNode): DataNode? { + operator fun invoke(workspace: Workspace, model: TaskModel, node: DataNode): DataNode? { val localData = if (from.isEmpty()) { node } else { - node[from.toName()].node ?: return null + node[from].node ?: return null } return transform(workspace.context, model, localData) } } - private val dataTransforms: MutableList = ArrayList(); +// override fun add(dependency: Dependency) { +// additionalDependencies.add(dependency) +// } fun model(modelTransform: TaskModelBuilder.(Meta) -> Unit) { this.modelTransform = modelTransform } - fun transform( - inputType: KClass, + /** + * Add a transformation on untyped data + */ + @JvmName("rawTransform") + fun transform( from: String = "", to: String = "", - block: TaskModel.(Context, DataNode) -> DataNode + block: TaskEnv.(DataNode<*>) -> DataNode ) { dataTransforms += DataTransformation(from, to) { context, model, data -> - block(model, context, data.cast(inputType)) + val env = TaskEnv(EmptyName, model.meta, context, data) + env.block(data) + } + } + + fun transform( + inputType: KClass, + from: String = "", + to: String = "", + block: TaskEnv.(DataNode) -> DataNode + ) { + dataTransforms += DataTransformation(from, to) { context, model, data -> + data.ensureType(inputType) + val env = TaskEnv(EmptyName, model.meta, context, data) + env.block(data.cast(inputType)) } } inline fun transform( from: String = "", to: String = "", - noinline block: TaskModel.(Context, DataNode) -> DataNode + noinline block: TaskEnv.(DataNode) -> DataNode ) { transform(T::class, from, to, block) } @@ -61,52 +85,57 @@ class TaskBuilder(val name: String) { /** * Perform given action on data elements in `from` node in input and put the result to `to` node */ - inline fun action( + inline fun action( from: String = "", to: String = "", - crossinline block: Context.() -> Action + crossinline block: TaskEnv.() -> Action ) { - transform(from, to) { context, data: DataNode -> - block(context).invoke(data, meta) + transform(from, to) { data: DataNode -> + block().invoke(data, meta) } } - class TaskEnv(val name: Name, val meta: Meta, val context: Context) + class TaskEnv(val name: Name, val meta: Meta, val context: Context, val data: DataNode) { + operator fun DirectTaskDependency.invoke(): DataNode = if(placement.isEmpty()){ + data.cast(task.type) + } else { + data[placement].node?.cast(task.type) + ?: error("Could not find results of direct task dependency $this at \"$placement\"") + } + } /** - * A customized pipe action with ability to change meta and name + * A customized map action with ability to change meta and name */ - inline fun customPipe( + inline fun mapAction( from: String = "", to: String = "", - crossinline block: PipeBuilder.(Context) -> Unit + crossinline block: MapActionBuilder.(TaskEnv) -> Unit ) { action(from, to) { - val context = this - PipeAction( + MapAction( inputType = T::class, - outputType = R::class - ) { block(context) } + outputType = type + ) { block(this@action) } } } /** - * A simple pipe action without changing meta or name + * A simple map action without changing meta or name */ - inline fun pipe( + inline fun map( from: String = "", to: String = "", crossinline block: suspend TaskEnv.(T) -> R ) { action(from, to) { - val context = this - PipeAction( + MapAction( inputType = T::class, - outputType = R::class + outputType = type ) { //TODO automatically append task meta result = { data -> - TaskEnv(name, meta, context).block(data) + block(data) } } } @@ -115,15 +144,15 @@ class TaskBuilder(val name: String) { /** * Join elements in gathered data by multiple groups */ - inline fun joinByGroup( + inline fun reduceByGroup( from: String = "", to: String = "", - crossinline block: JoinGroupBuilder.(Context) -> Unit + crossinline block: ReduceGroupBuilder.(TaskEnv) -> Unit //TODO needs KEEP-176 ) { action(from, to) { - JoinAction( + ReduceAction( inputType = T::class, - outputType = R::class + outputType = type ) { block(this@action) } } } @@ -131,21 +160,20 @@ class TaskBuilder(val name: String) { /** * Join all elemlents in gathered data matching input type */ - inline fun join( + inline fun reduce( from: String = "", to: String = "", crossinline block: suspend TaskEnv.(Map) -> R ) { action(from, to) { - val context = this - JoinAction( + ReduceAction( inputType = T::class, - outputType = R::class, + outputType = type, action = { result( actionMeta[TaskModel.MODEL_TARGET_KEY]?.string ?: "@anonymous" ) { data -> - TaskEnv(name, meta, context).block(data) + block(data) } } ) @@ -155,15 +183,15 @@ class TaskBuilder(val name: String) { /** * Split each element in gathered data into fixed number of fragments */ - inline fun split( + inline fun split( from: String = "", to: String = "", - crossinline block: SplitBuilder.(Context) -> Unit + crossinline block: SplitBuilder.(TaskEnv) -> Unit //TODO needs KEEP-176 ) { action(from, to) { SplitAction( inputType = T::class, - outputType = R::class + outputType = type ) { block(this@action) } } } @@ -175,22 +203,28 @@ class TaskBuilder(val name: String) { this.descriptor = NodeDescriptor.build(transform) } - internal fun build(): GenericTask = - GenericTask( + internal fun build(): GenericTask { +// val actualTransform: TaskModelBuilder.(Meta) -> Unit = { +// modelTransform +// dependencies.addAll(additionalDependencies) +// } + + return GenericTask( name, - Any::class, + type, descriptor ?: NodeDescriptor.empty(), modelTransform ) { val workspace = this - { data -> + return@GenericTask { data -> val model = this if (dataTransforms.isEmpty()) { //return data node as is - logger.warn("No transformation present, returning input data") - data + logger.warn { "No transformation present, returning input data" } + data.ensureType(type) + data.cast(type) } else { - val builder = DataTreeBuilder(Any::class) + val builder = DataTreeBuilder(type) dataTransforms.forEach { transformation -> val res = transformation(workspace, model, data) if (res != null) { @@ -205,14 +239,6 @@ class TaskBuilder(val name: String) { } } } + } } -fun Workspace.Companion.task(name: String, builder: TaskBuilder.() -> Unit): GenericTask { - return TaskBuilder(name).apply(builder).build() -} - -fun WorkspaceBuilder.task(name: String, builder: TaskBuilder.() -> Unit) { - task(TaskBuilder(name).apply(builder).build()) -} - -//TODO add delegates to build gradle-like tasks \ No newline at end of file diff --git a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskModel.kt b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskModel.kt index b6c4b2b6..b4ccb7ae 100644 --- a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskModel.kt +++ b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskModel.kt @@ -8,10 +8,10 @@ package hep.dataforge.workspace import hep.dataforge.data.DataFilter import hep.dataforge.data.DataTree import hep.dataforge.data.DataTreeBuilder -import hep.dataforge.data.dataSequence import hep.dataforge.meta.* import hep.dataforge.names.EmptyName import hep.dataforge.names.Name +import hep.dataforge.names.asName import hep.dataforge.names.toName import hep.dataforge.workspace.TaskModel.Companion.MODEL_TARGET_KEY @@ -23,7 +23,7 @@ import hep.dataforge.workspace.TaskModel.Companion.MODEL_TARGET_KEY * @param dependencies a list of direct dependencies for this task */ data class TaskModel( - val name: String, + val name: Name, val meta: Meta, val dependencies: Collection ) : MetaRepr { @@ -31,19 +31,19 @@ data class TaskModel( //TODO add pre-run check of task result type? override fun toMeta(): Meta = buildMeta { - "name" to name - "meta" to meta - "dependsOn" to { + "name" put name.toString() + "meta" put meta + "dependsOn" put { val dataDependencies = dependencies.filterIsInstance() - val taskDependencies = dependencies.filterIsInstance() + val taskDependencies = dependencies.filterIsInstance>() setIndexed("data".toName(), dataDependencies.map { it.toMeta() }) - setIndexed("task".toName(), taskDependencies.map { it.toMeta() }) { taskDependencies[it].name } + setIndexed("task".toName(), taskDependencies.map { it.toMeta() }) { taskDependencies[it].name.toString() } //TODO ensure all dependencies are listed } } companion object { - const val MODEL_TARGET_KEY = "@target" + val MODEL_TARGET_KEY = "@target".asName() } } @@ -52,9 +52,8 @@ data class TaskModel( */ fun TaskModel.buildInput(workspace: Workspace): DataTree { return DataTreeBuilder(Any::class).apply { - dependencies.asSequence().flatMap { it.apply(workspace).dataSequence() }.forEach { (name, data) -> - //TODO add concise error on replacement - this[name] = data + dependencies.forEach { dep -> + update(dep.apply(workspace)) } }.build() } @@ -62,54 +61,88 @@ fun TaskModel.buildInput(workspace: Workspace): DataTree { @DslMarker annotation class TaskBuildScope +interface TaskDependencyContainer { + val defaultMeta: Meta + fun add(dependency: Dependency) +} + +/** + * Add dependency for a task defined in a workspace and resolved by + */ +fun TaskDependencyContainer.dependsOn( + name: Name, + placement: Name = EmptyName, + meta: Meta = defaultMeta +): WorkspaceTaskDependency = + WorkspaceTaskDependency(name, meta, placement).also { add(it) } + +fun TaskDependencyContainer.dependsOn( + name: String, + placement: Name = EmptyName, + meta: Meta = defaultMeta +): WorkspaceTaskDependency = + dependsOn(name.toName(), placement, meta) + +fun TaskDependencyContainer.dependsOn( + task: Task, + placement: Name = EmptyName, + meta: Meta = defaultMeta +): DirectTaskDependency = + DirectTaskDependency(task, meta, placement).also { add(it) } + +fun TaskDependencyContainer.dependsOn( + task: Task, + placement: String, + meta: Meta = defaultMeta +): DirectTaskDependency = + DirectTaskDependency(task, meta, placement.toName()).also { add(it) } + +fun TaskDependencyContainer.dependsOn( + task: Task, + placement: Name = EmptyName, + metaBuilder: MetaBuilder.() -> Unit +): DirectTaskDependency = + dependsOn(task, placement, buildMeta(metaBuilder)) + +/** + * Add custom data dependency + */ +fun TaskDependencyContainer.data(action: DataFilter.() -> Unit): DataDependency = + DataDependency(DataFilter.build(action)).also { add(it) } + +/** + * User-friendly way to add data dependency + */ +fun TaskDependencyContainer.data(pattern: String? = null, from: String? = null, to: String? = null): DataDependency = + data { + pattern?.let { this.pattern = it } + from?.let { this.from = it } + to?.let { this.to = it } + } + +/** + * Add all data as root node + */ +fun TaskDependencyContainer.allData(to: Name = EmptyName) = AllDataDependency(to).also { add(it) } + /** * A builder for [TaskModel] */ -@TaskBuildScope -class TaskModelBuilder(val name: String, meta: Meta = EmptyMeta) { +class TaskModelBuilder(val name: Name, meta: Meta = EmptyMeta) : TaskDependencyContainer { /** * Meta for current task. By default uses the whole input meta */ var meta: MetaBuilder = meta.builder() val dependencies = HashSet() + override val defaultMeta: Meta get() = meta + + override fun add(dependency: Dependency) { + dependencies.add(dependency) + } + var target: String by this.meta.string(key = MODEL_TARGET_KEY, default = "") - /** - * Add dependency for - */ - fun dependsOn(name: String, meta: Meta = this.meta, placement: Name = EmptyName) { - dependencies.add(TaskModelDependency(name, meta, placement)) - } - - fun dependsOn(task: Task<*>, meta: Meta = this.meta, placement: Name = EmptyName) = - dependsOn(task.name, meta, placement) - - fun dependsOn(task: Task<*>, placement: Name = EmptyName, metaBuilder: MetaBuilder.() -> Unit) = - dependsOn(task.name, buildMeta(metaBuilder), placement) - - /** - * Add custom data dependency - */ - fun data(action: DataFilter.() -> Unit) { - dependencies.add(DataDependency(DataFilter.build(action))) - } - - /** - * User-friendly way to add data dependency - */ - fun data(pattern: String? = null, from: String? = null, to: String? = null) = data { - pattern?.let { this.pattern = it } - from?.let { this.from = it } - to?.let { this.to = it } - } - - /** - * Add all data as root node - */ - fun allData(to: Name = EmptyName) { - dependencies.add(AllDataDependency(to)) - } fun build(): TaskModel = TaskModel(name, meta.seal(), dependencies) } diff --git a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/Workspace.kt b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/Workspace.kt index f5f0f3a6..31da6c56 100644 --- a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/Workspace.kt +++ b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/Workspace.kt @@ -1,6 +1,8 @@ package hep.dataforge.workspace +import hep.dataforge.context.Context import hep.dataforge.context.ContextAware +import hep.dataforge.context.Global import hep.dataforge.data.Data import hep.dataforge.data.DataNode import hep.dataforge.data.dataSequence @@ -56,6 +58,8 @@ interface Workspace : ContextAware, Provider { companion object { const val TYPE = "workspace" + operator fun invoke(parent: Context = Global, block: SimpleWorkspaceBuilder.() -> Unit): SimpleWorkspace = + SimpleWorkspaceBuilder(parent).apply(block).build() } } @@ -73,3 +77,6 @@ fun Workspace.run(task: String, meta: Meta) = fun Workspace.run(task: String, block: MetaBuilder.() -> Unit = {}) = run(task, buildMeta(block)) + +fun Workspace.run(task: Task, metaBuilder: MetaBuilder.() -> Unit = {}): DataNode = + run(task, buildMeta(metaBuilder)) \ No newline at end of file diff --git a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/WorkspaceBuilder.kt b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/WorkspaceBuilder.kt index 999ee50c..2f717f78 100644 --- a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/WorkspaceBuilder.kt +++ b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/WorkspaceBuilder.kt @@ -2,12 +2,15 @@ package hep.dataforge.workspace import hep.dataforge.context.Context import hep.dataforge.context.ContextBuilder -import hep.dataforge.data.Data import hep.dataforge.data.DataNode import hep.dataforge.data.DataTreeBuilder import hep.dataforge.meta.* +import hep.dataforge.names.EmptyName import hep.dataforge.names.Name +import hep.dataforge.names.isEmpty import hep.dataforge.names.toName +import kotlin.jvm.JvmName +import kotlin.reflect.KClass @TaskBuildScope interface WorkspaceBuilder { @@ -28,32 +31,26 @@ fun WorkspaceBuilder.context(name: String = "WORKSPACE", block: ContextBuilder.( context = ContextBuilder(name, parentContext).apply(block).build() } -fun WorkspaceBuilder.data(name: Name, data: Data) { - this.data[name] = data +inline fun WorkspaceBuilder.data( + name: Name = EmptyName, + noinline block: DataTreeBuilder.() -> Unit +): DataNode { + val node = DataTreeBuilder(T::class).apply(block) + if (name.isEmpty()) { + @Suppress("UNCHECKED_CAST") + data = node as DataTreeBuilder + } else { + data[name] = node + } + return node.build() } -fun WorkspaceBuilder.data(name: String, data: Data) = data(name.toName(), data) +@JvmName("rawData") +fun WorkspaceBuilder.data( + name: Name = EmptyName, + block: DataTreeBuilder.() -> Unit +): DataNode = data(name, block) -fun WorkspaceBuilder.static(name: Name, data: Any, meta: Meta = EmptyMeta) = - data(name, Data.static(data, meta)) - -fun WorkspaceBuilder.static(name: Name, data: Any, block: MetaBuilder.() -> Unit = {}) = - data(name, Data.static(data, buildMeta(block))) - -fun WorkspaceBuilder.static(name: String, data: Any, block: MetaBuilder.() -> Unit = {}) = - data(name, Data.static(data, buildMeta(block))) - -fun WorkspaceBuilder.data(name: Name, node: DataNode) { - this.data[name] = node -} - -fun WorkspaceBuilder.data(name: String, node: DataNode) = data(name.toName(), node) - -fun WorkspaceBuilder.data(name: Name, block: DataTreeBuilder.() -> Unit) { - this.data[name] = DataNode.build(Any::class, block) -} - -fun WorkspaceBuilder.data(name: String, block: DataTreeBuilder.() -> Unit) = data(name.toName(), block) fun WorkspaceBuilder.target(name: String, block: MetaBuilder.() -> Unit) { targets[name] = buildMeta(block).seal() @@ -65,22 +62,34 @@ fun WorkspaceBuilder.target(name: String, block: MetaBuilder.() -> Unit) { fun WorkspaceBuilder.target(name: String, base: String, block: MetaBuilder.() -> Unit) { val parentTarget = targets[base] ?: error("Base target with name $base not found") targets[name] = parentTarget.builder() - .apply { "@baseTarget" to base } + .apply { "@baseTarget" put base } .apply(block) .seal() } -fun WorkspaceBuilder.task(task: Task) { - this.tasks.add(task) -} +fun WorkspaceBuilder.task( + name: String, + type: KClass, + builder: TaskBuilder.() -> Unit +): Task = TaskBuilder(name.toName(), type).apply(builder).build().also { tasks.add(it) } +inline fun WorkspaceBuilder.task( + name: String, + noinline builder: TaskBuilder.() -> Unit +): Task = task(name, T::class, builder) + +@JvmName("rawTask") +fun WorkspaceBuilder.task( + name: String, + builder: TaskBuilder.() -> Unit +): Task = task(name, Any::class, builder) /** * A builder for a simple workspace */ class SimpleWorkspaceBuilder(override val parentContext: Context) : WorkspaceBuilder { override var context: Context = parentContext - override var data = DataTreeBuilder(Any::class) + override var data: DataTreeBuilder = DataTreeBuilder(Any::class) override var tasks: MutableSet> = HashSet() override var targets: MutableMap = HashMap() diff --git a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/WorkspacePlugin.kt b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/WorkspacePlugin.kt index 2fefd4f8..a377c023 100644 --- a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/WorkspacePlugin.kt +++ b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/WorkspacePlugin.kt @@ -1,19 +1,42 @@ package hep.dataforge.workspace import hep.dataforge.context.AbstractPlugin +import hep.dataforge.context.toMap import hep.dataforge.names.Name import hep.dataforge.names.toName +import kotlin.reflect.KClass /** * An abstract plugin with some additional boilerplate to effectively work with workspace context */ abstract class WorkspacePlugin : AbstractPlugin() { - abstract val tasks: Collection> + private val _tasks = HashSet>() + val tasks: Collection> get() = _tasks override fun provideTop(target: String): Map { - return when(target){ - Task.TYPE -> tasks.associateBy { it.name.toName() } + return when (target) { + Task.TYPE -> tasks.toMap() else -> emptyMap() } } + + fun task(task: Task<*>){ + _tasks.add(task) + } + + fun task( + name: String, + type: KClass, + builder: TaskBuilder.() -> Unit + ): GenericTask = TaskBuilder(name.toName(), type).apply(builder).build().also { + _tasks.add(it) + } + + inline fun task( + name: String, + noinline builder: TaskBuilder.() -> Unit + ) = task(name, T::class, builder) + +// +////TODO add delegates to build gradle-like tasks } \ No newline at end of file diff --git a/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/fileData.kt b/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/fileData.kt index f20e18cf..a483c78b 100644 --- a/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/fileData.kt +++ b/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/fileData.kt @@ -1,12 +1,14 @@ package hep.dataforge.workspace import hep.dataforge.data.Data +import hep.dataforge.data.DataNode +import hep.dataforge.data.DataTreeBuilder +import hep.dataforge.data.datum import hep.dataforge.descriptors.NodeDescriptor import hep.dataforge.io.* import hep.dataforge.meta.EmptyMeta import hep.dataforge.meta.Meta import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.coroutineScope import kotlinx.coroutines.withContext import kotlinx.io.nio.asInput import kotlinx.io.nio.asOutput @@ -16,71 +18,94 @@ import java.nio.file.StandardOpenOption import kotlin.reflect.KClass /** - * Read meta from file in a given [format] + * Read meta from file in a given [MetaFormat] */ -suspend fun Path.readMeta(format: MetaFormat, descriptor: NodeDescriptor? = null): Meta { - return withContext(Dispatchers.IO) { - format.run { - Files.newByteChannel(this@readMeta, StandardOpenOption.READ) - .asInput() - .readMeta(descriptor) - } - } +fun MetaFormat.readMetaFile(path: Path, descriptor: NodeDescriptor? = null): Meta { + return Files.newByteChannel(path, StandardOpenOption.READ) + .asInput() + .readMeta(descriptor) } /** - * Write meta to file in a given [format] + * Write meta to file using given [MetaFormat] */ -suspend fun Meta.write(path: Path, format: MetaFormat, descriptor: NodeDescriptor? = null) { - withContext(Dispatchers.IO) { - format.run { - Files.newByteChannel(path, StandardOpenOption.WRITE, StandardOpenOption.CREATE_NEW) - .asOutput() - .writeMeta(this@write, descriptor) - } - } +fun MetaFormat.writeMetaFile(path: Path, meta: Meta, descriptor: NodeDescriptor? = null) { + return Files.newByteChannel(path, StandardOpenOption.WRITE, StandardOpenOption.CREATE_NEW) + .asOutput() + .writeMeta(meta, descriptor) } /** * Read data with supported envelope format and binary format. If envelope format is null, then read binary directly from file. + * The operation is blocking since it must read meta header. The reading of envelope body is lazy * @param type explicit type of data read - * @param format binary format - * @param envelopeFormat the format of envelope. If null, file is read directly + * @param dataFormat binary format + * @param envelopeFormatFactory the format of envelope. If null, file is read directly * @param metaFile the relative file for optional meta override * @param metaFileFormat the meta format for override */ -suspend fun Path.readData( +fun IOPlugin.readData( + path: Path, type: KClass, - format: IOFormat, - envelopeFormat: EnvelopeFormat? = null, - metaFile: Path = resolveSibling("$fileName.meta"), - metaFileFormat: MetaFormat = JsonMetaFormat + dataFormat: IOFormat, + envelopeFormatFactory: EnvelopeFormatFactory? = null, + metaFile: Path = path.resolveSibling("${path.fileName}.meta"), + metaFileFormat: MetaFormat = JsonMetaFormat.default ): Data { - return coroutineScope { - val externalMeta = if (Files.exists(metaFile)) { - metaFile.readMeta(metaFileFormat) - } else { - null - } - if (envelopeFormat == null) { - Data(type, externalMeta ?: EmptyMeta) { - withContext(Dispatchers.IO) { - format.run { - Files.newByteChannel(this@readData, StandardOpenOption.READ) - .asInput() - .readThis() - } + val externalMeta = if (Files.exists(metaFile)) { + metaFileFormat.readMetaFile(metaFile) + } else { + null + } + return if (envelopeFormatFactory == null) { + Data(type, externalMeta ?: EmptyMeta) { + withContext(Dispatchers.IO) { + dataFormat.run { + Files.newByteChannel(path, StandardOpenOption.READ) + .asInput() + .readObject() } } - } else { - withContext(Dispatchers.IO) { - readEnvelope(envelopeFormat).let { - if (externalMeta == null) { - it - } else { - it.withMetaLayers(externalMeta) - } - }.toData(type, format) + } + } else { + readEnvelopeFile(path, envelopeFormatFactory).let { + if (externalMeta == null) { + it + } else { + it.withMetaLayers(externalMeta) + } + }.toData(type, dataFormat) + } +} + +//TODO wants multi-receiver +fun DataTreeBuilder.file( + plugin: IOPlugin, + path: Path, + dataFormat: IOFormat, + envelopeFormatFactory: EnvelopeFormatFactory? = null +) { + plugin.run { + val data = readData(path, type, dataFormat, envelopeFormatFactory) + val name = path.fileName.toString().replace(".df", "") + datum(name, data) + } +} + +/** + * Read the directory as a data node + */ +fun IOPlugin.readDataNode( + path: Path, + type: KClass, + dataFormat: IOFormat, + envelopeFormatFactory: EnvelopeFormatFactory? = null +): DataNode { + if (!Files.isDirectory(path)) error("Provided path $this is not a directory") + return DataNode(type) { + Files.list(path).forEach { path -> + if (!path.fileName.toString().endsWith(".meta")) { + file(this@readDataNode,path, dataFormat, envelopeFormatFactory) } } } diff --git a/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/DataPropagationTest.kt b/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/DataPropagationTest.kt new file mode 100644 index 00000000..c449ffc3 --- /dev/null +++ b/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/DataPropagationTest.kt @@ -0,0 +1,95 @@ +package hep.dataforge.workspace + +import hep.dataforge.context.Context +import hep.dataforge.context.PluginFactory +import hep.dataforge.context.PluginTag +import hep.dataforge.data.* +import hep.dataforge.meta.Meta +import hep.dataforge.names.asName +import org.junit.Test +import kotlin.reflect.KClass +import kotlin.test.assertEquals + + +class DataPropagationTestPlugin : WorkspacePlugin() { + override val tag: PluginTag = Companion.tag + + val testAllData = task("allData", Int::class) { + model { + allData() + } + transform { data -> + return@transform DataNode { + val result = data.dataSequence().map { it.second.get() }.reduce { acc, pair -> acc + pair } + set("result".asName(), Data { result }) + } + } + } + + + val testSingleData = task("singleData", Int::class) { + model { + data("myData\\[12\\]") + } + transform { data -> + return@transform DataNode { + val result = data.dataSequence().map { it.second.get() }.reduce { acc, pair -> acc + pair } + set("result".asName(), Data { result }) + } + } + } + + val testAllRegexData = task("allRegexData", Int::class) { + model { + data(pattern = "myData.*") + } + transform { data -> + return@transform DataNode { + val result = data.dataSequence().map { it.second.get() }.reduce { acc, pair -> acc + pair } + set("result".asName(), Data { result }) + } + } + } + + + companion object : PluginFactory { + + override val type: KClass = DataPropagationTestPlugin::class + + override fun invoke(meta: Meta, context: Context): DataPropagationTestPlugin = + DataPropagationTestPlugin(meta) + + override val tag: PluginTag = PluginTag("Test") + } +} + +class DataPropagationTest { + val testWorkspace = Workspace { + context { + plugin(DataPropagationTestPlugin()) + } + data { + repeat(100) { + static("myData[$it]", it) + } + } + } + + @Test + fun testAllData() { + val node = testWorkspace.run("Test.allData") + assertEquals(4950, node.first()!!.get()) + } + + @Test + fun testAllRegexData() { + val node = testWorkspace.run("Test.allRegexData") + assertEquals(4950, node.first()!!.get()) + } + + @Test + fun testSingleData() { + val node = testWorkspace.run("Test.singleData") + assertEquals(12, node.first()!!.get()) + } +} \ No newline at end of file diff --git a/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/SimpleWorkspaceTest.kt b/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/SimpleWorkspaceTest.kt index 51471525..3a40e783 100644 --- a/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/SimpleWorkspaceTest.kt +++ b/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/SimpleWorkspaceTest.kt @@ -1,10 +1,12 @@ package hep.dataforge.workspace import hep.dataforge.context.PluginTag -import hep.dataforge.data.first -import hep.dataforge.data.get +import hep.dataforge.data.* import hep.dataforge.meta.boolean +import hep.dataforge.meta.builder import hep.dataforge.meta.get +import hep.dataforge.meta.int +import hep.dataforge.names.plus import org.junit.Test import kotlin.test.assertEquals import kotlin.test.assertTrue @@ -14,30 +16,36 @@ class SimpleWorkspaceTest { val testPlugin = object : WorkspacePlugin() { override val tag: PluginTag = PluginTag("test") - val contextTask = Workspace.task("test") { - pipe { + val contextTask = task("test", Any::class) { + map { context.logger.info { "Test: $it" } } } - override val tasks: Collection> = listOf(contextTask) } - val workspace = SimpleWorkspace.build { + val workspace = Workspace { context { plugin(testPlugin) } - repeat(100) { - static("myData[$it]", it) + data { + repeat(100) { + static("myData[$it]", it) + } } - - task("square") { + val filterTask = task("filterOne") { model { - allData() + data("myData\\[12\\]") } - pipe { data -> + map{ + it + } + } + + val square = task("square") { + map { data -> if (meta["testFlag"].boolean == true) { println("flag") } @@ -46,30 +54,55 @@ class SimpleWorkspaceTest { } } - task("sum") { - model { - dependsOn("square") + val linear = task("linear") { + map { data -> + context.logger.info { "Starting linear on $data" } + data * 2 + 1 } - join { data -> + } + + val fullSquare = task("fullsquare") { + model { + val squareDep = dependsOn(square, placement = "square") + val linearDep = dependsOn(linear, placement = "linear") + } + transform { data -> + val squareNode = data["square"].node!!.cast()//squareDep() + val linearNode = data["linear"].node!!.cast()//linearDep() + return@transform DataNode(Int::class) { + squareNode.dataSequence().forEach { (name, _) -> + val newData = Data { + val squareValue = squareNode[name].data!!.get() + val linearValue = linearNode[name].data!!.get() + squareValue + linearValue + } + set(name, newData) + } + } + } + } + + task("sum") { + model { + dependsOn(square) + } + reduce { data -> context.logger.info { "Starting sum" } data.values.sum() } } - task("average") { - model { - allData() - } - joinByGroup { context -> + val average = task("average") { + reduceByGroup { env -> group("even", filter = { name, _ -> name.toString().toInt() % 2 == 0 }) { result { data -> - context.logger.info { "Starting even" } + env.context.logger.info { "Starting even" } data.values.average() } } group("odd", filter = { name, _ -> name.toString().toInt() % 2 == 1 }) { result { data -> - context.logger.info { "Starting odd" } + env.context.logger.info { "Starting odd" } data.values.average() } } @@ -78,13 +111,25 @@ class SimpleWorkspaceTest { task("delta") { model { - dependsOn("average") + dependsOn(average) } - join { data -> + reduce { data -> data["even"]!! - data["odd"]!! } } + val customPipeTask = task("custom") { + mapAction { + meta = meta.builder().apply { + "newValue" put 22 + } + name += "new" + result { + meta["value"].int ?: 0 + it + } + } + } + target("empty") {} } @@ -97,7 +142,7 @@ class SimpleWorkspaceTest { @Test fun testMetaPropagation() { - val node = workspace.run("sum") { "testFlag" to true } + val node = workspace.run("sum") { "testFlag" put true } val res = node.first()?.get() } @@ -107,4 +152,16 @@ class SimpleWorkspaceTest { assertTrue { tasks["test.test"] != null } //val node = workspace.run("test.test", "empty") } + + @Test + fun testFullSquare() { + val node = workspace.run("fullsquare") + println(node.toMeta()) + } + + @Test + fun testGather() { + val node = workspace.run("filterOne") + assertEquals(12, node.first()?.get()) + } } \ No newline at end of file diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index 87b738cb..5c2d1cf0 100644 Binary files a/gradle/wrapper/gradle-wrapper.jar and b/gradle/wrapper/gradle-wrapper.jar differ diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 558870da..7c4388a9 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-5.1.1-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-5.6.2-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/gradlew b/gradlew index af6708ff..83f2acfd 100644 --- a/gradlew +++ b/gradlew @@ -1,5 +1,21 @@ #!/usr/bin/env sh +# +# Copyright 2015 the original author or authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + ############################################################################## ## ## Gradle start up script for UN*X @@ -28,7 +44,7 @@ APP_NAME="Gradle" APP_BASE_NAME=`basename "$0"` # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -DEFAULT_JVM_OPTS='"-Xmx64m"' +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' # Use the maximum available, or set MAX_FD != -1 to use that value. MAX_FD="maximum" @@ -109,8 +125,8 @@ if $darwin; then GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" fi -# For Cygwin, switch paths to Windows format before running java -if $cygwin ; then +# For Cygwin or MSYS, switch paths to Windows format before running java +if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then APP_HOME=`cygpath --path --mixed "$APP_HOME"` CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` JAVACMD=`cygpath --unix "$JAVACMD"` diff --git a/gradlew.bat b/gradlew.bat index 6d57edc7..9618d8d9 100644 --- a/gradlew.bat +++ b/gradlew.bat @@ -1,3 +1,19 @@ +@rem +@rem Copyright 2015 the original author or authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem https://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem + @if "%DEBUG%" == "" @echo off @rem ########################################################################## @rem @@ -14,7 +30,7 @@ set APP_BASE_NAME=%~n0 set APP_HOME=%DIRNAME% @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -set DEFAULT_JVM_OPTS="-Xmx64m" +set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" @rem Find java.exe if defined JAVA_HOME goto findJavaFromJavaHome diff --git a/settings.gradle.kts b/settings.gradle.kts index 011123b9..b486c03f 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -12,7 +12,7 @@ pluginManagement { eachPlugin { when (requested.id.id) { "kotlinx-atomicfu" -> useModule("org.jetbrains.kotlinx:atomicfu-gradle-plugin:${requested.version}") - "scientifik.mpp", "scientifik.publish" -> useModule("scientifik:gradle-tools:${requested.version}") + "scientifik.mpp", "scientifik.jvm", "scientifik.publish" -> useModule("scientifik:gradle-tools:${requested.version}") } } } @@ -24,6 +24,7 @@ enableFeaturePreview("GRADLE_METADATA") include( ":dataforge-meta", ":dataforge-io", + ":dataforge-io:dataforge-io-yaml", ":dataforge-context", ":dataforge-data", ":dataforge-output",