From 62916d0c6710b4a89ef2beaef994a0af13280986 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Mon, 19 Aug 2019 22:42:08 +0300 Subject: [PATCH 01/48] Name and serialization fix --- build.gradle.kts | 2 +- .../commonMain/kotlin/hep/dataforge/io/MetaSerializer.kt | 6 +++++- .../src/commonMain/kotlin/hep/dataforge/names/Name.kt | 4 +--- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index 016a350e..0ebdc138 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -3,7 +3,7 @@ plugins { id("scientifik.publish") version "0.1.4" apply false } -val dataforgeVersion by extra("0.1.3") +val dataforgeVersion by extra("0.1.4-dev-1") val bintrayRepo by extra("dataforge") val githubProject by extra("dataforge-core") diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/MetaSerializer.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/MetaSerializer.kt index e1a754eb..d72dc71d 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/MetaSerializer.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/MetaSerializer.kt @@ -4,6 +4,7 @@ import hep.dataforge.meta.Config import hep.dataforge.meta.Meta import hep.dataforge.meta.toConfig import hep.dataforge.names.Name +import hep.dataforge.names.NameToken import hep.dataforge.names.toName import kotlinx.serialization.* import kotlinx.serialization.internal.StringDescriptor @@ -11,7 +12,7 @@ import kotlinx.serialization.json.JsonObjectSerializer @Serializer(Name::class) object NameSerializer : KSerializer { - override val descriptor: SerialDescriptor = StringDescriptor + override val descriptor: SerialDescriptor = StringDescriptor.withName("Name") override fun deserialize(decoder: Decoder): Name { return decoder.decodeString().toName() @@ -22,6 +23,9 @@ object NameSerializer : KSerializer { } } +@Serializer(NameToken::class) +object NameTokenSerializer: KSerializer + /** * Serialized for meta */ diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/names/Name.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/names/Name.kt index 070a8f75..0531a818 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/names/Name.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/names/Name.kt @@ -139,9 +139,7 @@ fun String.toName(): Name { * Convert the [String] to a [Name] by simply wrapping it in a single name token without parsing. * The input string could contain dots and braces, but they are just escaped, not parsed. */ -fun String.asName(): Name { - return NameToken(this).asName() -} +fun String.asName(): Name = if(isBlank()) EmptyName else NameToken(this).asName() operator fun NameToken.plus(other: Name): Name = Name(listOf(this) + other.tokens) From 7045f34c2cc950dad30f82089c89c105b53ab754 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Mon, 2 Sep 2019 19:08:28 +0300 Subject: [PATCH 02/48] Updated to build plugin 0.1.6 --- build.gradle.kts | 4 ++-- .../kotlin/hep/dataforge/io/JsonMetaFormat.kt | 2 +- .../kotlin/hep/dataforge/io/MetaSerializer.kt | 12 +++++++++++- .../hep/dataforge/workspace/SimpleWorkspace.kt | 2 +- 4 files changed, 15 insertions(+), 5 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index 0ebdc138..495d9a74 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -1,6 +1,6 @@ plugins { - id("scientifik.mpp") version "0.1.4" apply false - id("scientifik.publish") version "0.1.4" apply false + id("scientifik.mpp") version "0.1.6" apply false + id("scientifik.publish") version "0.1.6" apply false } val dataforgeVersion by extra("0.1.4-dev-1") diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/JsonMetaFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/JsonMetaFormat.kt index 64b368e2..8b3bb1d6 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/JsonMetaFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/JsonMetaFormat.kt @@ -54,7 +54,7 @@ fun Value.toJson(descriptor: ValueDescriptor? = null): JsonElement { } } -//Use theese methods to customize JSON key mapping +//Use these methods to customize JSON key mapping private fun NameToken.toJsonKey(descriptor: ItemDescriptor?) = toString() private fun NodeDescriptor?.getDescriptor(key: String) = this?.items?.get(key) diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/MetaSerializer.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/MetaSerializer.kt index d72dc71d..5f644f47 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/MetaSerializer.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/MetaSerializer.kt @@ -24,7 +24,17 @@ object NameSerializer : KSerializer { } @Serializer(NameToken::class) -object NameTokenSerializer: KSerializer +object NameTokenSerializer : KSerializer { + override val descriptor: SerialDescriptor = StringDescriptor.withName("NameToken") + + override fun deserialize(decoder: Decoder): NameToken { + return decoder.decodeString().toName().first()!! + } + + override fun serialize(encoder: Encoder, obj: NameToken) { + encoder.encodeString(obj.toString()) + } +} /** * Serialized for meta diff --git a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/SimpleWorkspace.kt b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/SimpleWorkspace.kt index 68438440..abdb70e3 100644 --- a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/SimpleWorkspace.kt +++ b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/SimpleWorkspace.kt @@ -20,7 +20,7 @@ class SimpleWorkspace( ) : Workspace { override val tasks: Map> by lazy { - context.content>(Task.TYPE) + tasks.associate { it.name.toName() to it } + context.content>(Task.TYPE) + tasks.associateBy { it.name.toName() } } companion object { From 418e1c21340a95e74c308053be183ee7825aa34a Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Wed, 4 Sep 2019 09:50:56 +0300 Subject: [PATCH 03/48] Refactored TaskBuilder to be more predictable --- .../kotlin/hep/dataforge/data/DataNode.kt | 4 ++- .../kotlin/hep/dataforge/data/checkType.kt | 10 ------- .../kotlin/hep/dataforge/data/dataJS.kt | 18 +++++++++++++ .../kotlin/hep/dataforge/data/CastDataNode.kt | 16 +---------- .../kotlin/hep/dataforge/data/dataJVM.kt | 18 +++++++++++-- .../hep/dataforge/workspace/TaskBuilder.kt | 27 +++++++++---------- 6 files changed, 51 insertions(+), 42 deletions(-) delete mode 100644 dataforge-data/src/jsMain/kotlin/hep/dataforge/data/checkType.kt create mode 100644 dataforge-data/src/jsMain/kotlin/hep/dataforge/data/dataJS.kt diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataNode.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataNode.kt index a407b512..108d9f97 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataNode.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataNode.kt @@ -227,4 +227,6 @@ fun DataNode.first(): Data? = dataSequence().first().second /** * Check that node is compatible with given type meaning that each element could be cast to the type */ -expect fun DataNode<*>.checkType(type: KClass<*>) \ No newline at end of file +expect fun DataNode<*>.checkType(type: KClass) + +//expect fun DataNode.cast(type: KClass): DataNode \ No newline at end of file diff --git a/dataforge-data/src/jsMain/kotlin/hep/dataforge/data/checkType.kt b/dataforge-data/src/jsMain/kotlin/hep/dataforge/data/checkType.kt deleted file mode 100644 index 56d1d0c9..00000000 --- a/dataforge-data/src/jsMain/kotlin/hep/dataforge/data/checkType.kt +++ /dev/null @@ -1,10 +0,0 @@ -package hep.dataforge.data - -import kotlin.reflect.KClass - -/** - * Check that node is compatible with given type meaning that each element could be cast to the type - */ -actual fun DataNode<*>.checkType(type: KClass<*>) { - //Not supported in js yet -} \ No newline at end of file diff --git a/dataforge-data/src/jsMain/kotlin/hep/dataforge/data/dataJS.kt b/dataforge-data/src/jsMain/kotlin/hep/dataforge/data/dataJS.kt new file mode 100644 index 00000000..7706efad --- /dev/null +++ b/dataforge-data/src/jsMain/kotlin/hep/dataforge/data/dataJS.kt @@ -0,0 +1,18 @@ +package hep.dataforge.data + +import kotlin.reflect.KClass + +/** + * Check that node is compatible with given type meaning that each element could be cast to the type + */ +actual fun DataNode<*>.checkType(type: KClass) { + //Not supported in js yet +} + +///** +// * Performing +// */ +//@Suppress("UNCHECKED_CAST") +//actual fun DataNode.cast(type: KClass): DataNode{ +// return this as DataNode +//} \ No newline at end of file diff --git a/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/CastDataNode.kt b/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/CastDataNode.kt index 324864fc..01938ddb 100644 --- a/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/CastDataNode.kt +++ b/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/CastDataNode.kt @@ -8,7 +8,7 @@ import kotlin.reflect.KClass import kotlin.reflect.full.isSubclassOf @Suppress("UNCHECKED_CAST") -fun Data.safeCast(type: KClass): Data? { +private fun Data.safeCast(type: KClass): Data? { return if (this.type.isSubclassOf(type)) { return object : Data { override val meta: Meta get() = this@safeCast.meta @@ -23,20 +23,6 @@ fun Data.safeCast(type: KClass): Data? { } } -/** - * Filter a node by data and node type. Resulting node and its subnodes is guaranteed to have border type [type], - * but could contain empty nodes - */ -fun DataNode.cast(type: KClass): DataNode { - return if (this is CastDataNode) { - origin.cast(type) - } else { - CastDataNode(this, type) - } -} - -inline fun DataNode.cast(): DataNode = cast(R::class) - class CastDataNode(val origin: DataNode, override val type: KClass) : DataNode { override val items: Map> by lazy { origin.items.mapNotNull { (key, item) -> diff --git a/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/dataJVM.kt b/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/dataJVM.kt index f87c4155..32bcec5f 100644 --- a/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/dataJVM.kt +++ b/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/dataJVM.kt @@ -12,8 +12,22 @@ fun Data.get(): T = runBlocking { await() } /** * Check that node is compatible with given type meaning that each element could be cast to the type */ -actual fun DataNode<*>.checkType(type: KClass<*>) { +actual fun DataNode<*>.checkType(type: KClass) { if (!type.isSuperclassOf(type)) { error("$type expected, but $type received") } -} \ No newline at end of file +} + +/** + * Filter a node by data and node type. Resulting node and its subnodes is guaranteed to have border type [type], + * but could contain empty nodes + */ +fun DataNode.cast(type: KClass): DataNode { + return if (this is CastDataNode) { + origin.cast(type) + } else { + CastDataNode(this, type) + } +} + +inline fun DataNode.cast(): DataNode = cast(R::class) \ No newline at end of file diff --git a/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/TaskBuilder.kt b/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/TaskBuilder.kt index 175c4d87..04e1efcc 100644 --- a/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/TaskBuilder.kt +++ b/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/TaskBuilder.kt @@ -6,6 +6,7 @@ import hep.dataforge.descriptors.NodeDescriptor import hep.dataforge.meta.Meta import hep.dataforge.meta.get import hep.dataforge.meta.string +import hep.dataforge.names.EmptyName import hep.dataforge.names.Name import hep.dataforge.names.toName import kotlin.reflect.KClass @@ -14,6 +15,7 @@ import kotlin.reflect.KClass class TaskBuilder(val name: String) { private var modelTransform: TaskModelBuilder.(Meta) -> Unit = { data("*") } var descriptor: NodeDescriptor? = null + private val dataTransforms: MutableList = ArrayList() /** * TODO will look better as extension class @@ -33,8 +35,6 @@ class TaskBuilder(val name: String) { } } - private val dataTransforms: MutableList = ArrayList(); - fun model(modelTransform: TaskModelBuilder.(Meta) -> Unit) { this.modelTransform = modelTransform } @@ -43,17 +43,19 @@ class TaskBuilder(val name: String) { inputType: KClass, from: String = "", to: String = "", - block: TaskModel.(Context, DataNode) -> DataNode + block: TaskEnv.(DataNode) -> DataNode ) { dataTransforms += DataTransformation(from, to) { context, model, data -> - block(model, context, data.cast(inputType)) + data.checkType(inputType) + val env = TaskEnv(EmptyName, model.meta, context) + env.block(data.cast(inputType)) } } inline fun transform( from: String = "", to: String = "", - noinline block: TaskModel.(Context, DataNode) -> DataNode + noinline block: TaskEnv.(DataNode) -> DataNode ) { transform(T::class, from, to, block) } @@ -64,10 +66,10 @@ class TaskBuilder(val name: String) { inline fun action( from: String = "", to: String = "", - crossinline block: Context.() -> Action + crossinline block: TaskEnv.() -> Action ) { - transform(from, to) { context, data: DataNode -> - block(context).invoke(data, meta) + transform(from, to) { data: DataNode -> + block().invoke(data, meta) } } @@ -82,7 +84,6 @@ class TaskBuilder(val name: String) { crossinline block: PipeBuilder.(Context) -> Unit ) { action(from, to) { - val context = this PipeAction( inputType = T::class, outputType = R::class @@ -99,7 +100,6 @@ class TaskBuilder(val name: String) { crossinline block: suspend TaskEnv.(T) -> R ) { action(from, to) { - val context = this PipeAction( inputType = T::class, outputType = R::class @@ -124,7 +124,7 @@ class TaskBuilder(val name: String) { JoinAction( inputType = T::class, outputType = R::class - ) { block(this@action) } + ) { block(context) } } } @@ -137,7 +137,6 @@ class TaskBuilder(val name: String) { crossinline block: suspend TaskEnv.(Map) -> R ) { action(from, to) { - val context = this JoinAction( inputType = T::class, outputType = R::class, @@ -164,7 +163,7 @@ class TaskBuilder(val name: String) { SplitAction( inputType = T::class, outputType = R::class - ) { block(this@action) } + ) { block(context) } } } @@ -187,7 +186,7 @@ class TaskBuilder(val name: String) { val model = this if (dataTransforms.isEmpty()) { //return data node as is - logger.warn("No transformation present, returning input data") + logger.warn { "No transformation present, returning input data" } data } else { val builder = DataTreeBuilder(Any::class) From 8bac218715b7be5e27e871dc307468ac9cc131cf Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Wed, 4 Sep 2019 22:22:29 +0300 Subject: [PATCH 04/48] Working on remote function call --- .../kotlin/hep/dataforge/io/Binary.kt | 10 ++ .../kotlin/hep/dataforge/io/Envelope.kt | 37 ++++++- .../kotlin/hep/dataforge/io/FunctionServer.kt | 38 ------- .../kotlin/hep/dataforge/io/IOFormat.kt | 35 +++++- .../kotlin/hep/dataforge/io/Responder.kt | 8 ++ .../dataforge/io/functions/FunctionServer.kt | 34 ++++++ .../dataforge/io/functions/FunctionSpec.kt | 9 ++ .../dataforge/io/functions/FunctionsPlugin.kt | 58 ++++++++++ .../io/functions/RemoteFunctionClient.kt | 100 ++++++++++++++++++ .../io/functions/RemoteFunctionServer.kt | 58 ++++++++++ .../hep/dataforge/workspace/Dependency.kt | 32 ++++-- .../hep/dataforge/workspace/TaskModel.kt | 13 +-- .../hep/dataforge/workspace/TaskBuilder.kt | 12 +-- .../workspace/SimpleWorkspaceTest.kt | 6 +- 14 files changed, 385 insertions(+), 65 deletions(-) delete mode 100644 dataforge-io/src/commonMain/kotlin/hep/dataforge/io/FunctionServer.kt create mode 100644 dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Responder.kt create mode 100644 dataforge-io/src/commonMain/kotlin/hep/dataforge/io/functions/FunctionServer.kt create mode 100644 dataforge-io/src/commonMain/kotlin/hep/dataforge/io/functions/FunctionSpec.kt create mode 100644 dataforge-io/src/commonMain/kotlin/hep/dataforge/io/functions/FunctionsPlugin.kt create mode 100644 dataforge-io/src/commonMain/kotlin/hep/dataforge/io/functions/RemoteFunctionClient.kt create mode 100644 dataforge-io/src/commonMain/kotlin/hep/dataforge/io/functions/RemoteFunctionServer.kt diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Binary.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Binary.kt index 67c962dd..811986b6 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Binary.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Binary.kt @@ -64,6 +64,16 @@ class ArrayBinary(val array: ByteArray) : RandomAccessBinary { } } +class PacketBinary(val packet: ByteReadPacket): Binary{ + override val size: ULong + get() = TODO("not implemented") //To change initializer of created properties use File | Settings | File Templates. + + override fun read(block: Input.() -> R): R { + TODO("not implemented") //To change body of created functions use File | Settings | File Templates. + } + +} + /** * Read given binary as object using given format */ diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Envelope.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Envelope.kt index c2abca21..606a5ffb 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Envelope.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Envelope.kt @@ -1,9 +1,9 @@ package hep.dataforge.io -import hep.dataforge.meta.Laminate -import hep.dataforge.meta.Meta -import hep.dataforge.meta.get -import hep.dataforge.meta.string +import hep.dataforge.meta.* +import kotlinx.io.core.Output +import kotlinx.io.core.buildPacket +import kotlinx.io.core.readBytes interface Envelope { val meta: Meta @@ -20,6 +20,10 @@ interface Envelope { const val ENVELOPE_DESCRIPTION_KEY = "$ENVELOPE_NODE.description" //const val ENVELOPE_TIME_KEY = "@envelope.time" + /** + * Build a static envelope using provided builder + */ + fun build(block: EnvelopeBuilder.() -> Unit) = EnvelopeBuilder().apply(block).build() } } @@ -63,4 +67,29 @@ fun Envelope.withMetaLayers(vararg layers: Meta): Envelope { this is ProxyEnvelope -> ProxyEnvelope(source, *layers, *this.meta.layers.toTypedArray()) else -> ProxyEnvelope(this, *layers) } +} + +class EnvelopeBuilder { + private val metaBuilder = MetaBuilder() + var data: Binary? = null + + fun meta(block: MetaBuilder.() -> Unit) { + metaBuilder.apply(block) + } + + var type by metaBuilder.string(key = Envelope.ENVELOPE_TYPE_KEY) + var dataType by metaBuilder.string(key = Envelope.ENVELOPE_DATA_TYPE_KEY) + var description by metaBuilder.string(key = Envelope.ENVELOPE_DESCRIPTION_KEY) + + /** + * Construct a binary and transform it into byte-array based buffer + */ + fun data(block: Output.() -> Unit) { + val bytes = buildPacket { + block() + } + data = ArrayBinary(bytes.readBytes()) + } + + internal fun build() = SimpleEnvelope(metaBuilder.seal(), data) } \ No newline at end of file diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/FunctionServer.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/FunctionServer.kt deleted file mode 100644 index 2b254716..00000000 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/FunctionServer.kt +++ /dev/null @@ -1,38 +0,0 @@ -package hep.dataforge.io - -import kotlin.reflect.KClass - -/** - * A descriptor for specific type of functions - */ -interface FunctionSpec { - val inputType: KClass - val outputType: KClass -} - -/** - * A server that could produce asynchronous function values - */ -interface FunctionServer { - /** - * Call a function with given name and descriptor - */ - suspend fun > call(name: String, descriptor: D, arg: T): R - - /** - * Resolve a function descriptor for given types - */ - fun resolveType(inputType: KClass, outputType: KClass): FunctionSpec - - /** - * Get a generic suspended function with given name and descriptor - */ - operator fun > get(name: String, descriptor: D): (suspend (T) -> R) = - { call(name, descriptor, it) } -} - -suspend inline fun FunctionServer.call(name: String, arg: T): R = - call(name, resolveType(T::class, R::class), arg) - -inline operator fun FunctionServer.get(name: String): (suspend (T) -> R) = - get(name, resolveType(T::class, R::class)) diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOFormat.kt index 9cc9a584..faa110f7 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOFormat.kt @@ -1,14 +1,47 @@ package hep.dataforge.io +import hep.dataforge.io.IOFormat.Companion.TYPE +import hep.dataforge.provider.Type import kotlinx.io.core.* +import kotlinx.serialization.KSerializer +import kotlinx.serialization.cbor.Cbor /** * And interface for serialization facilities */ +@Type(TYPE) interface IOFormat { fun Output.writeThis(obj: T) fun Input.readThis(): T + + companion object { + const val TYPE = "ioFormat" + } } fun IOFormat.writePacket(obj: T): ByteReadPacket = buildPacket { writeThis(obj) } -fun IOFormat.writeBytes(obj: T): ByteArray = buildPacket { writeThis(obj) }.readBytes() \ No newline at end of file +fun IOFormat.writeBytes(obj: T): ByteArray = buildPacket { writeThis(obj) }.readBytes() + + +object DoubleIOFormat: IOFormat{ + override fun Output.writeThis(obj: Double) { + writeDouble(obj) + } + override fun Input.readThis(): Double = readDouble() +} + +/** + * Experimental + */ +class SerializerIOFormat(val serializer: KSerializer) : IOFormat { + override fun Output.writeThis(obj: T) { + val bytes = Cbor.plain.dump(serializer, obj) + writeFully(bytes) + } + + override fun Input.readThis(): T { + //FIXME reads the whole input + val bytes = readBytes() + return Cbor.plain.load(serializer, bytes) + } +} \ No newline at end of file diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Responder.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Responder.kt new file mode 100644 index 00000000..e3545782 --- /dev/null +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Responder.kt @@ -0,0 +1,8 @@ +package hep.dataforge.io + +interface Responder { + /** + * Send a request and wait for response for this specific request + */ + suspend fun respond(request: Envelope): Envelope +} \ No newline at end of file diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/functions/FunctionServer.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/functions/FunctionServer.kt new file mode 100644 index 00000000..5fd81711 --- /dev/null +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/functions/FunctionServer.kt @@ -0,0 +1,34 @@ +package hep.dataforge.io + +import hep.dataforge.context.ContextAware +import hep.dataforge.io.functions.FunctionSpec + + +/** + * A server that could produce asynchronous function values + */ +interface FunctionServer : ContextAware { + /** + * Call a function with given name and descriptor + */ + suspend fun call(name: String, spec: FunctionSpec, arg: T): R + + suspend fun callMany( + name: String, + spec: FunctionSpec, + arg: List + ): List = List(arg.size) { + call(name, spec, arg[it]) + } + + /** + * Get a generic suspended function with given name and descriptor + */ + fun get( + name: String, + spec: FunctionSpec + ): (suspend (T) -> R) = + { call(name, spec, it) } +} + + diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/functions/FunctionSpec.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/functions/FunctionSpec.kt new file mode 100644 index 00000000..3ed6f18e --- /dev/null +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/functions/FunctionSpec.kt @@ -0,0 +1,9 @@ +package hep.dataforge.io.functions + +import hep.dataforge.io.IOFormat +import hep.dataforge.meta.MetaRepr + +interface FunctionSpec: MetaRepr { + val inputFormat: IOFormat + val outputFormat: IOFormat +} \ No newline at end of file diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/functions/FunctionsPlugin.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/functions/FunctionsPlugin.kt new file mode 100644 index 00000000..495a4c47 --- /dev/null +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/functions/FunctionsPlugin.kt @@ -0,0 +1,58 @@ +package hep.dataforge.io.functions + +import hep.dataforge.context.AbstractPlugin +import hep.dataforge.context.PluginFactory +import hep.dataforge.context.PluginTag +import hep.dataforge.io.DoubleIOFormat +import hep.dataforge.io.IOFormat +import hep.dataforge.io.IOPlugin +import hep.dataforge.meta.Meta +import hep.dataforge.meta.buildMeta +import kotlin.reflect.KClass + +class FunctionsPlugin(meta: Meta) : AbstractPlugin(meta) { + override val tag: PluginTag get() = Companion.tag + + override fun dependsOn(): List> = listOf(IOPlugin) + + private val specs: Collection> = listOf( + DoubleToDoubleFunctionSpec + ) + + fun resolve(meta: Meta): FunctionSpec<*, *>? { + return specs.find { it.toMeta() == meta } + } + +// fun resolve(inputType: KClass, outputType: KClass): FunctionSpec { +// +// } + + companion object : PluginFactory { + + override val tag: PluginTag = PluginTag("io.functions", group = PluginTag.DATAFORGE_GROUP) + override val type: KClass = FunctionsPlugin::class + override fun invoke(meta: Meta): FunctionsPlugin = FunctionsPlugin(meta) + } +} + +object DoubleToDoubleFunctionSpec : FunctionSpec { + override val inputFormat: IOFormat get() = DoubleIOFormat + override val outputFormat: IOFormat get() = DoubleIOFormat + + override fun toMeta(): Meta = buildMeta { + "input" to "Double" + "output" to "Double" + } +} + +//suspend inline fun FunctionServer.call(name: String, arg: T): R { +// val plugin = context.plugins.load(FunctionsPlugin) +// val spec = plugin.resolve(T::class, R::class) +// return call(name, spec, arg) +//} +// +//inline operator fun FunctionServer.get(name: String): (suspend (T) -> R) { +// val plugin = context.plugins.load(FunctionsPlugin) +// val spec = plugin.resolve(T::class, R::class) +// return get(name, spec) +//} \ No newline at end of file diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/functions/RemoteFunctionClient.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/functions/RemoteFunctionClient.kt new file mode 100644 index 00000000..b285e80d --- /dev/null +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/functions/RemoteFunctionClient.kt @@ -0,0 +1,100 @@ +package hep.dataforge.io + +import hep.dataforge.context.Context +import hep.dataforge.context.ContextAware +import hep.dataforge.io.functions.FunctionSpec +import hep.dataforge.io.functions.FunctionsPlugin +import hep.dataforge.meta.get +import hep.dataforge.meta.int +import kotlin.reflect.KClass + +class RemoteFunctionClient(override val context: Context, val responder: Responder) : FunctionServer, ContextAware { + + private fun encodeOne(name: String, spec: FunctionSpec, value: T): Envelope = + Envelope.build { + type = REQUEST_TYPE + meta { + FUNCTION_NAME_KEY to name + FUNCTION_SPEC_KEY to spec.toMeta() + } + data { + spec.inputFormat.run { + writeThis(value) + } + } + } + + private fun encodeMany(name: String, spec: FunctionSpec, values: List): Envelope = + Envelope.build { + type = REQUEST_TYPE + meta { + FUNCTION_NAME_KEY to name + FUNCTION_SPEC_KEY to spec.toMeta() + SIZE_KEY to values.size + } + data { + spec.inputFormat.run { + values.forEach { + writeThis(it) + } + } + } + } + + private fun decode(spec: FunctionSpec<*, R>, envelope: Envelope): List { + require(envelope.type == RESPONSE_TYPE) { "Unexpected message type: ${envelope.type}" } + val size = envelope.meta[SIZE_KEY].int ?: 1 + + return if (size == 0) { + emptyList() + } else { + envelope.data?.read { + List(size) { + spec.outputFormat.run { + readThis() + } + } + } ?: error("Message does not contain data") + } + } + + override suspend fun call( + name: String, + spec: FunctionSpec, + arg: T + ): R { + val request = encodeOne(name, spec, arg) + val response = responder.respond(request) + return decode(spec, response).first() + } + + override suspend fun callMany( + name: String, + spec: FunctionSpec, + arg: List + ): List { + val request = encodeMany(name, spec, arg) + val response = responder.respond(request) + return decode(spec, response) + } + + private val plugin by lazy { + context.plugins.load(FunctionsPlugin) + } + + fun resolveSpec( + inputType: KClass, + outputType: KClass + ): FunctionSpec { + return plugin.resolve(inputType, outputType) + } + + companion object { + const val REQUEST_TYPE = "function.request" + const val RESPONSE_TYPE = "function.response" + + const val FUNCTION_NAME_KEY = "function" + const val SIZE_KEY = "size" + const val FUNCTION_SPEC_KEY = "spec" + } +} \ No newline at end of file diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/functions/RemoteFunctionServer.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/functions/RemoteFunctionServer.kt new file mode 100644 index 00000000..ba93c0a0 --- /dev/null +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/functions/RemoteFunctionServer.kt @@ -0,0 +1,58 @@ +package hep.dataforge.io.functions + +import hep.dataforge.context.Context +import hep.dataforge.context.ContextAware +import hep.dataforge.io.* +import hep.dataforge.meta.get +import hep.dataforge.meta.int +import hep.dataforge.meta.node +import hep.dataforge.meta.string + +class RemoteFunctionServer( + override val context: Context, + val functionServer: FunctionServer +) : ContextAware, Responder { + + private val plugin by lazy { + context.plugins.load(FunctionsPlugin) + } + + override suspend fun respond(request: Envelope): Envelope { + require(request.type == RemoteFunctionClient.REQUEST_TYPE) { "Unexpected message type: ${request.type}" } + val functionName = request.meta[RemoteFunctionClient.FUNCTION_NAME_KEY].string ?: "" + + @Suppress("UNCHECKED_CAST") val spec = request.meta[RemoteFunctionClient.FUNCTION_SPEC_KEY].node?.let { + plugin.resolve(it) as FunctionSpec + } ?: error("Function specification not found") + + val size = request + .meta[RemoteFunctionClient.SIZE_KEY].int ?: 1 + + val input = request.data?.read { + spec.inputFormat.run { + List(size) { + readThis() + } + } + } ?: error("Input is empty") + + val output = functionServer.callMany(functionName, spec, input) + + return Envelope.build { + type = RemoteFunctionClient.RESPONSE_TYPE + meta { + RemoteFunctionClient.FUNCTION_NAME_KEY to functionName + RemoteFunctionClient.FUNCTION_SPEC_KEY to spec.toMeta() + RemoteFunctionClient.SIZE_KEY to output.size + } + data { + spec.outputFormat.run { + output.forEach { + writeThis(it) + } + } + } + + } + } +} \ No newline at end of file diff --git a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/Dependency.kt b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/Dependency.kt index 24c012ea..cb3e790e 100644 --- a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/Dependency.kt +++ b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/Dependency.kt @@ -6,10 +6,7 @@ import hep.dataforge.data.filter import hep.dataforge.meta.Meta import hep.dataforge.meta.MetaRepr import hep.dataforge.meta.buildMeta -import hep.dataforge.names.EmptyName -import hep.dataforge.names.Name -import hep.dataforge.names.get -import hep.dataforge.names.isEmpty +import hep.dataforge.names.* /** * A dependency of the task which allows to lazily create a data tree for single dependency @@ -45,12 +42,18 @@ class AllDataDependency(val placement: Name = EmptyName) : Dependency() { "data" to "*" "to" to placement } - } -class TaskModelDependency(val name: String, val meta: Meta, val placement: Name = EmptyName) : Dependency() { +abstract class TaskDependency(val meta: Meta, val placement: Name = EmptyName) : Dependency() { + abstract fun resolveTask(workspace: Workspace): Task<*> + + /** + * A name of the dependency for logging and serialization + */ + abstract val name: Name + override fun apply(workspace: Workspace): DataNode { - val task = workspace.tasks[name] ?: error("Task with name $name is not found in the workspace") + val task = resolveTask(workspace) if (task.isTerminal) TODO("Support terminal task") val result = workspace.run(task, meta) return if (placement.isEmpty()) { @@ -65,4 +68,19 @@ class TaskModelDependency(val name: String, val meta: Meta, val placement: Name "meta" to meta "to" to placement } +} + +class DirectTaskDependency(val task: Task<*>, meta: Meta, placement: Name) : TaskDependency(meta, placement) { + override fun resolveTask(workspace: Workspace): Task<*> = task + + override val name: Name get() = DIRECT_TASK_NAME + task.name + + companion object { + val DIRECT_TASK_NAME = "@direct".asName() + } +} + +class WorkspaceTaskDependency(override val name: Name, meta: Meta, placement: Name) : TaskDependency(meta, placement) { + override fun resolveTask(workspace: Workspace): Task<*> = + workspace.tasks[name] ?: error("Task with name $name is not found in the workspace") } \ No newline at end of file diff --git a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskModel.kt b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskModel.kt index b6c4b2b6..07fb6638 100644 --- a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskModel.kt +++ b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskModel.kt @@ -35,9 +35,9 @@ data class TaskModel( "meta" to meta "dependsOn" to { val dataDependencies = dependencies.filterIsInstance() - val taskDependencies = dependencies.filterIsInstance() + val taskDependencies = dependencies.filterIsInstance() setIndexed("data".toName(), dataDependencies.map { it.toMeta() }) - setIndexed("task".toName(), taskDependencies.map { it.toMeta() }) { taskDependencies[it].name } + setIndexed("task".toName(), taskDependencies.map { it.toMeta() }) { taskDependencies[it].name.toString() } //TODO ensure all dependencies are listed } } @@ -76,14 +76,15 @@ class TaskModelBuilder(val name: String, meta: Meta = EmptyMeta) { var target: String by this.meta.string(key = MODEL_TARGET_KEY, default = "") /** - * Add dependency for + * Add dependency for a task defined in a workspace and resolved by */ fun dependsOn(name: String, meta: Meta = this.meta, placement: Name = EmptyName) { - dependencies.add(TaskModelDependency(name, meta, placement)) + dependencies.add(WorkspaceTaskDependency(name.toName(), meta, placement)) } - fun dependsOn(task: Task<*>, meta: Meta = this.meta, placement: Name = EmptyName) = - dependsOn(task.name, meta, placement) + fun dependsOn(task: Task<*>, meta: Meta = this.meta, placement: Name = EmptyName) { + dependencies.add(DirectTaskDependency(task, meta, placement)) + } fun dependsOn(task: Task<*>, placement: Name = EmptyName, metaBuilder: MetaBuilder.() -> Unit) = dependsOn(task.name, buildMeta(metaBuilder), placement) diff --git a/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/TaskBuilder.kt b/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/TaskBuilder.kt index 04e1efcc..c35003d3 100644 --- a/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/TaskBuilder.kt +++ b/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/TaskBuilder.kt @@ -81,13 +81,13 @@ class TaskBuilder(val name: String) { inline fun customPipe( from: String = "", to: String = "", - crossinline block: PipeBuilder.(Context) -> Unit + crossinline block: PipeBuilder.(TaskEnv) -> Unit ) { action(from, to) { PipeAction( inputType = T::class, outputType = R::class - ) { block(context) } + ) { block(this@action) } } } @@ -118,13 +118,13 @@ class TaskBuilder(val name: String) { inline fun joinByGroup( from: String = "", to: String = "", - crossinline block: JoinGroupBuilder.(Context) -> Unit + crossinline block: JoinGroupBuilder.(TaskEnv) -> Unit ) { action(from, to) { JoinAction( inputType = T::class, outputType = R::class - ) { block(context) } + ) { block(this@action) } } } @@ -157,13 +157,13 @@ class TaskBuilder(val name: String) { inline fun split( from: String = "", to: String = "", - crossinline block: SplitBuilder.(Context) -> Unit + crossinline block: SplitBuilder.(TaskEnv) -> Unit ) { action(from, to) { SplitAction( inputType = T::class, outputType = R::class - ) { block(context) } + ) { block(this@action) } } } diff --git a/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/SimpleWorkspaceTest.kt b/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/SimpleWorkspaceTest.kt index 51471525..b64ee467 100644 --- a/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/SimpleWorkspaceTest.kt +++ b/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/SimpleWorkspaceTest.kt @@ -60,16 +60,16 @@ class SimpleWorkspaceTest { model { allData() } - joinByGroup { context -> + joinByGroup {env-> group("even", filter = { name, _ -> name.toString().toInt() % 2 == 0 }) { result { data -> - context.logger.info { "Starting even" } + env.context.logger.info { "Starting even" } data.values.average() } } group("odd", filter = { name, _ -> name.toString().toInt() % 2 == 1 }) { result { data -> - context.logger.info { "Starting odd" } + env.context.logger.info { "Starting odd" } data.values.average() } } From 10b83853240304f24d7c4aa1010a2b66a54ed432 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Sun, 8 Sep 2019 15:11:46 +0300 Subject: [PATCH 05/48] Named return Name instead of String --- .../hep/dataforge/context/AbstractPlugin.kt | 25 ++++- .../kotlin/hep/dataforge/context/Context.kt | 16 +-- .../hep/dataforge/context/ContextBuilder.kt | 5 +- .../kotlin/hep/dataforge/context/Named.kt | 13 ++- .../kotlin/hep/dataforge/context/Plugin.kt | 6 +- .../hep/dataforge/context/ContextTest.kt | 3 +- .../kotlin/hep/dataforge/io/Binary.kt | 10 +- .../hep/dataforge/io/BinaryMetaFormat.kt | 8 +- .../kotlin/hep/dataforge/io/Envelope.kt | 4 + .../kotlin/hep/dataforge/io/EnvelopeFormat.kt | 6 ++ .../kotlin/hep/dataforge/io/IOFormat.kt | 51 ++++++++- .../kotlin/hep/dataforge/io/IOPlugin.kt | 26 ++++- .../kotlin/hep/dataforge/io/JsonMetaFormat.kt | 4 +- .../kotlin/hep/dataforge/io/MetaFormat.kt | 9 +- .../hep/dataforge/io/TaggedEnvelopeFormat.kt | 4 +- .../dataforge/io/functions/FunctionServer.kt | 43 ++++++-- .../dataforge/io/functions/FunctionSpec.kt | 9 -- .../dataforge/io/functions/FunctionsPlugin.kt | 58 ---------- .../io/functions/RemoteFunctionClient.kt | 101 ++++++++---------- .../io/functions/RemoteFunctionServer.kt | 34 +++--- .../kotlin/hep/dataforge/io/FileBinary.kt | 4 +- .../kotlin/hep/dataforge/io/ioFormatsJVM.kt | 34 ++++++ .../hep/dataforge/io/tcp/EnvelopeClient.kt | 61 +++++++++++ .../hep/dataforge/io/tcp/EnvelopeServer.kt | 70 ++++++++++++ .../dataforge/io/tcp/EnvelopeServerTest.kt | 58 ++++++++++ .../hep/dataforge/workspace/GenericTask.kt | 5 +- .../dataforge/workspace/SimpleWorkspace.kt | 4 +- .../hep/dataforge/workspace/TaskModel.kt | 11 +- .../dataforge/workspace/WorkspacePlugin.kt | 4 +- .../hep/dataforge/workspace/TaskBuilder.kt | 3 +- gradle/wrapper/gradle-wrapper.jar | Bin 55190 -> 55616 bytes gradle/wrapper/gradle-wrapper.properties | 2 +- gradlew | 22 +++- gradlew.bat | 18 +++- 34 files changed, 523 insertions(+), 208 deletions(-) delete mode 100644 dataforge-io/src/commonMain/kotlin/hep/dataforge/io/functions/FunctionSpec.kt delete mode 100644 dataforge-io/src/commonMain/kotlin/hep/dataforge/io/functions/FunctionsPlugin.kt create mode 100644 dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/ioFormatsJVM.kt create mode 100644 dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/tcp/EnvelopeClient.kt create mode 100644 dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/tcp/EnvelopeServer.kt create mode 100644 dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/tcp/EnvelopeServerTest.kt diff --git a/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/AbstractPlugin.kt b/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/AbstractPlugin.kt index c9268790..232c7cd4 100644 --- a/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/AbstractPlugin.kt +++ b/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/AbstractPlugin.kt @@ -3,10 +3,13 @@ package hep.dataforge.context import hep.dataforge.meta.EmptyMeta import hep.dataforge.meta.Meta import hep.dataforge.names.Name -import hep.dataforge.names.toName +import kotlin.properties.ReadOnlyProperty +import kotlin.reflect.KClass +import kotlin.reflect.KProperty abstract class AbstractPlugin(override val meta: Meta = EmptyMeta) : Plugin { private var _context: Context? = null + private val dependencies = ArrayList>() override val context: Context get() = _context ?: error("Plugin $tag is not attached") @@ -19,9 +22,23 @@ abstract class AbstractPlugin(override val meta: Meta = EmptyMeta) : Plugin { this._context = null } - override fun provideTop(target: String): Map = emptyMap() + final override fun dependsOn(): List> = dependencies - companion object{ - fun Collection.toMap(): Map = associate { it.name.toName() to it } + /** + * Register plugin dependency and return a delegate which provides lazily initialized reference to dependent plugin + */ + protected fun

require(factory: PluginFactory

): ReadOnlyProperty { + dependencies.add(factory) + return PluginDependencyDelegate(factory.type) + } + + override fun provideTop(target: String): Map = emptyMap() +} + +fun Collection.toMap(): Map = associate { it.name to it } + +private class PluginDependencyDelegate

(val type: KClass) : ReadOnlyProperty { + override fun getValue(thisRef: AbstractPlugin, property: KProperty<*>): P { + return thisRef.context.plugins[type] ?: error("Plugin with type $type not found") } } \ No newline at end of file diff --git a/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/Context.kt b/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/Context.kt index 80746456..cc79ec44 100644 --- a/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/Context.kt +++ b/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/Context.kt @@ -2,8 +2,8 @@ package hep.dataforge.context import hep.dataforge.meta.* import hep.dataforge.names.Name -import hep.dataforge.names.appendLeft -import hep.dataforge.names.toName +import hep.dataforge.names.asName +import hep.dataforge.names.plus import hep.dataforge.provider.Provider import hep.dataforge.provider.top import hep.dataforge.values.Value @@ -27,7 +27,7 @@ import kotlin.jvm.JvmName * @author Alexander Nozik */ open class Context( - final override val name: String, + final override val name: Name, val parent: Context? = Global ) : Named, MetaRepr, Provider, CoroutineScope { @@ -45,7 +45,7 @@ open class Context( /** * Context logger */ - val logger: KLogger = KotlinLogging.logger(name) + val logger: KLogger = KotlinLogging.logger(name.toString()) /** * A [PluginManager] for current context @@ -64,7 +64,7 @@ open class Context( override fun provideTop(target: String): Map { return when (target) { Value.TYPE -> properties.sequence().toMap() - Plugin.PLUGIN_TARGET -> plugins.sequence(true).associateBy { it.name.toName() } + Plugin.PLUGIN_TARGET -> plugins.sequence(true).associateBy { it.name } else -> emptyMap() } } @@ -118,14 +118,14 @@ fun Context.content(target: String): Map = content(target) @JvmName("typedContent") inline fun Context.content(target: String): Map = plugins.flatMap { plugin -> - plugin.top(target).entries.map { (it.key.appendLeft(plugin.name)) to it.value } + plugin.top(target).entries.map { (plugin.name + it.key) to it.value } }.associate { it } /** * A global root context. Closing [Global] terminates the framework. */ -object Global : Context("GLOBAL", null) { +object Global : Context("GLOBAL".asName(), null) { /** * Closing all contexts * @@ -173,7 +173,7 @@ interface ContextAware { val logger: KLogger get() = if (this is Named) { - KotlinLogging.logger(context.name + "." + (this as Named).name) + KotlinLogging.logger((context.name + this.name).toString()) } else { context.logger } diff --git a/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/ContextBuilder.kt b/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/ContextBuilder.kt index 92840862..58a03554 100644 --- a/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/ContextBuilder.kt +++ b/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/ContextBuilder.kt @@ -2,11 +2,12 @@ package hep.dataforge.context import hep.dataforge.meta.MetaBuilder import hep.dataforge.meta.buildMeta +import hep.dataforge.names.toName /** * A convenience builder for context */ -class ContextBuilder(var name: String = "@anonimous", val parent: Context = Global) { +class ContextBuilder(var name: String = "@anonymous", val parent: Context = Global) { private val plugins = ArrayList() private var meta = MetaBuilder() @@ -31,7 +32,7 @@ class ContextBuilder(var name: String = "@anonimous", val parent: Context = Glob } fun build(): Context { - return Context(name, parent).apply { + return Context(name.toName(), parent).apply { this@ContextBuilder.plugins.forEach { plugins.load(it) } diff --git a/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/Named.kt b/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/Named.kt index 1ac31702..9e9db17f 100644 --- a/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/Named.kt +++ b/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/Named.kt @@ -15,6 +15,10 @@ */ package hep.dataforge.context +import hep.dataforge.names.Name +import hep.dataforge.names.asName +import hep.dataforge.names.isEmpty + /** * Any object that have name * @@ -27,10 +31,9 @@ interface Named { * * @return */ - val name: String + val name: Name companion object { - const val ANONYMOUS = "" /** * Get the name of given object. If object is Named its name is used, @@ -39,11 +42,11 @@ interface Named { * @param obj * @return */ - fun nameOf(obj: Any): String { + fun nameOf(obj: Any): Name { return if (obj is Named) { obj.name } else { - obj.toString() + obj.toString().asName() } } } @@ -54,4 +57,4 @@ interface Named { * @return */ val Named.isAnonymous: Boolean - get() = this.name == Named.ANONYMOUS + get() = this.name.isEmpty() diff --git a/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/Plugin.kt b/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/Plugin.kt index e06bf3f9..d795d881 100644 --- a/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/Plugin.kt +++ b/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/Plugin.kt @@ -3,6 +3,8 @@ package hep.dataforge.context import hep.dataforge.meta.Meta import hep.dataforge.meta.MetaRepr import hep.dataforge.meta.buildMeta +import hep.dataforge.names.Name +import hep.dataforge.names.toName import hep.dataforge.provider.Provider /** @@ -37,7 +39,7 @@ interface Plugin : Named, ContextAware, Provider, MetaRepr { * * @return */ - override val name: String get() = tag.name + override val name: Name get() = tag.name.toName() /** * Plugin dependencies which are required to attach this plugin. Plugin @@ -46,7 +48,7 @@ interface Plugin : Named, ContextAware, Provider, MetaRepr { * * @return */ - fun dependsOn(): List> = emptyList() + fun dependsOn(): Collection> /** * Start this plugin and attach registration info to the context. This method diff --git a/dataforge-context/src/commonTest/kotlin/hep/dataforge/context/ContextTest.kt b/dataforge-context/src/commonTest/kotlin/hep/dataforge/context/ContextTest.kt index c77439d6..584e500c 100644 --- a/dataforge-context/src/commonTest/kotlin/hep/dataforge/context/ContextTest.kt +++ b/dataforge-context/src/commonTest/kotlin/hep/dataforge/context/ContextTest.kt @@ -5,7 +5,6 @@ import hep.dataforge.names.appendLeft import hep.dataforge.names.toName import kotlin.test.Test import kotlin.test.assertEquals -import kotlin.test.assertTrue class ContextTest { @@ -26,7 +25,7 @@ class ContextTest { val members = Global.content("test") assertEquals(3, members.count()) members.forEach { - assertTrue{it.key == it.value.appendLeft("test")} + assertEquals(it.key, it.value.appendLeft("test")) } } diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Binary.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Binary.kt index 811986b6..da62113b 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Binary.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Binary.kt @@ -4,6 +4,7 @@ import kotlinx.io.core.ByteReadPacket import kotlinx.io.core.Input import kotlinx.io.core.buildPacket import kotlinx.io.core.readBytes +import kotlin.math.min /** * A source of binary data @@ -30,6 +31,8 @@ interface RandomAccessBinary : Binary { /** * Read at most [size] of bytes starting at [from] offset from the beginning of the binary. * This method could be called multiple times simultaneously. + * + * If size */ fun read(from: UInt, size: UInt = UInt.MAX_VALUE, block: Input.() -> R): R @@ -60,11 +63,12 @@ class ArrayBinary(val array: ByteArray) : RandomAccessBinary { override val size: ULong get() = array.size.toULong() override fun read(from: UInt, size: UInt, block: Input.() -> R): R { - return ByteReadPacket(array, from.toInt(), size.toInt()).block() + val theSize = min(size, array.size.toUInt() - from) + return ByteReadPacket(array, from.toInt(), theSize.toInt()).block() } } -class PacketBinary(val packet: ByteReadPacket): Binary{ +class PacketBinary(val packet: ByteReadPacket) : Binary { override val size: ULong get() = TODO("not implemented") //To change initializer of created properties use File | Settings | File Templates. @@ -87,7 +91,7 @@ fun Binary.readWith(format: IOFormat): T = format.run { * Write this object to a binary * TODO make a lazy binary that does not use intermediate array */ -fun T.writeWith(format: IOFormat): Binary = format.run{ +fun T.writeWith(format: IOFormat): Binary = format.run { val packet = buildPacket { writeThis(this@writeWith) } diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/BinaryMetaFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/BinaryMetaFormat.kt index daf08756..8774387b 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/BinaryMetaFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/BinaryMetaFormat.kt @@ -2,6 +2,8 @@ package hep.dataforge.io import hep.dataforge.descriptors.NodeDescriptor import hep.dataforge.meta.* +import hep.dataforge.names.Name +import hep.dataforge.names.plus import hep.dataforge.values.* import kotlinx.io.core.Input import kotlinx.io.core.Output @@ -9,7 +11,7 @@ import kotlinx.io.core.readText import kotlinx.io.core.writeText object BinaryMetaFormat : MetaFormat { - override val name: String = "bin" + override val name: Name = super.name + "bin" override val key: Short = 0x4249//BI override fun Input.readMeta(descriptor: NodeDescriptor?): Meta { @@ -23,7 +25,7 @@ object BinaryMetaFormat : MetaFormat { writeText(str) } - private fun Output.writeValue(value: Value) { + fun Output.writeValue(value: Value) { if (value.isList()) { writeChar('L') writeInt(value.list.size) @@ -92,7 +94,7 @@ object BinaryMetaFormat : MetaFormat { } @Suppress("UNCHECKED_CAST") - private fun Input.readMetaItem(): MetaItem { + fun Input.readMetaItem(): MetaItem { return when (val keyChar = readByte().toChar()) { 'S' -> MetaItem.ValueItem(StringValue(readString())) 'N' -> MetaItem.ValueItem(Null) diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Envelope.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Envelope.kt index 606a5ffb..36178d98 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Envelope.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Envelope.kt @@ -77,6 +77,10 @@ class EnvelopeBuilder { metaBuilder.apply(block) } + fun meta(meta: Meta) { + metaBuilder.update(meta) + } + var type by metaBuilder.string(key = Envelope.ENVELOPE_TYPE_KEY) var dataType by metaBuilder.string(key = Envelope.ENVELOPE_DATA_TYPE_KEY) var description by metaBuilder.string(key = Envelope.ENVELOPE_DESCRIPTION_KEY) diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/EnvelopeFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/EnvelopeFormat.kt index 24217e14..a0734c04 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/EnvelopeFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/EnvelopeFormat.kt @@ -3,9 +3,12 @@ package hep.dataforge.io import hep.dataforge.context.Named import hep.dataforge.io.EnvelopeFormat.Companion.ENVELOPE_FORMAT_TYPE import hep.dataforge.meta.Meta +import hep.dataforge.names.Name +import hep.dataforge.names.asName import hep.dataforge.provider.Type import kotlinx.io.core.Input import kotlinx.io.core.Output +import kotlin.reflect.KClass /** * A partially read envelope with meta, but without data @@ -15,6 +18,9 @@ data class PartialEnvelope(val meta: Meta, val dataOffset: UInt, val dataSize: U @Type(ENVELOPE_FORMAT_TYPE) interface EnvelopeFormat : IOFormat, Named { + override val name: Name get() = "envelope".asName() + override val type: KClass get() = Envelope::class + fun Input.readPartial(formats: Collection = IOPlugin.defaultMetaFormats): PartialEnvelope fun Input.readEnvelope(formats: Collection = IOPlugin.defaultMetaFormats): Envelope diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOFormat.kt index faa110f7..0ebf9485 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOFormat.kt @@ -1,16 +1,31 @@ package hep.dataforge.io +import hep.dataforge.context.Named import hep.dataforge.io.IOFormat.Companion.TYPE +import hep.dataforge.meta.MetaItem +import hep.dataforge.names.EmptyName +import hep.dataforge.names.Name +import hep.dataforge.names.asName +import hep.dataforge.names.toName import hep.dataforge.provider.Type +import hep.dataforge.values.Value import kotlinx.io.core.* +import kotlinx.serialization.ImplicitReflectionSerializer import kotlinx.serialization.KSerializer import kotlinx.serialization.cbor.Cbor +import kotlinx.serialization.serializer +import kotlin.reflect.KClass /** * And interface for serialization facilities */ @Type(TYPE) -interface IOFormat { +interface IOFormat : Named { + /** + * Explicit type for dynamic type checks + */ + val type: KClass + fun Output.writeThis(obj: T) fun Input.readThis(): T @@ -23,17 +38,47 @@ fun IOFormat.writePacket(obj: T): ByteReadPacket = buildPacket { wr fun IOFormat.writeBytes(obj: T): ByteArray = buildPacket { writeThis(obj) }.readBytes() -object DoubleIOFormat: IOFormat{ +object DoubleIOFormat : IOFormat { + + override val name: Name = "double".asName() + + override val type: KClass get() = Double::class + override fun Output.writeThis(obj: Double) { writeDouble(obj) } + override fun Input.readThis(): Double = readDouble() } +object ValueIOFormat : IOFormat { + + override val name: Name = "value".asName() + + override val type: KClass get() = Value::class + + override fun Output.writeThis(obj: Value) { + BinaryMetaFormat.run { writeValue(obj) } + } + + override fun Input.readThis(): Value { + return (BinaryMetaFormat.run { readMetaItem() } as? MetaItem.ValueItem)?.value + ?: error("The item is not a value") + } +} + /** * Experimental */ -class SerializerIOFormat(val serializer: KSerializer) : IOFormat { +@ImplicitReflectionSerializer +class SerializerIOFormat( + override val type: KClass, + val serializer: KSerializer = type.serializer() +) : IOFormat { + + override val name: Name = type.simpleName?.toName() ?: EmptyName + + override fun Output.writeThis(obj: T) { val bytes = Cbor.plain.dump(serializer, obj) writeFully(bytes) diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOPlugin.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOPlugin.kt index b65e4982..2dbd2b88 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOPlugin.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOPlugin.kt @@ -1,11 +1,11 @@ package hep.dataforge.io -import hep.dataforge.context.AbstractPlugin -import hep.dataforge.context.PluginFactory -import hep.dataforge.context.PluginTag -import hep.dataforge.context.content +import hep.dataforge.context.* import hep.dataforge.meta.Meta +import hep.dataforge.meta.MetaItem +import hep.dataforge.meta.string import hep.dataforge.names.Name +import hep.dataforge.names.get import kotlin.reflect.KClass class IOPlugin(meta: Meta) : AbstractPlugin(meta) { @@ -16,20 +16,36 @@ class IOPlugin(meta: Meta) : AbstractPlugin(meta) { } fun metaFormat(key: Short): MetaFormat? = metaFormats.find { it.key == key } - fun metaFormat(name: String): MetaFormat? = metaFormats.find { it.name == name } + fun metaFormat(name: String): MetaFormat? = metaFormats.find { it.name.toString() == name } override fun provideTop(target: String): Map { return when (target) { MetaFormat.META_FORMAT_TYPE -> defaultMetaFormats.toMap() EnvelopeFormat.ENVELOPE_FORMAT_TYPE -> defaultEnvelopeFormats.toMap() + IOFormat.TYPE -> defaultIOFormats.toMap() else -> super.provideTop(target) } } + val ioFormats: Map> by lazy { + context.content>(IOFormat.TYPE) + } + + fun resolveIOFormat(item: MetaItem<*>): IOFormat<*>? { + val key = item.string ?: error("Not a string value!") + return ioFormats[key] + } + companion object : PluginFactory { val defaultMetaFormats: List = listOf(JsonMetaFormat, BinaryMetaFormat) val defaultEnvelopeFormats = listOf(TaggedEnvelopeFormat) + val defaultIOFormats = listOf( + DoubleIOFormat, + ValueIOFormat, + BinaryMetaFormat + ) + override val tag: PluginTag = PluginTag("io", group = PluginTag.DATAFORGE_GROUP) override val type: KClass = IOPlugin::class override fun invoke(meta: Meta): IOPlugin = IOPlugin(meta) diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/JsonMetaFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/JsonMetaFormat.kt index 8b3bb1d6..41946768 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/JsonMetaFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/JsonMetaFormat.kt @@ -6,7 +6,9 @@ import hep.dataforge.descriptors.ValueDescriptor import hep.dataforge.meta.Meta import hep.dataforge.meta.MetaBase import hep.dataforge.meta.MetaItem +import hep.dataforge.names.Name import hep.dataforge.names.NameToken +import hep.dataforge.names.plus import hep.dataforge.names.toName import hep.dataforge.values.* import kotlinx.io.core.Input @@ -21,7 +23,7 @@ import kotlin.collections.set object JsonMetaFormat : MetaFormat { - override val name: String = "json" + override val name: Name = super.name + "json" override val key: Short = 0x4a53//"JS" override fun Output.writeMeta(meta: Meta, descriptor: NodeDescriptor?) { diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/MetaFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/MetaFormat.kt index 3185e29e..c208701d 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/MetaFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/MetaFormat.kt @@ -4,17 +4,22 @@ import hep.dataforge.context.Named import hep.dataforge.descriptors.NodeDescriptor import hep.dataforge.io.MetaFormat.Companion.META_FORMAT_TYPE import hep.dataforge.meta.Meta +import hep.dataforge.names.Name +import hep.dataforge.names.asName import hep.dataforge.provider.Type import kotlinx.io.core.* +import kotlin.reflect.KClass /** * A format for meta serialization */ @Type(META_FORMAT_TYPE) interface MetaFormat : IOFormat, Named { - override val name: String + override val name: Name get() = "meta".asName() val key: Short + override val type: KClass get() = Meta::class + override fun Output.writeThis(obj: Meta) { writeMeta(obj, null) } @@ -24,7 +29,7 @@ interface MetaFormat : IOFormat, Named { fun Output.writeMeta(meta: Meta, descriptor: NodeDescriptor? = null) fun Input.readMeta(descriptor: NodeDescriptor? = null): Meta - companion object{ + companion object { const val META_FORMAT_TYPE = "metaFormat" } } diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt index 291f539d..47e8ab5d 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt @@ -1,5 +1,7 @@ package hep.dataforge.io +import hep.dataforge.names.Name +import hep.dataforge.names.plus import kotlinx.io.core.* @@ -10,7 +12,7 @@ object TaggedEnvelopeFormat : EnvelopeFormat { private const val END_SEQUENCE = "~#\r\n" private const val TAG_SIZE = 26u - override val name: String get() = VERSION + override val name: Name = super.name + VERSION private fun Tag.toBytes(): ByteReadPacket = buildPacket(24) { writeText(START_SEQUENCE) diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/functions/FunctionServer.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/functions/FunctionServer.kt index 5fd81711..872a5989 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/functions/FunctionServer.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/functions/FunctionServer.kt @@ -1,7 +1,12 @@ -package hep.dataforge.io +package hep.dataforge.io.functions import hep.dataforge.context.ContextAware -import hep.dataforge.io.functions.FunctionSpec +import hep.dataforge.io.IOFormat +import hep.dataforge.io.IOPlugin +import hep.dataforge.meta.Meta +import hep.dataforge.meta.get +import hep.dataforge.names.asName +import hep.dataforge.names.plus /** @@ -11,24 +16,40 @@ interface FunctionServer : ContextAware { /** * Call a function with given name and descriptor */ - suspend fun call(name: String, spec: FunctionSpec, arg: T): R + suspend fun call(meta: Meta, arg: T): R suspend fun callMany( - name: String, - spec: FunctionSpec, + meta: Meta, arg: List ): List = List(arg.size) { - call(name, spec, arg[it]) + call(meta, arg[it]) } /** * Get a generic suspended function with given name and descriptor */ - fun get( - name: String, - spec: FunctionSpec - ): (suspend (T) -> R) = - { call(name, spec, it) } + fun function( + meta: Meta + ): (suspend (T) -> R) = { call(meta, it) } + + companion object { + const val FUNCTION_NAME_KEY = "function" + val FORMAT_KEY = "format".asName() + val INPUT_FORMAT_KEY = FORMAT_KEY + "input" + val OUTPUT_FORMAT_KEY = FORMAT_KEY + "output" + } +} + +fun IOPlugin.getInputFormat(meta: Meta): IOFormat { + return meta[FunctionServer.INPUT_FORMAT_KEY]?.let { + resolveIOFormat(it) as IOFormat + } ?: error("Input format not resolved") +} + +fun IOPlugin.getOutputFormat(meta: Meta): IOFormat { + return meta[FunctionServer.OUTPUT_FORMAT_KEY]?.let { + resolveIOFormat(it) as IOFormat + } ?: error("Input format not resolved") } diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/functions/FunctionSpec.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/functions/FunctionSpec.kt deleted file mode 100644 index 3ed6f18e..00000000 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/functions/FunctionSpec.kt +++ /dev/null @@ -1,9 +0,0 @@ -package hep.dataforge.io.functions - -import hep.dataforge.io.IOFormat -import hep.dataforge.meta.MetaRepr - -interface FunctionSpec: MetaRepr { - val inputFormat: IOFormat - val outputFormat: IOFormat -} \ No newline at end of file diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/functions/FunctionsPlugin.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/functions/FunctionsPlugin.kt deleted file mode 100644 index 495a4c47..00000000 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/functions/FunctionsPlugin.kt +++ /dev/null @@ -1,58 +0,0 @@ -package hep.dataforge.io.functions - -import hep.dataforge.context.AbstractPlugin -import hep.dataforge.context.PluginFactory -import hep.dataforge.context.PluginTag -import hep.dataforge.io.DoubleIOFormat -import hep.dataforge.io.IOFormat -import hep.dataforge.io.IOPlugin -import hep.dataforge.meta.Meta -import hep.dataforge.meta.buildMeta -import kotlin.reflect.KClass - -class FunctionsPlugin(meta: Meta) : AbstractPlugin(meta) { - override val tag: PluginTag get() = Companion.tag - - override fun dependsOn(): List> = listOf(IOPlugin) - - private val specs: Collection> = listOf( - DoubleToDoubleFunctionSpec - ) - - fun resolve(meta: Meta): FunctionSpec<*, *>? { - return specs.find { it.toMeta() == meta } - } - -// fun resolve(inputType: KClass, outputType: KClass): FunctionSpec { -// -// } - - companion object : PluginFactory { - - override val tag: PluginTag = PluginTag("io.functions", group = PluginTag.DATAFORGE_GROUP) - override val type: KClass = FunctionsPlugin::class - override fun invoke(meta: Meta): FunctionsPlugin = FunctionsPlugin(meta) - } -} - -object DoubleToDoubleFunctionSpec : FunctionSpec { - override val inputFormat: IOFormat get() = DoubleIOFormat - override val outputFormat: IOFormat get() = DoubleIOFormat - - override fun toMeta(): Meta = buildMeta { - "input" to "Double" - "output" to "Double" - } -} - -//suspend inline fun FunctionServer.call(name: String, arg: T): R { -// val plugin = context.plugins.load(FunctionsPlugin) -// val spec = plugin.resolve(T::class, R::class) -// return call(name, spec, arg) -//} -// -//inline operator fun FunctionServer.get(name: String): (suspend (T) -> R) { -// val plugin = context.plugins.load(FunctionsPlugin) -// val spec = plugin.resolve(T::class, R::class) -// return get(name, spec) -//} \ No newline at end of file diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/functions/RemoteFunctionClient.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/functions/RemoteFunctionClient.kt index b285e80d..0ba90282 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/functions/RemoteFunctionClient.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/functions/RemoteFunctionClient.kt @@ -1,56 +1,58 @@ -package hep.dataforge.io +package hep.dataforge.io.functions import hep.dataforge.context.Context import hep.dataforge.context.ContextAware -import hep.dataforge.io.functions.FunctionSpec -import hep.dataforge.io.functions.FunctionsPlugin +import hep.dataforge.io.* +import hep.dataforge.meta.Meta import hep.dataforge.meta.get import hep.dataforge.meta.int -import kotlin.reflect.KClass class RemoteFunctionClient(override val context: Context, val responder: Responder) : FunctionServer, ContextAware { - private fun encodeOne(name: String, spec: FunctionSpec, value: T): Envelope = - Envelope.build { - type = REQUEST_TYPE - meta { - FUNCTION_NAME_KEY to name - FUNCTION_SPEC_KEY to spec.toMeta() + private fun IOPlugin.encodeOne( + meta: Meta, + value: T + ): Envelope = Envelope.build { + meta(meta) + type = REQUEST_TYPE + data { + val inputFormat: IOFormat = getInputFormat(meta) + inputFormat.run { + writeThis(value) } - data { - spec.inputFormat.run { - writeThis(value) + } + } + + private fun IOPlugin.encodeMany( + meta: Meta, + values: List + ): Envelope = Envelope.build { + meta(meta) + type = REQUEST_TYPE + meta { + SIZE_KEY to values.size + } + data { + val inputFormat: IOFormat = getInputFormat(meta) + inputFormat.run { + values.forEach { + writeThis(it) } } } + } - private fun encodeMany(name: String, spec: FunctionSpec, values: List): Envelope = - Envelope.build { - type = REQUEST_TYPE - meta { - FUNCTION_NAME_KEY to name - FUNCTION_SPEC_KEY to spec.toMeta() - SIZE_KEY to values.size - } - data { - spec.inputFormat.run { - values.forEach { - writeThis(it) - } - } - } - } - - private fun decode(spec: FunctionSpec<*, R>, envelope: Envelope): List { + private fun IOPlugin.decode(envelope: Envelope): List { require(envelope.type == RESPONSE_TYPE) { "Unexpected message type: ${envelope.type}" } val size = envelope.meta[SIZE_KEY].int ?: 1 return if (size == 0) { emptyList() } else { + val outputFormat: IOFormat = getOutputFormat(envelope.meta) envelope.data?.read { List(size) { - spec.outputFormat.run { + outputFormat.run { readThis() } } @@ -58,43 +60,32 @@ class RemoteFunctionClient(override val context: Context, val responder: Respond } } + private val plugin by lazy { + context.plugins.load(IOPlugin) + } + override suspend fun call( - name: String, - spec: FunctionSpec, + meta: Meta, arg: T - ): R { - val request = encodeOne(name, spec, arg) + ): R = plugin.run { + val request = encodeOne(meta, arg) val response = responder.respond(request) - return decode(spec, response).first() + return decode(response).first() } override suspend fun callMany( - name: String, - spec: FunctionSpec, + meta: Meta, arg: List - ): List { - val request = encodeMany(name, spec, arg) + ): List = plugin.run { + val request = encodeMany(meta, arg) val response = responder.respond(request) - return decode(spec, response) - } - - private val plugin by lazy { - context.plugins.load(FunctionsPlugin) - } - - fun resolveSpec( - inputType: KClass, - outputType: KClass - ): FunctionSpec { - return plugin.resolve(inputType, outputType) + return decode(response) } companion object { const val REQUEST_TYPE = "function.request" const val RESPONSE_TYPE = "function.response" - const val FUNCTION_NAME_KEY = "function" const val SIZE_KEY = "size" - const val FUNCTION_SPEC_KEY = "spec" } } \ No newline at end of file diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/functions/RemoteFunctionServer.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/functions/RemoteFunctionServer.kt index ba93c0a0..41a2a271 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/functions/RemoteFunctionServer.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/functions/RemoteFunctionServer.kt @@ -2,11 +2,12 @@ package hep.dataforge.io.functions import hep.dataforge.context.Context import hep.dataforge.context.ContextAware -import hep.dataforge.io.* +import hep.dataforge.io.Envelope +import hep.dataforge.io.IOPlugin +import hep.dataforge.io.Responder +import hep.dataforge.io.type import hep.dataforge.meta.get import hep.dataforge.meta.int -import hep.dataforge.meta.node -import hep.dataforge.meta.string class RemoteFunctionServer( override val context: Context, @@ -14,39 +15,38 @@ class RemoteFunctionServer( ) : ContextAware, Responder { private val plugin by lazy { - context.plugins.load(FunctionsPlugin) + context.plugins.load(IOPlugin) } + override suspend fun respond(request: Envelope): Envelope { require(request.type == RemoteFunctionClient.REQUEST_TYPE) { "Unexpected message type: ${request.type}" } - val functionName = request.meta[RemoteFunctionClient.FUNCTION_NAME_KEY].string ?: "" - @Suppress("UNCHECKED_CAST") val spec = request.meta[RemoteFunctionClient.FUNCTION_SPEC_KEY].node?.let { - plugin.resolve(it) as FunctionSpec - } ?: error("Function specification not found") + val inputFormat = plugin.getInputFormat(request.meta) + val outputFormat = plugin.getOutputFormat(request.meta) - val size = request - .meta[RemoteFunctionClient.SIZE_KEY].int ?: 1 + val size = request.meta[RemoteFunctionClient.SIZE_KEY].int ?: 1 val input = request.data?.read { - spec.inputFormat.run { + inputFormat.run { List(size) { readThis() } } } ?: error("Input is empty") - val output = functionServer.callMany(functionName, spec, input) + val output = functionServer.callMany( + request.meta, + input + ) return Envelope.build { - type = RemoteFunctionClient.RESPONSE_TYPE meta { - RemoteFunctionClient.FUNCTION_NAME_KEY to functionName - RemoteFunctionClient.FUNCTION_SPEC_KEY to spec.toMeta() - RemoteFunctionClient.SIZE_KEY to output.size + meta(request.meta) } + type = RemoteFunctionClient.RESPONSE_TYPE data { - spec.outputFormat.run { + outputFormat.run { output.forEach { writeThis(it) } diff --git a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileBinary.kt b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileBinary.kt index 038281d4..b69969ed 100644 --- a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileBinary.kt +++ b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileBinary.kt @@ -6,6 +6,7 @@ import java.nio.channels.FileChannel import java.nio.file.Files import java.nio.file.Path import java.nio.file.StandardOpenOption +import kotlin.math.min @ExperimentalUnsignedTypes class FileBinary(val path: Path, private val offset: UInt = 0u, size: ULong? = null) : RandomAccessBinary { @@ -14,7 +15,8 @@ class FileBinary(val path: Path, private val offset: UInt = 0u, size: ULong? = n override fun read(from: UInt, size: UInt, block: Input.() -> R): R { FileChannel.open(path, StandardOpenOption.READ).use { - val buffer = it.map(FileChannel.MapMode.READ_ONLY, (from + offset).toLong(), size.toLong()) + val theSize: UInt = min(size, Files.size(path).toUInt() - offset) + val buffer = it.map(FileChannel.MapMode.READ_ONLY, (from + offset).toLong(), theSize.toLong()) return ByteReadPacket(buffer).block() } } diff --git a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/ioFormatsJVM.kt b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/ioFormatsJVM.kt new file mode 100644 index 00000000..8ba97fc3 --- /dev/null +++ b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/ioFormatsJVM.kt @@ -0,0 +1,34 @@ +package hep.dataforge.io + +import hep.dataforge.io.functions.FunctionServer +import hep.dataforge.io.functions.FunctionServer.Companion.FUNCTION_NAME_KEY +import hep.dataforge.io.functions.FunctionServer.Companion.INPUT_FORMAT_KEY +import hep.dataforge.io.functions.FunctionServer.Companion.OUTPUT_FORMAT_KEY +import hep.dataforge.meta.Meta +import hep.dataforge.meta.buildMeta +import hep.dataforge.names.Name +import kotlin.reflect.KClass +import kotlin.reflect.full.isSuperclassOf + +inline fun IOPlugin.resolveIOFormat(): IOFormat? { + return ioFormats.values.find { it.type.isSuperclassOf(T::class) } as IOFormat? +} + +fun IOPlugin.resolveIOFormatName(type: KClass<*>): Name { + return ioFormats.entries.find { it.value.type.isSuperclassOf(type) }?.key + ?: error("Can't resolve IOFormat for type $type") +} + +inline fun IOPlugin.generateFunctionMeta(functionName: String): Meta = buildMeta { + FUNCTION_NAME_KEY to functionName + INPUT_FORMAT_KEY to resolveIOFormatName(T::class) + OUTPUT_FORMAT_KEY to resolveIOFormatName(R::class) +} + +inline fun FunctionServer.function( + functionName: String +): (suspend (T) -> R) { + val plugin = context.plugins.get() ?: error("IO plugin not loaded") + val meta = plugin.generateFunctionMeta(functionName) + return function(meta) +} \ No newline at end of file diff --git a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/tcp/EnvelopeClient.kt b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/tcp/EnvelopeClient.kt new file mode 100644 index 00000000..61b792fd --- /dev/null +++ b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/tcp/EnvelopeClient.kt @@ -0,0 +1,61 @@ +package hep.dataforge.io.tcp + +import hep.dataforge.context.Context +import hep.dataforge.context.ContextAware +import hep.dataforge.io.* +import kotlinx.coroutines.asCoroutineDispatcher +import kotlinx.coroutines.withContext +import kotlinx.coroutines.withTimeout +import kotlinx.io.streams.asInput +import kotlinx.io.streams.asOutput +import java.net.Socket +import java.util.concurrent.Executors +import kotlin.time.Duration +import kotlin.time.ExperimentalTime +import kotlin.time.seconds + +@ExperimentalTime +class EnvelopeClient( + override val context: Context, + val host: String, + val port: Int, + val timeout: Duration = 2.seconds, + val format: EnvelopeFormat = TaggedEnvelopeFormat +) : Responder, ContextAware { + + private val dispatcher = Executors.newSingleThreadExecutor().asCoroutineDispatcher() + + private var socket: Socket? = null + + private fun getSocket(): Socket { + val socket = socket ?: Socket(host, port).also { this.socket = it } + return if (socket.isConnected) { + socket + } else { + Socket(host, port).also { this.socket = it } + } + } + + suspend fun close() { + respond( + Envelope.build { + type = EnvelopeServer.SHUTDOWN_ENVELOPE_TYPE + } + ) + } + + override suspend fun respond(request: Envelope): Envelope = withContext(dispatcher) { + withTimeout(timeout.toLongMilliseconds()) { + val socket = getSocket() + val input = socket.getInputStream().asInput() + val output = socket.getOutputStream().asOutput() + format.run { + output.writeThis(request) + logger.debug { "Sent request with type ${request.type} to ${socket.remoteSocketAddress}" } + val res = input.readThis() + logger.debug { "Received response with type ${res.type} from ${socket.remoteSocketAddress}" } + return@withTimeout res + } + } + } +} \ No newline at end of file diff --git a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/tcp/EnvelopeServer.kt b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/tcp/EnvelopeServer.kt new file mode 100644 index 00000000..9d6c5826 --- /dev/null +++ b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/tcp/EnvelopeServer.kt @@ -0,0 +1,70 @@ +package hep.dataforge.io.tcp + +import hep.dataforge.context.Context +import hep.dataforge.context.ContextAware +import hep.dataforge.io.EnvelopeFormat +import hep.dataforge.io.Responder +import hep.dataforge.io.TaggedEnvelopeFormat +import hep.dataforge.io.type +import kotlinx.coroutines.* +import kotlinx.io.streams.asInput +import kotlinx.io.streams.asOutput +import java.net.ServerSocket +import java.net.Socket + +class EnvelopeServer( + override val context: Context, + val port: Int, + val responder: Responder, + val scope: CoroutineScope, + val format: EnvelopeFormat = TaggedEnvelopeFormat +) : ContextAware { + + private var job: Job? = null + + fun start() { + if (job == null) { + logger.info { "Starting envelope server on port $port" } + val job = scope.launch(Dispatchers.IO) { + val serverSocket = ServerSocket(port) + //TODO add handshake and format negotiation + while (!serverSocket.isClosed) { + val socket = serverSocket.accept() + logger.info { "Accepted connection from ${socket.remoteSocketAddress}" } + readSocket(socket) + } + } + } + } + + fun stop() { + logger.info { "Stopping envelope server on port $port" } + job?.cancel() + job = null + } + + private fun CoroutineScope.readSocket(socket: Socket) { + val input = socket.getInputStream().asInput() + val output = socket.getOutputStream().asOutput() + format.run { + launch { + while (isActive && socket.isConnected) { + val request = input.readThis() + logger.debug { "Accepted request with type ${request.type} from ${socket.remoteSocketAddress}" } + if (request.type == SHUTDOWN_ENVELOPE_TYPE) { + //Echo shutdown command + logger.info { "Accepted graceful shutdown signal from ${socket.inetAddress}" } + socket.close() + cancel("Graceful connection shutdown requested by client") + } + val response = responder.respond(request) + output.writeThis(response) + } + } + } + } + + companion object { + const val SHUTDOWN_ENVELOPE_TYPE = "@shutdown" + } +} \ No newline at end of file diff --git a/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/tcp/EnvelopeServerTest.kt b/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/tcp/EnvelopeServerTest.kt new file mode 100644 index 00000000..a2642564 --- /dev/null +++ b/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/tcp/EnvelopeServerTest.kt @@ -0,0 +1,58 @@ +package hep.dataforge.io.tcp + +import hep.dataforge.context.Global +import hep.dataforge.io.Envelope +import hep.dataforge.io.Responder +import kotlinx.coroutines.GlobalScope +import org.junit.AfterClass +import org.junit.BeforeClass +import kotlin.time.ExperimentalTime + +object EchoResponder : Responder { + override suspend fun respond(request: Envelope): Envelope = request +} + +@ExperimentalTime +class EnvelopeServerTest { + companion object { + @JvmStatic + val echoEnvelopeServer = EnvelopeServer(Global, 7778, EchoResponder, GlobalScope) + + @BeforeClass + @JvmStatic + fun start() { + echoEnvelopeServer.start() + } + + @AfterClass + @JvmStatic + fun close() { + echoEnvelopeServer.stop() + } + } + + +// @Test +// fun doEchoTest() { +// val client = EnvelopeClient(Global, host = "localhost", port = 7778) +// val request = Envelope.build { +// type = "test.echo" +// meta { +// "test.value" to 22 +// } +// data { +// writeDouble(22.7) +// } +// } +// val response = runBlocking { +// client.respond(request) +// } +// +// assertEquals(request.meta, response.meta) +// assertEquals(request.data, response.data) +// +// runBlocking { +// client.close() +// } +// } +} \ No newline at end of file diff --git a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/GenericTask.kt b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/GenericTask.kt index 278098d1..a46862a0 100644 --- a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/GenericTask.kt +++ b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/GenericTask.kt @@ -1,17 +1,18 @@ package hep.dataforge.workspace -import hep.dataforge.data.* +import hep.dataforge.data.DataNode import hep.dataforge.descriptors.NodeDescriptor import hep.dataforge.meta.Meta import hep.dataforge.meta.get import hep.dataforge.meta.node +import hep.dataforge.names.Name import kotlin.reflect.KClass //data class TaskEnv(val workspace: Workspace, val model: TaskModel) class GenericTask( - override val name: String, + override val name: Name, override val type: KClass, override val descriptor: NodeDescriptor, private val modelTransform: TaskModelBuilder.(Meta) -> Unit, diff --git a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/SimpleWorkspace.kt b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/SimpleWorkspace.kt index abdb70e3..cb621963 100644 --- a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/SimpleWorkspace.kt +++ b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/SimpleWorkspace.kt @@ -3,10 +3,10 @@ package hep.dataforge.workspace import hep.dataforge.context.Context import hep.dataforge.context.Global import hep.dataforge.context.content +import hep.dataforge.context.toMap import hep.dataforge.data.DataNode import hep.dataforge.meta.Meta import hep.dataforge.names.Name -import hep.dataforge.names.toName /** @@ -20,7 +20,7 @@ class SimpleWorkspace( ) : Workspace { override val tasks: Map> by lazy { - context.content>(Task.TYPE) + tasks.associateBy { it.name.toName() } + context.content>(Task.TYPE) + tasks.toMap() } companion object { diff --git a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskModel.kt b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskModel.kt index 07fb6638..cda164c7 100644 --- a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskModel.kt +++ b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskModel.kt @@ -23,7 +23,7 @@ import hep.dataforge.workspace.TaskModel.Companion.MODEL_TARGET_KEY * @param dependencies a list of direct dependencies for this task */ data class TaskModel( - val name: String, + val name: Name, val meta: Meta, val dependencies: Collection ) : MetaRepr { @@ -66,7 +66,7 @@ annotation class TaskBuildScope * A builder for [TaskModel] */ @TaskBuildScope -class TaskModelBuilder(val name: String, meta: Meta = EmptyMeta) { +class TaskModelBuilder(val name: Name, meta: Meta = EmptyMeta) { /** * Meta for current task. By default uses the whole input meta */ @@ -78,10 +78,13 @@ class TaskModelBuilder(val name: String, meta: Meta = EmptyMeta) { /** * Add dependency for a task defined in a workspace and resolved by */ - fun dependsOn(name: String, meta: Meta = this.meta, placement: Name = EmptyName) { - dependencies.add(WorkspaceTaskDependency(name.toName(), meta, placement)) + fun dependsOn(name: Name, meta: Meta = this.meta, placement: Name = EmptyName) { + dependencies.add(WorkspaceTaskDependency(name, meta, placement)) } + fun dependsOn(name: String, meta: Meta = this.meta, placement: Name = EmptyName) = + dependsOn(name.toName(),meta,placement) + fun dependsOn(task: Task<*>, meta: Meta = this.meta, placement: Name = EmptyName) { dependencies.add(DirectTaskDependency(task, meta, placement)) } diff --git a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/WorkspacePlugin.kt b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/WorkspacePlugin.kt index 2fefd4f8..1ddd291b 100644 --- a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/WorkspacePlugin.kt +++ b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/WorkspacePlugin.kt @@ -1,8 +1,8 @@ package hep.dataforge.workspace import hep.dataforge.context.AbstractPlugin +import hep.dataforge.context.toMap import hep.dataforge.names.Name -import hep.dataforge.names.toName /** * An abstract plugin with some additional boilerplate to effectively work with workspace context @@ -12,7 +12,7 @@ abstract class WorkspacePlugin : AbstractPlugin() { override fun provideTop(target: String): Map { return when(target){ - Task.TYPE -> tasks.associateBy { it.name.toName() } + Task.TYPE -> tasks.toMap() else -> emptyMap() } } diff --git a/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/TaskBuilder.kt b/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/TaskBuilder.kt index c35003d3..afe9ddd1 100644 --- a/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/TaskBuilder.kt +++ b/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/TaskBuilder.kt @@ -8,6 +8,7 @@ import hep.dataforge.meta.get import hep.dataforge.meta.string import hep.dataforge.names.EmptyName import hep.dataforge.names.Name +import hep.dataforge.names.asName import hep.dataforge.names.toName import kotlin.reflect.KClass @@ -176,7 +177,7 @@ class TaskBuilder(val name: String) { internal fun build(): GenericTask = GenericTask( - name, + name.asName(), Any::class, descriptor ?: NodeDescriptor.empty(), modelTransform diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index 87b738cbd051603d91cc39de6cb000dd98fe6b02..5c2d1cf016b3885f6930543d57b744ea8c220a1a 100644 GIT binary patch delta 3320 zcmai0c|2768`iN!wwN(!Oxeo5?`tVU3{m#%jC~noTx!q_nHtNnR`zAgWC@krB#b55 znJk4YA);()+(!K-w|npJuix)IpYu7-^SqzuJ>T~|?;j_-ma(;-@!<_I_B>B@4FVej z11CRtM@$8afpkN^v*te{ycR9yTldxXJbmio?@}x{9}zaw&=aQt(a^ZXN9S3i8a+Z% zGc@&(5}jplZjJKk2wNlTp(mbeKL5J9Gjo==yT{-eVKj?*rT1%bQ@%#Xce~~1f{19^ zoD75QEoSzDVh@!9qG4yl`;9=Ysp?rRX=(8$VDRz=R+oA3>jLxjW-H!-2biNSYuy)U z7-B-qC5l;>qjMTg!DbWPY}h7qxi6xp)_T)_O2+*&NDg?v;RyY@5XtWHx%(ImQ_3E% zA%$s3xrxE0Fk>DhG!pG)4}I!pWJl~QtV_3Jl2W4PuWWssMq^UpGatK+4CING9pB#5 z_NDc)aonVrZuXsr5!RcE#?aXFZQjt2VMd)-p00K$EheT?H!m_D2Mdqq;0moaO=C&y zgJnvzgUn!wkx^{r049pU#gsIMhl`%{MDNl;}JRbneC zSTB=5f;o9=2Rt24_lt&%%f~m{Ts)zu8H9j`INrgMp>l-|k%Kj%U`OXL1J2e+CJHJxreHLD_#o*ZeuXE4uGDQAJS_PpEGt7hmd7psmLEBL^h zD#JbHiklZEXkk9(6uF$ErsUu^jg7c~1oRS&CuTq*Xg_cOvGw~FZ&1#p(6|jz9lJnP zSIJ)sX_W2$PSksX&}*_ejz+t*X)xK|JcakaMRGd%c*R)cQcT|?sM^#{fdjh5_I$iK zBX_d;wz+cf>b}r!i3yo6eaua)d`|Mi_|Q3mAz5Qn?#~xgE9In<;TwYN^~mtaYy#WU z*ffWtxwlk&!e@UfqQ$bn23RDFV3o-H_WM}44yQpYw;JuRf$at#XX-qmuVnKqg-Bo# zJjZE39)!{i$qJh?oJzVzWFDlSW;{Wf`Z)33Y$Fh^+qasrsEJsfy9yhyTFe?Lej&3n zEAS(D8WCt(ew(SGD z-J#7@l?KI*ZbS)AVQ23qV&{c=$@zUp0@6=kZp+5by+gnAWdB||7e=!yJ|WTpG0OC7 zKlKWFv6#(>nrEq@d1i-#L9SVxTDNb1DaY%2$=@)`k&3s8wz$M*;THa&!2Isj%6CQS zY>A4HtmWY3@9e@F)mCHJQzBz~Lt(wcJE{!CAr=wxn4|5n(jslTy)~IF?tNK zD^2#hTM0d6MDg>`9;s5*(4W1V8y}F8OT6Xap{`=h1XVKO3zrBh=;JnIs*RB>@7t5T zwV=G^T)L=(9P7tS={6`tEBBBm^u~_!-#m75G*h}y_Jj7|STtiY_LDR5UUHI@awWmB zDn6q9{2M-EHaTm53ln%ENJ$HpLwRcL>7^hUrM=}&`qmWTgtr{Ul*Lqcd_9S0xZ1s>F2dVd(s)3&$`gxFAu6jXYIS ze#M~w@=X@lm)sFI4EEiqKh7JxN=_?+}D=iHCc&S2<^VPZ6 zYKXZgvi(Yne9}k6o=ezgquABVB77}x$nKXh`@LjH&lQPqm_;MTL>4RGO|E#_7AS4@43rz=ij?gcMZalnd-JK4ILhL)Ee(3G zN}g99HmhxoBjHR~y@b>-7{f+`p zIZ<^8%d;wCA#xfwSc6$DNVPjAX6FCkb|MQ|6hFyz9UhoLF0^xUd#*^2Ofn zOJgmwDyb1=Z8T)ArRy|VQOM+BrhZ>W_ELJ6u(d^JTu|j%*6g8JKZ-ewoj)sXJCdS= zHOo?HscL;Z`H18}%WnE1&o42KZ+=fg(*VN>t>kRkcd{mP9NF6;MnzH&m2WsD)sX~h zbhv|Ux$w2avQwoI`IKiGMLrL;Z>R}Y_0K*L=63V z)ut+5tM74Glzb?92kbu5@3M#1Hi7K3$c)?TL$}`aKf0hC3`r!>Xy3!f{ z`}Y#@$`|mG1JlKzVE!vD04aX}x#hV*+AC>bQ|%XJ1<&;=0?uX!RM?CIB=+!tgkB-w zu*HF--^U4#nG1mXz0v^0@|UCs1lt}!1zTaTwoe+k?sPym`pyB-F25ivXx)#1|1%|e zJ7Vpujkk#Lu%U{v6xiQ5LW2`~QXrR`ja@*L=b0ejT977v%C)0WAik0gV7U z6a-7##p#p>>>3a{^Z}e3Z~?A|foBFU12bqaEE*0vqdCCVLFq%{;F%$Dkb6i8;Qo!C z&;zkU(!i5zbSMd)zQzg8(kU^HPQ^flVIzR)<^jwbwget09YD?zV*rx+mx@0IN{#S< zsB|8Ve>>sJI7sHE!@=(((ttqL0ks%C4M^r5!0H?rJ;MV|jtT)1cMl{|9xo_Okp@Ka ze^CzbCPf?IDFWLlE`V1FDDpZ0C@7~VMZt%!6%SFtxz{!Tb1UfBDEg~49x!4|2#_L! zX=6UXeh28_?VY*suC^Sy!?XXp?9-G{ zEbF`ELqycMcTK-$-pw|Jox9S^<_NX$7{PI7aX1p5N>aOyj&D01H#;3?=q^!=_mq@k zUHheWO_|CDYA~8r<-%q8&Gm$uPSx4S`reKPnv?Nif4kS)^smTg&m@kLYT87txGxGxw+Qc zTAi=`vzavOlyLrgf2A~;1~Gx$jcb|fkhfctRt6CjRooL|#wr)(*8D4n;2cBe>p9_T zCeJf!IgCH0h1m)UPLk3hZz120oe5YH$oXjSMHcPv@#wX;OP5bBSJMavm2}5Q8(V&# zXGA!+dAwOiXuQ)|+XwF2HW1@_MPm3*v{M86V_~+xk1K7cI7mxBKU5#bofCjZqqjs$ z(sipv#Ul%KJ)h?ua}a3Dg(6yaxeJ(HD-&`AT9kZJVLJTz?WIfgao$bYwEhXh+&GA= zkpI03HVxtWc*H!~z~9%DC;;Qej=WppOD!i1$MO1`&8LW%IWd2sbnS7j+<0b`v1%qx!owUU+ZIHJFp1yH9BFvUYI^up=ZYX$K_YM|Bn2fCG3sq#(EpRB$|A9~9*^M%Sq)EAjr0&W`hHyz96Z9h*odHK|Ju$JQ0c zO9oayZQv;2b{pLJo`T)C%yS@sAKO*WC%22XDmrdRTd;uFr*sb_{GDl=*Y`l*;>lNWh=XCbn#V}C&jmw3>t zNH(fnG%j@AI$TSggf(e3DxrpHjnpeKExsb|hC`kxjD4HUSmu)&aJNt&DtCWh#51*} zS!qfplP(f0`hJ)VHrXFD_uB7ia4#%U)3S8lGY9^(T1)M8xQxP*3w4&QJr~O`$A&N5 z_taom$34zt+reJDV?oZ*qr5ERUH7#~xm7)D(u#q#m`~~-F+TZ6Q*L)s_#T3GZUuZM zhCH9!{qXnD)9jln$|GDeDPqo=+D6#vQkAjdHtT>{VxU#AQJW-je=UWN5*R>v5vWF6 zK_6z?#thq>&%@fu5epvO$rfx`v9GojdOLGFaQ2V8?Ri z(?L2JBK(;G)bIF7r5T6Ahzst5k4j#hvhl3a`@Ksfyj3^Cx}zGE)vm$ecB$?~2`S&e zE)Nx6TiDO*JO6UmWWc+zLDmnII+)ROEvW3_{*%Fjs8Q^k4+Z&cJ0lp=@p*N!fw0>L zPSWrxar=HPDCwZnmN%orA-K2142{bJ0el>N{KM(xoHJu_HWSQihq^y%SEmj>CsBjl zj6)jxqm7NwiVHh-xQ`ex^02-y_ZO`A`P(1UwLK5G_T8=uI8@e%Kh31Xay z>H$7OG8cQ%>c_RjXhRA|Yh=93MnM)V0JlD#yP-1YNx}5`sg}-vE%slfve&}e$*L>+ zSAq_CMc5SYx6N)5h%-)?JOAhiVM5`TWT7?<9 zKKxMMb9GXHpQ1ajAr?!hxcauobJLf{IpvJ=9ny}FwdGCYmwgj?0qhIG{5zbTTVc2b zo+3h|{F_Yg96k{?rVn`m`%d??#avI-eh^XnTH2r*o>5n>`UuIsuCIeN5Br62W!Yy#8)0uWcVG%-QnMHczpWoe zftoSf-WJq~x8`|ws<-9{Va9@s#SoH3uw`>4!~uyB-(lV)SD9f(TPNa!o7JLL%!a)@gUmedno%~}$ z#zZLYah$5mf@Z2}a(oDDM^$qq>*nb;?aVn?D`($Om=?j+T%S?eSgR1t=zzwGw|kvM zt~WiOO&UVW=7N=8ERxM<4?Wbj4bPIP4z3=hjp(uuT}ne*E9ct0)Lsk?bG=1nNo=oB z0JEoKzAw45q-lB!IbJKsY=Lpru48qY6ql!Z#J13ywC&7??l&AtxiowZ|Cg(k*UE#@ zrJm|m^EV_6jz}f($PrOb`S;imdEwtu`#cCu3aMXBgUUH4t2j_qu=KmOO645(v(_DL z^G5PF%RR0@X5D{(V%x5L{xD1Sa>^wR+$0j(DeVfwk;tp3<@i$~qOsvx^uUy!zV8G0~0`$f?VV=?vm zOwYnZB>UV_b#sh6ibtN`5I+l%mTE9T%*J!xaz}cWisUNLg@>nEiKv4hgmv`5C)GIDbBOgq{?5K-!=>z{CLJ$wIBkL-~yV{}~e*^#eZ1f%)RR;DgcM zfOqnA#42!t$D;@!QT3n50ve1d0$Zl^m}ABc){bz2HDhq#o&{ZLlQ=*lO9Alv7y_uW z`bTL2KkVsP<{%6$`1yeL}DmCZuxPZRJp*( z*Kk1M23@g@UjhQ6PEZ{58CL@Aqv>cB0|#ltT;SR`95{}ptMe0@zz&v<>j{GNDt-bE zn5EFw?u0e)Ee+J0^aq@C>E_j>A%MyU^@?Rcohe{^TCd{d<=ub5$bWAh Date: Mon, 9 Sep 2019 16:15:35 +0300 Subject: [PATCH 06/48] Envelope client-sever EAP --- dataforge-io/build.gradle.kts | 9 +- .../kotlin/hep/dataforge/io/Binary.kt | 27 ++---- .../kotlin/hep/dataforge/io/Envelope.kt | 30 +++++-- .../kotlin/hep/dataforge/io/IOFormat.kt | 1 + .../hep/dataforge/io/TaggedEnvelopeFormat.kt | 7 +- .../hep/dataforge/io/EnvelopeFormatTest.kt | 32 +++++++ .../hep/dataforge/io/tcp/EnvelopeClient.kt | 61 ++++++------- .../hep/dataforge/io/tcp/EnvelopeServer.kt | 51 ++++++++--- .../dataforge/io/tcp/InputStreamAsInput.kt | 34 +++++++ .../dataforge/io/tcp/EnvelopeServerTest.kt | 58 +++++++----- .../kotlin/hep/dataforge/meta/Meta.kt | 5 -- .../kotlin/hep/dataforge/meta/Specific.kt | 4 +- .../hep/dataforge/meta/configDelegates.kt | 59 ++++++------ .../hep/dataforge/meta/metaDelegates.kt | 89 ++++++++++--------- 14 files changed, 291 insertions(+), 176 deletions(-) create mode 100644 dataforge-io/src/commonTest/kotlin/hep/dataforge/io/EnvelopeFormatTest.kt create mode 100644 dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/tcp/InputStreamAsInput.kt diff --git a/dataforge-io/build.gradle.kts b/dataforge-io/build.gradle.kts index b4306efc..ce3a8be8 100644 --- a/dataforge-io/build.gradle.kts +++ b/dataforge-io/build.gradle.kts @@ -15,11 +15,18 @@ kotlin { commonMain{ dependencies { api(project(":dataforge-context")) + api("org.jetbrains.kotlinx:kotlinx-io:0.1.14") + } + } + jvmMain{ + dependencies { + api("org.jetbrains.kotlinx:kotlinx-io-jvm:0.1.14") } } jsMain{ dependencies{ - api(npm("text-encoding")) + //api(npm("text-encoding")) + api("org.jetbrains.kotlinx:kotlinx-io-js:0.1.14") } } } diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Binary.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Binary.kt index da62113b..429ab185 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Binary.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Binary.kt @@ -39,8 +39,8 @@ interface RandomAccessBinary : Binary { override fun read(block: Input.() -> R): R = read(0.toUInt(), UInt.MAX_VALUE, block) } -fun Binary.readAll(): ByteReadPacket = read { - ByteReadPacket(this.readBytes()) +fun Binary.toBytes(): ByteArray = read { + this.readBytes() } @ExperimentalUnsignedTypes @@ -56,10 +56,11 @@ object EmptyBinary : RandomAccessBinary { override fun read(from: UInt, size: UInt, block: Input.() -> R): R { error("The binary is empty") } + } @ExperimentalUnsignedTypes -class ArrayBinary(val array: ByteArray) : RandomAccessBinary { +inline class ArrayBinary(val array: ByteArray) : RandomAccessBinary { override val size: ULong get() = array.size.toULong() override fun read(from: UInt, size: UInt, block: Input.() -> R): R { @@ -68,16 +69,6 @@ class ArrayBinary(val array: ByteArray) : RandomAccessBinary { } } -class PacketBinary(val packet: ByteReadPacket) : Binary { - override val size: ULong - get() = TODO("not implemented") //To change initializer of created properties use File | Settings | File Templates. - - override fun read(block: Input.() -> R): R { - TODO("not implemented") //To change body of created functions use File | Settings | File Templates. - } - -} - /** * Read given binary as object using given format */ @@ -87,13 +78,9 @@ fun Binary.readWith(format: IOFormat): T = format.run { } } -/** - * Write this object to a binary - * TODO make a lazy binary that does not use intermediate array - */ -fun T.writeWith(format: IOFormat): Binary = format.run { +fun IOFormat.writeBinary(obj: T): Binary { val packet = buildPacket { - writeThis(this@writeWith) + writeThis(obj) } - return@run ArrayBinary(packet.readBytes()) + return ArrayBinary(packet.readBytes()) } \ No newline at end of file diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Envelope.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Envelope.kt index 36178d98..6c3b45b9 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Envelope.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Envelope.kt @@ -1,6 +1,8 @@ package hep.dataforge.io import hep.dataforge.meta.* +import hep.dataforge.names.asName +import hep.dataforge.names.plus import kotlinx.io.core.Output import kotlinx.io.core.buildPacket import kotlinx.io.core.readBytes @@ -14,10 +16,11 @@ interface Envelope { /** * meta keys */ - const val ENVELOPE_NODE = "@envelope" - const val ENVELOPE_TYPE_KEY = "$ENVELOPE_NODE.type" - const val ENVELOPE_DATA_TYPE_KEY = "$ENVELOPE_NODE.dataType" - const val ENVELOPE_DESCRIPTION_KEY = "$ENVELOPE_NODE.description" + val ENVELOPE_NODE_KEY = "@envelope".asName() + val ENVELOPE_TYPE_KEY = ENVELOPE_NODE_KEY + "type" + val ENVELOPE_DATA_TYPE_KEY = ENVELOPE_NODE_KEY + "dataType" + val ENVELOPE_DATA_ID_KEY = ENVELOPE_NODE_KEY + "dataID" + val ENVELOPE_DESCRIPTION_KEY = ENVELOPE_NODE_KEY + "description" //const val ENVELOPE_TIME_KEY = "@envelope.time" /** @@ -32,24 +35,35 @@ class SimpleEnvelope(override val meta: Meta, override val data: Binary?) : Enve /** * The purpose of the envelope * - * @return */ val Envelope.type: String? get() = meta[Envelope.ENVELOPE_TYPE_KEY].string /** * The type of data encoding * - * @return */ val Envelope.dataType: String? get() = meta[Envelope.ENVELOPE_DATA_TYPE_KEY].string /** * Textual user friendly description * - * @return */ val Envelope.description: String? get() = meta[Envelope.ENVELOPE_DESCRIPTION_KEY].string +/** + * An optional unique identifier that is used for data comparison. Data without identifier could not be compared to another data. + */ +val Envelope.dataID: String? get() = meta[Envelope.ENVELOPE_DATA_ID_KEY].string + +fun Envelope.metaEquals(other: Envelope): Boolean = this.meta == other.meta + +fun Envelope.dataEquals(other: Envelope): Boolean = this.dataID != null && this.dataID == other.dataID + +fun Envelope.contentEquals(other: Envelope): Boolean { + return (this === other || (metaEquals(other) && dataEquals(other))) +} + + /** * An envelope, which wraps existing envelope and adds one or several additional layers of meta */ @@ -59,7 +73,7 @@ class ProxyEnvelope(val source: Envelope, vararg meta: Meta) : Envelope { } /** - * Add few meta layers to existing envelope + * Add few meta layers to existing envelope (on top of existing meta) */ fun Envelope.withMetaLayers(vararg layers: Meta): Envelope { return when { diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOFormat.kt index 0ebf9485..e2a1f16e 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOFormat.kt @@ -36,6 +36,7 @@ interface IOFormat : Named { fun IOFormat.writePacket(obj: T): ByteReadPacket = buildPacket { writeThis(obj) } fun IOFormat.writeBytes(obj: T): ByteArray = buildPacket { writeThis(obj) }.readBytes() +fun IOFormat.readBytes(array: ByteArray): T = ByteReadPacket(array).readThis() object DoubleIOFormat : IOFormat { diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt index 47e8ab5d..7bb04ec4 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt @@ -10,7 +10,7 @@ object TaggedEnvelopeFormat : EnvelopeFormat { const val VERSION = "DF03" private const val START_SEQUENCE = "#~" private const val END_SEQUENCE = "~#\r\n" - private const val TAG_SIZE = 26u + private const val TAG_SIZE = 24u override val name: Name = super.name + VERSION @@ -31,6 +31,8 @@ object TaggedEnvelopeFormat : EnvelopeFormat { val metaFormatKey = readShort() val metaLength = readUInt() val dataLength = readULong() + val end = readTextExactBytes(4) + if (end != END_SEQUENCE) error("The input is not an envelope") return Tag(metaFormatKey, metaLength, dataLength) } @@ -55,10 +57,9 @@ object TaggedEnvelopeFormat : EnvelopeFormat { ?: error("Meta format with key ${tag.metaFormatKey} not found") val metaPacket = ByteReadPacket(readBytes(tag.metaSize.toInt())) - val meta = metaFormat.run { metaPacket.readThis() } - val dataBytes = readBytes(tag.dataSize.toInt()) + val meta = metaFormat.run { metaPacket.readThis() } return SimpleEnvelope(meta, ArrayBinary(dataBytes)) } diff --git a/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/EnvelopeFormatTest.kt b/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/EnvelopeFormatTest.kt new file mode 100644 index 00000000..059c0f19 --- /dev/null +++ b/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/EnvelopeFormatTest.kt @@ -0,0 +1,32 @@ +package hep.dataforge.io + +import kotlin.test.Test +import kotlin.test.assertEquals + + +class EnvelopeFormatTest { + val envelope = Envelope.build { + type = "test.format" + meta{ + "d" to 22.2 + } + data{ + writeDouble(22.2) + } + } + + @ExperimentalStdlibApi + @Test + fun testTaggedFormat(){ + TaggedEnvelopeFormat.run { + val bytes = writeBytes(envelope) + println(bytes.decodeToString()) + val res = readBytes(bytes) + assertEquals(envelope.meta,res.meta) + val double = res.data?.read { + readDouble() + } + assertEquals(22.2, double) + } + } +} \ No newline at end of file diff --git a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/tcp/EnvelopeClient.kt b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/tcp/EnvelopeClient.kt index 61b792fd..17b0429d 100644 --- a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/tcp/EnvelopeClient.kt +++ b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/tcp/EnvelopeClient.kt @@ -5,57 +5,58 @@ import hep.dataforge.context.ContextAware import hep.dataforge.io.* import kotlinx.coroutines.asCoroutineDispatcher import kotlinx.coroutines.withContext -import kotlinx.coroutines.withTimeout -import kotlinx.io.streams.asInput -import kotlinx.io.streams.asOutput +import kotlinx.io.streams.writePacket import java.net.Socket import java.util.concurrent.Executors -import kotlin.time.Duration import kotlin.time.ExperimentalTime -import kotlin.time.seconds @ExperimentalTime class EnvelopeClient( override val context: Context, val host: String, val port: Int, - val timeout: Duration = 2.seconds, val format: EnvelopeFormat = TaggedEnvelopeFormat ) : Responder, ContextAware { private val dispatcher = Executors.newSingleThreadExecutor().asCoroutineDispatcher() - private var socket: Socket? = null +// private var socket: SocketChannel? = null +// +// private fun getSocket(): Socket { +// val socket = socket ?: Socket(host, port).also { this.socket = it } +// return if (socket.isConnected) { +// socket +// } else { +// Socket(host, port).also { this.socket = it } +// } +// } - private fun getSocket(): Socket { - val socket = socket ?: Socket(host, port).also { this.socket = it } - return if (socket.isConnected) { - socket - } else { - Socket(host, port).also { this.socket = it } + suspend fun close() { + try { + respond( + Envelope.build { + type = EnvelopeServer.SHUTDOWN_ENVELOPE_TYPE + } + ) + } catch (ex: Exception) { + logger.error { ex } } } - suspend fun close() { - respond( - Envelope.build { - type = EnvelopeServer.SHUTDOWN_ENVELOPE_TYPE - } - ) - } override suspend fun respond(request: Envelope): Envelope = withContext(dispatcher) { - withTimeout(timeout.toLongMilliseconds()) { - val socket = getSocket() - val input = socket.getInputStream().asInput() - val output = socket.getOutputStream().asOutput() - format.run { - output.writeThis(request) - logger.debug { "Sent request with type ${request.type} to ${socket.remoteSocketAddress}" } - val res = input.readThis() - logger.debug { "Received response with type ${res.type} from ${socket.remoteSocketAddress}" } - return@withTimeout res + //val address = InetSocketAddress(host,port) + val socket = Socket(host, port) + val input = socket.getInputStream().asInput() + val output = socket.getOutputStream() + format.run { + output.writePacket { + writeThis(request) } + logger.debug { "Sent request with type ${request.type} to ${socket.remoteSocketAddress}" } + val res = input.readThis() + logger.debug { "Received response with type ${res.type} from ${socket.remoteSocketAddress}" } + return@withContext res } } } \ No newline at end of file diff --git a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/tcp/EnvelopeServer.kt b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/tcp/EnvelopeServer.kt index 9d6c5826..68da8375 100644 --- a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/tcp/EnvelopeServer.kt +++ b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/tcp/EnvelopeServer.kt @@ -7,10 +7,10 @@ import hep.dataforge.io.Responder import hep.dataforge.io.TaggedEnvelopeFormat import hep.dataforge.io.type import kotlinx.coroutines.* -import kotlinx.io.streams.asInput -import kotlinx.io.streams.asOutput +import kotlinx.io.streams.writePacket import java.net.ServerSocket import java.net.Socket +import kotlin.concurrent.thread class EnvelopeServer( override val context: Context, @@ -28,7 +28,7 @@ class EnvelopeServer( val job = scope.launch(Dispatchers.IO) { val serverSocket = ServerSocket(port) //TODO add handshake and format negotiation - while (!serverSocket.isClosed) { + while (isActive && !serverSocket.isClosed) { val socket = serverSocket.accept() logger.info { "Accepted connection from ${socket.remoteSocketAddress}" } readSocket(socket) @@ -43,22 +43,49 @@ class EnvelopeServer( job = null } - private fun CoroutineScope.readSocket(socket: Socket) { - val input = socket.getInputStream().asInput() - val output = socket.getOutputStream().asOutput() - format.run { - launch { - while (isActive && socket.isConnected) { +// private fun CoroutineScope.readSocket(socket: Socket) { +// launch(Dispatchers.IO) { +// val input = socket.getInputStream().asInput() +// val output = socket.getOutputStream().asOutput() +// format.run { +// while (isActive && socket.isConnected) { +// val request = input.readThis() +// logger.debug { "Accepted request with type ${request.type} from ${socket.remoteSocketAddress}" } +// if (request.type == SHUTDOWN_ENVELOPE_TYPE) { +// //Echo shutdown command +// logger.info { "Accepted graceful shutdown signal from ${socket.inetAddress}" } +// socket.close() +// cancel("Graceful connection shutdown requested by client") +// } +// val response = responder.respond(request) +// output.writeThis(response) +// } +// } +// } +// } + + private fun readSocket(socket: Socket) { + thread { + val input = socket.getInputStream().asInput() + val outputStream = socket.getOutputStream() + format.run { + while (socket.isConnected) { val request = input.readThis() logger.debug { "Accepted request with type ${request.type} from ${socket.remoteSocketAddress}" } if (request.type == SHUTDOWN_ENVELOPE_TYPE) { //Echo shutdown command logger.info { "Accepted graceful shutdown signal from ${socket.inetAddress}" } socket.close() - cancel("Graceful connection shutdown requested by client") + return@thread +// cancel("Graceful connection shutdown requested by client") + } + runBlocking { + val response = responder.respond(request) + outputStream.writePacket { + writeThis(response) + } + logger.debug { "Sent response with type ${response.type} to ${socket.remoteSocketAddress}" } } - val response = responder.respond(request) - output.writeThis(response) } } } diff --git a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/tcp/InputStreamAsInput.kt b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/tcp/InputStreamAsInput.kt new file mode 100644 index 00000000..6dab414d --- /dev/null +++ b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/tcp/InputStreamAsInput.kt @@ -0,0 +1,34 @@ +package hep.dataforge.io.tcp + +import kotlinx.io.core.AbstractInput +import kotlinx.io.core.Input +import kotlinx.io.core.IoBuffer +import kotlinx.io.core.IoBuffer.Companion.NoPool +import kotlinx.io.core.writePacket +import kotlinx.io.streams.readPacketAtMost +import java.io.InputStream + +/** + * Modified version of InputStream to Input converter that supports waiting for input + */ +internal class InputStreamAsInput( + private val stream: InputStream +) : AbstractInput(pool = NoPool) { + + + override fun fill(): IoBuffer? { + + val packet = stream.readPacketAtMost(4096) + return pool.borrow().apply { + resetForWrite(4096) + writePacket(packet) + } + } + + override fun closeSource() { + stream.close() + } +} + +fun InputStream.asInput(): Input = + InputStreamAsInput(this) diff --git a/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/tcp/EnvelopeServerTest.kt b/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/tcp/EnvelopeServerTest.kt index a2642564..f00a3754 100644 --- a/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/tcp/EnvelopeServerTest.kt +++ b/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/tcp/EnvelopeServerTest.kt @@ -3,16 +3,28 @@ package hep.dataforge.io.tcp import hep.dataforge.context.Global import hep.dataforge.io.Envelope import hep.dataforge.io.Responder +import hep.dataforge.io.TaggedEnvelopeFormat +import hep.dataforge.io.writeBytes import kotlinx.coroutines.GlobalScope +import kotlinx.coroutines.runBlocking import org.junit.AfterClass import org.junit.BeforeClass +import org.junit.Test +import kotlin.test.assertEquals import kotlin.time.ExperimentalTime +@ExperimentalStdlibApi object EchoResponder : Responder { - override suspend fun respond(request: Envelope): Envelope = request + override suspend fun respond(request: Envelope): Envelope { + val string = TaggedEnvelopeFormat.run { writeBytes(request).decodeToString() } + println("ECHO:") + println(string) + return request + } } @ExperimentalTime +@ExperimentalStdlibApi class EnvelopeServerTest { companion object { @JvmStatic @@ -32,27 +44,25 @@ class EnvelopeServerTest { } -// @Test -// fun doEchoTest() { -// val client = EnvelopeClient(Global, host = "localhost", port = 7778) -// val request = Envelope.build { -// type = "test.echo" -// meta { -// "test.value" to 22 -// } -// data { -// writeDouble(22.7) -// } -// } -// val response = runBlocking { -// client.respond(request) -// } -// -// assertEquals(request.meta, response.meta) -// assertEquals(request.data, response.data) -// -// runBlocking { -// client.close() -// } -// } + @Test + fun doEchoTest() { + val client = EnvelopeClient(Global, host = "localhost", port = 7778) + val request = Envelope.build { + type = "test.echo" + meta { + "test.value" to 22 + } + data { + writeDouble(22.7) + } + } + runBlocking { + val response = client.respond(request) + + + assertEquals(request.meta, response.meta) +// assertEquals(request.data?.toBytes()?.decodeToString(), response.data?.toBytes()?.decodeToString()) + client.close() + } + } } \ No newline at end of file diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Meta.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Meta.kt index 9e0dedf9..45a27250 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Meta.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Meta.kt @@ -187,11 +187,6 @@ abstract class MetaBase: Meta{ override fun equals(other: Any?): Boolean = if(other is Meta) { this.items == other.items -// val items = items -// val otherItems = other.items -// (items.keys == otherItems.keys) && items.keys.all { -// items[it] == otherItems[it] -// } } else { false } diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Specific.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Specific.kt index a89a44a1..571e1ee6 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Specific.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Specific.kt @@ -1,5 +1,7 @@ package hep.dataforge.meta +import hep.dataforge.names.Name + /** * Marker interface for classes with specifications */ @@ -61,5 +63,5 @@ fun > S.createStyle(action: C.() -> Unit): Me fun Specific.spec( spec: Specification, - key: String? = null + key: Name? = null ): MutableMorphDelegate = MutableMorphDelegate(config, key) { spec.wrap(it) } \ No newline at end of file diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/configDelegates.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/configDelegates.kt index 6871b6e3..45b6b4d2 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/configDelegates.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/configDelegates.kt @@ -1,5 +1,6 @@ package hep.dataforge.meta +import hep.dataforge.names.Name import hep.dataforge.values.DoubleArrayValue import hep.dataforge.values.Null import hep.dataforge.values.Value @@ -11,126 +12,126 @@ import kotlin.jvm.JvmName /** * A property delegate that uses custom key */ -fun Configurable.value(default: Any = Null, key: String? = null): MutableValueDelegate = +fun Configurable.value(default: Any = Null, key: Name? = null): MutableValueDelegate = MutableValueDelegate(config, key, Value.of(default)) fun Configurable.value( default: T? = null, - key: String? = null, + key: Name? = null, writer: (T) -> Value = { Value.of(it) }, reader: (Value?) -> T ): ReadWriteDelegateWrapper = MutableValueDelegate(config, key, default?.let { Value.of(it) }).transform(reader = reader, writer = writer) -fun Configurable.string(default: String? = null, key: String? = null): MutableStringDelegate = +fun Configurable.string(default: String? = null, key: Name? = null): MutableStringDelegate = MutableStringDelegate(config, key, default) -fun Configurable.boolean(default: Boolean? = null, key: String? = null): MutableBooleanDelegate = +fun Configurable.boolean(default: Boolean? = null, key: Name? = null): MutableBooleanDelegate = MutableBooleanDelegate(config, key, default) -fun Configurable.number(default: Number? = null, key: String? = null): MutableNumberDelegate = +fun Configurable.number(default: Number? = null, key: Name? = null): MutableNumberDelegate = MutableNumberDelegate(config, key, default) /* Number delegates*/ -fun Configurable.int(default: Int? = null, key: String? = null) = +fun Configurable.int(default: Int? = null, key: Name? = null) = number(default, key).int -fun Configurable.double(default: Double? = null, key: String? = null) = +fun Configurable.double(default: Double? = null, key: Name? = null) = number(default, key).double -fun Configurable.long(default: Long? = null, key: String? = null) = +fun Configurable.long(default: Long? = null, key: Name? = null) = number(default, key).long -fun Configurable.short(default: Short? = null, key: String? = null) = +fun Configurable.short(default: Short? = null, key: Name? = null) = number(default, key).short -fun Configurable.float(default: Float? = null, key: String? = null) = +fun Configurable.float(default: Float? = null, key: Name? = null) = number(default, key).float @JvmName("safeString") -fun Configurable.string(default: String, key: String? = null) = +fun Configurable.string(default: String, key: Name? = null) = MutableSafeStringDelegate(config, key) { default } @JvmName("safeBoolean") -fun Configurable.boolean(default: Boolean, key: String? = null) = +fun Configurable.boolean(default: Boolean, key: Name? = null) = MutableSafeBooleanDelegate(config, key) { default } @JvmName("safeNumber") -fun Configurable.number(default: Number, key: String? = null) = +fun Configurable.number(default: Number, key: Name? = null) = MutableSafeNumberDelegate(config, key) { default } @JvmName("safeString") -fun Configurable.string(key: String? = null, default: () -> String) = +fun Configurable.string(key: Name? = null, default: () -> String) = MutableSafeStringDelegate(config, key, default) @JvmName("safeBoolean") -fun Configurable.boolean(key: String? = null, default: () -> Boolean) = +fun Configurable.boolean(key: Name? = null, default: () -> Boolean) = MutableSafeBooleanDelegate(config, key, default) @JvmName("safeNumber") -fun Configurable.number(key: String? = null, default: () -> Number) = +fun Configurable.number(key: Name? = null, default: () -> Number) = MutableSafeNumberDelegate(config, key, default) /* Safe number delegates*/ @JvmName("safeInt") -fun Configurable.int(default: Int, key: String? = null) = +fun Configurable.int(default: Int, key: Name? = null) = number(default, key).int @JvmName("safeDouble") -fun Configurable.double(default: Double, key: String? = null) = +fun Configurable.double(default: Double, key: Name? = null) = number(default, key).double @JvmName("safeLong") -fun Configurable.long(default: Long, key: String? = null) = +fun Configurable.long(default: Long, key: Name? = null) = number(default, key).long @JvmName("safeShort") -fun Configurable.short(default: Short, key: String? = null) = +fun Configurable.short(default: Short, key: Name? = null) = number(default, key).short @JvmName("safeFloat") -fun Configurable.float(default: Float, key: String? = null) = +fun Configurable.float(default: Float, key: Name? = null) = number(default, key).float /** * Enum delegate */ -inline fun > Configurable.enum(default: E, key: String? = null) = +inline fun > Configurable.enum(default: E, key: Name? = null) = MutableSafeEnumvDelegate(config, key, default) { enumValueOf(it) } /* Node delegates */ -fun Configurable.node(key: String? = null) = MutableNodeDelegate(config, key) +fun Configurable.node(key: Name? = null) = MutableNodeDelegate(config, key) -fun Configurable.spec(spec: Specification, key: String? = null) = +fun Configurable.spec(spec: Specification, key: Name? = null) = MutableMorphDelegate(config, key) { spec.wrap(it) } -fun Configurable.spec(builder: (Config) -> T, key: String? = null) = +fun Configurable.spec(builder: (Config) -> T, key: Name? = null) = MutableMorphDelegate(config, key) { specification(builder).wrap(it) } /* * Extra delegates for special cases */ -fun Configurable.stringList(key: String? = null): ReadWriteDelegateWrapper> = +fun Configurable.stringList(key: Name? = null): ReadWriteDelegateWrapper> = value(emptyList(), key) { it?.list?.map { value -> value.string } ?: emptyList() } -fun Configurable.numberList(key: String? = null): ReadWriteDelegateWrapper> = +fun Configurable.numberList(key: Name? = null): ReadWriteDelegateWrapper> = value(emptyList(), key) { it?.list?.map { value -> value.number } ?: emptyList() } /** * A special delegate for double arrays */ -fun Configurable.doubleArray(key: String? = null): ReadWriteDelegateWrapper = +fun Configurable.doubleArray(key: Name? = null): ReadWriteDelegateWrapper = value(doubleArrayOf(), key) { (it as? DoubleArrayValue)?.value ?: it?.list?.map { value -> value.number.toDouble() }?.toDoubleArray() ?: doubleArrayOf() } -fun Configurable.child(key: String? = null, converter: (Meta) -> T) = +fun Configurable.child(key: Name? = null, converter: (Meta) -> T) = MutableMorphDelegate(config, key, converter) diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/metaDelegates.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/metaDelegates.kt index 9d606b6f..e9c19864 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/metaDelegates.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/metaDelegates.kt @@ -1,5 +1,7 @@ package hep.dataforge.meta +import hep.dataforge.names.Name +import hep.dataforge.names.asName import hep.dataforge.values.Null import hep.dataforge.values.Value import hep.dataforge.values.asValue @@ -173,15 +175,15 @@ fun Metoid.child(key: String? = null, converter: (Meta) -> T) = Chi class MutableValueDelegate>( val meta: M, - private val key: String? = null, + private val key: Name? = null, private val default: Value? = null ) : ReadWriteProperty { override fun getValue(thisRef: Any?, property: KProperty<*>): Value? { - return meta[key ?: property.name]?.value ?: default + return meta[key ?: property.name.asName()]?.value ?: default } override fun setValue(thisRef: Any?, property: KProperty<*>, value: Value?) { - val name = key ?: property.name + val name = key ?: property.name.asName() if (value == null) { meta.remove(name) } else { @@ -195,15 +197,15 @@ class MutableValueDelegate>( class MutableStringDelegate>( val meta: M, - private val key: String? = null, + private val key: Name? = null, private val default: String? = null ) : ReadWriteProperty { override fun getValue(thisRef: Any?, property: KProperty<*>): String? { - return meta[key ?: property.name]?.string ?: default + return meta[key ?: property.name.asName()]?.string ?: default } override fun setValue(thisRef: Any?, property: KProperty<*>, value: String?) { - val name = key ?: property.name + val name = key ?: property.name.asName() if (value == null) { meta.remove(name) } else { @@ -214,15 +216,15 @@ class MutableStringDelegate>( class MutableBooleanDelegate>( val meta: M, - private val key: String? = null, + private val key: Name? = null, private val default: Boolean? = null ) : ReadWriteProperty { override fun getValue(thisRef: Any?, property: KProperty<*>): Boolean? { - return meta[key ?: property.name]?.boolean ?: default + return meta[key ?: property.name.asName()]?.boolean ?: default } override fun setValue(thisRef: Any?, property: KProperty<*>, value: Boolean?) { - val name = key ?: property.name + val name = key ?: property.name.asName() if (value == null) { meta.remove(name) } else { @@ -233,15 +235,15 @@ class MutableBooleanDelegate>( class MutableNumberDelegate>( val meta: M, - private val key: String? = null, + private val key: Name? = null, private val default: Number? = null ) : ReadWriteProperty { override fun getValue(thisRef: Any?, property: KProperty<*>): Number? { - return meta[key ?: property.name]?.number ?: default + return meta[key ?: property.name.asName()]?.number ?: default } override fun setValue(thisRef: Any?, property: KProperty<*>, value: Number?) { - val name = key ?: property.name + val name = key ?: property.name.asName() if (value == null) { meta.remove(name) } else { @@ -260,52 +262,52 @@ class MutableNumberDelegate>( class MutableSafeStringDelegate>( val meta: M, - private val key: String? = null, + private val key: Name? = null, default: () -> String ) : ReadWriteProperty { private val default: String by lazy(default) override fun getValue(thisRef: Any?, property: KProperty<*>): String { - return meta[key ?: property.name]?.string ?: default + return meta[key ?: property.name.asName()]?.string ?: default } override fun setValue(thisRef: Any?, property: KProperty<*>, value: String) { - meta.setValue(key ?: property.name, value.asValue()) + meta.setValue(key ?: property.name.asName(), value.asValue()) } } class MutableSafeBooleanDelegate>( val meta: M, - private val key: String? = null, + private val key: Name? = null, default: () -> Boolean ) : ReadWriteProperty { private val default: Boolean by lazy(default) override fun getValue(thisRef: Any?, property: KProperty<*>): Boolean { - return meta[key ?: property.name]?.boolean ?: default + return meta[key ?: property.name.asName()]?.boolean ?: default } override fun setValue(thisRef: Any?, property: KProperty<*>, value: Boolean) { - meta.setValue(key ?: property.name, value.asValue()) + meta.setValue(key ?: property.name.asName(), value.asValue()) } } class MutableSafeNumberDelegate>( val meta: M, - private val key: String? = null, + private val key: Name? = null, default: () -> Number ) : ReadWriteProperty { private val default: Number by lazy(default) override fun getValue(thisRef: Any?, property: KProperty<*>): Number { - return meta[key ?: property.name]?.number ?: default + return meta[key ?: property.name.asName()]?.number ?: default } override fun setValue(thisRef: Any?, property: KProperty<*>, value: Number) { - meta.setValue(key ?: property.name, value.asValue()) + meta.setValue(key ?: property.name.asName(), value.asValue()) } val double get() = ReadWriteDelegateWrapper(this, reader = { it.toDouble() }, writer = { it }) @@ -317,16 +319,16 @@ class MutableSafeNumberDelegate>( class MutableSafeEnumvDelegate, E : Enum>( val meta: M, - private val key: String? = null, + private val key: Name? = null, private val default: E, private val resolver: (String) -> E ) : ReadWriteProperty { override fun getValue(thisRef: Any?, property: KProperty<*>): E { - return (meta[key ?: property.name]?.string)?.let { resolver(it) } ?: default + return (meta[key ?: property.name.asName()]?.string)?.let { resolver(it) } ?: default } override fun setValue(thisRef: Any?, property: KProperty<*>, value: E) { - meta.setValue(key ?: property.name, value.name.asValue()) + meta.setValue(key ?: property.name.asName(), value.name.asValue()) } } @@ -334,31 +336,31 @@ class MutableSafeEnumvDelegate, E : Enum>( class MutableNodeDelegate>( val meta: M, - private val key: String? = null + private val key: Name? = null ) : ReadWriteProperty { override fun getValue(thisRef: Any?, property: KProperty<*>): Meta? { - return meta[key ?: property.name]?.node + return meta[key ?: property.name.asName()]?.node } override fun setValue(thisRef: Any?, property: KProperty<*>, value: Meta?) { - meta[key ?: property.name] = value + meta[key ?: property.name.asName()] = value } } class MutableMorphDelegate, T : Configurable>( val meta: M, - private val key: String? = null, + private val key: Name? = null, private val converter: (Meta) -> T ) : ReadWriteProperty { override fun getValue(thisRef: Any?, property: KProperty<*>): T? { - return meta[key ?: property.name]?.node?.let(converter) + return meta[key ?: property.name.asName()]?.node?.let(converter) } override fun setValue(thisRef: Any?, property: KProperty<*>, value: T?) { if (value == null) { - meta.remove(key ?: property.name) + meta.remove(key ?: property.name.asName()) } else { - meta[key ?: property.name] = value.config + meta[key ?: property.name.asName()] = value.config } } } @@ -383,44 +385,45 @@ class ReadWriteDelegateWrapper( /** * A property delegate that uses custom key */ -fun > M.value(default: Value = Null, key: String? = null) = +fun > M.value(default: Value = Null, key: Name? = null) = MutableValueDelegate(this, key, default) -fun > M.string(default: String? = null, key: String? = null) = +fun > M.string(default: String? = null, key: Name? = null) = MutableStringDelegate(this, key, default) -fun > M.boolean(default: Boolean? = null, key: String? = null) = +fun > M.boolean(default: Boolean? = null, key: Name? = null) = MutableBooleanDelegate(this, key, default) -fun > M.number(default: Number? = null, key: String? = null) = +fun > M.number(default: Number? = null, key: Name? = null) = MutableNumberDelegate(this, key, default) -fun > M.node(key: String? = null) = MutableNodeDelegate(this, key) +fun > M.node(key: Name? = null) = + MutableNodeDelegate(this, key) @JvmName("safeString") -fun > M.string(default: String, key: String? = null) = +fun > M.string(default: String, key: Name? = null) = MutableSafeStringDelegate(this, key) { default } @JvmName("safeBoolean") -fun > M.boolean(default: Boolean, key: String? = null) = +fun > M.boolean(default: Boolean, key: Name? = null) = MutableSafeBooleanDelegate(this, key) { default } @JvmName("safeNumber") -fun > M.number(default: Number, key: String? = null) = +fun > M.number(default: Number, key: Name? = null) = MutableSafeNumberDelegate(this, key) { default } @JvmName("safeString") -fun > M.string(key: String? = null, default: () -> String) = +fun > M.string(key: Name? = null, default: () -> String) = MutableSafeStringDelegate(this, key, default) @JvmName("safeBoolean") -fun > M.boolean(key: String? = null, default: () -> Boolean) = +fun > M.boolean(key: Name? = null, default: () -> Boolean) = MutableSafeBooleanDelegate(this, key, default) @JvmName("safeNumber") -fun > M.number(key: String? = null, default: () -> Number) = +fun > M.number(key: Name? = null, default: () -> Number) = MutableSafeNumberDelegate(this, key, default) -inline fun , reified E : Enum> M.enum(default: E, key: String? = null) = +inline fun , reified E : Enum> M.enum(default: E, key: Name? = null) = MutableSafeEnumvDelegate(this, key, default) { enumValueOf(it) } \ No newline at end of file From b730d49e6eb37af759e00270b5a423709351e446 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Wed, 11 Sep 2019 20:43:55 +0300 Subject: [PATCH 07/48] Convenience methods to access DataNode elements --- .../kotlin/hep/dataforge/data/DataFilter.kt | 11 +++++++++++ .../commonMain/kotlin/hep/dataforge/data/DataNode.kt | 2 ++ .../jvmMain/kotlin/hep/dataforge/data/CastDataNode.kt | 4 +++- dataforge-io/build.gradle.kts | 6 ++---- .../kotlin/hep/dataforge/workspace/TaskModel.kt | 5 +++-- 5 files changed, 21 insertions(+), 7 deletions(-) diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataFilter.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataFilter.kt index a23b550d..987d8412 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataFilter.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataFilter.kt @@ -5,12 +5,23 @@ import hep.dataforge.names.toName class DataFilter(override val config: Config) : Specific { + /** + * A source node for the filter + */ var from by string() + /** + * A target placement for the filtered node + */ var to by string() + /** + * A regular expression pattern for the filter + */ var pattern by string("*.") // val prefix by string() // val suffix by string() + fun isEmpty(): Boolean = config.isEmpty() + companion object : Specification { override fun wrap(config: Config): DataFilter = DataFilter(config) } diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataNode.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataNode.kt index 108d9f97..a66cff88 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataNode.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataNode.kt @@ -72,6 +72,8 @@ operator fun DataNode.get(name: Name): DataItem? = when (name.le else -> get(name.first()!!.asName()).node?.get(name.cutFirst()) } +operator fun DataNode.get(name: String): DataItem? = get(name.toString()) + /** * Sequence of all children including nodes */ diff --git a/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/CastDataNode.kt b/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/CastDataNode.kt index 01938ddb..c6451518 100644 --- a/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/CastDataNode.kt +++ b/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/CastDataNode.kt @@ -8,7 +8,7 @@ import kotlin.reflect.KClass import kotlin.reflect.full.isSubclassOf @Suppress("UNCHECKED_CAST") -private fun Data.safeCast(type: KClass): Data? { +fun Data.safeCast(type: KClass): Data? { return if (this.type.isSubclassOf(type)) { return object : Data { override val meta: Meta get() = this@safeCast.meta @@ -23,6 +23,8 @@ private fun Data.safeCast(type: KClass): Data? { } } +inline fun Data<*>.safeCast(): Data? = safeCast(R::class) + class CastDataNode(val origin: DataNode, override val type: KClass) : DataNode { override val items: Map> by lazy { origin.items.mapNotNull { (key, item) -> diff --git a/dataforge-io/build.gradle.kts b/dataforge-io/build.gradle.kts index ce3a8be8..8a997017 100644 --- a/dataforge-io/build.gradle.kts +++ b/dataforge-io/build.gradle.kts @@ -15,18 +15,16 @@ kotlin { commonMain{ dependencies { api(project(":dataforge-context")) - api("org.jetbrains.kotlinx:kotlinx-io:0.1.14") } } jvmMain{ dependencies { - api("org.jetbrains.kotlinx:kotlinx-io-jvm:0.1.14") + } } jsMain{ dependencies{ - //api(npm("text-encoding")) - api("org.jetbrains.kotlinx:kotlinx-io-js:0.1.14") + api(npm("text-encoding")) } } } diff --git a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskModel.kt b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskModel.kt index cda164c7..e7443807 100644 --- a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskModel.kt +++ b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskModel.kt @@ -12,6 +12,7 @@ import hep.dataforge.data.dataSequence import hep.dataforge.meta.* import hep.dataforge.names.EmptyName import hep.dataforge.names.Name +import hep.dataforge.names.asName import hep.dataforge.names.toName import hep.dataforge.workspace.TaskModel.Companion.MODEL_TARGET_KEY @@ -43,7 +44,7 @@ data class TaskModel( } companion object { - const val MODEL_TARGET_KEY = "@target" + val MODEL_TARGET_KEY = "@target".asName() } } @@ -83,7 +84,7 @@ class TaskModelBuilder(val name: Name, meta: Meta = EmptyMeta) { } fun dependsOn(name: String, meta: Meta = this.meta, placement: Name = EmptyName) = - dependsOn(name.toName(),meta,placement) + dependsOn(name.toName(), meta, placement) fun dependsOn(task: Task<*>, meta: Meta = this.meta, placement: Name = EmptyName) { dependencies.add(DirectTaskDependency(task, meta, placement)) From 352b98be9bd22e71260222f4736b228aeb51fb14 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Wed, 11 Sep 2019 22:44:56 +0300 Subject: [PATCH 08/48] Added unsafe data casts for all platforms and safeCast for JVM --- .../kotlin/hep/dataforge/data/Data.kt | 25 ++++++++++++--- .../kotlin/hep/dataforge/data/DataNode.kt | 2 +- .../kotlin/hep/dataforge/data/CastDataNode.kt | 31 ++++++------------- .../kotlin/hep/dataforge/data/dataJVM.kt | 16 +++++++--- .../dataforge/io/tcp/EnvelopeServerTest.kt | 4 +-- .../hep/dataforge/workspace/TaskBuilder.kt | 2 +- 6 files changed, 47 insertions(+), 33 deletions(-) diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/Data.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/Data.kt index 9b0d9027..098c5c4c 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/Data.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/Data.kt @@ -4,6 +4,7 @@ import hep.dataforge.meta.EmptyMeta import hep.dataforge.meta.Meta import hep.dataforge.meta.MetaRepr import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Deferred import kotlin.coroutines.CoroutineContext import kotlin.coroutines.EmptyCoroutineContext import kotlin.reflect.KClass @@ -34,7 +35,7 @@ interface Data : Goal, MetaRepr { block: suspend CoroutineScope.() -> T ): Data = DynamicData(type, meta, context, dependencies, block) - operator inline fun invoke( + inline operator fun invoke( meta: Meta = EmptyMeta, context: CoroutineContext = EmptyCoroutineContext, dependencies: Collection> = emptyList(), @@ -50,7 +51,7 @@ interface Data : Goal, MetaRepr { block: suspend CoroutineScope.() -> T ): Data = NamedData(name, invoke(type, meta, context, dependencies, block)) - operator inline fun invoke( + inline operator fun invoke( name: String, meta: Meta = EmptyMeta, context: CoroutineContext = EmptyCoroutineContext, @@ -65,7 +66,7 @@ interface Data : Goal, MetaRepr { } -fun Data.cast(type: KClass): Data { +fun Data.upcast(type: KClass): Data { return object : Data by this { override val type: KClass = type } @@ -74,8 +75,24 @@ fun Data.cast(type: KClass): Data { /** * Upcast a [Data] to a supertype */ -inline fun Data.cast(): Data = cast(R::class) +inline fun Data.upcast(): Data = upcast(R::class) +/** + * Unsafe cast of data node + */ +@Suppress("UNCHECKED_CAST") +fun Data<*>.cast(type: KClass): Data{ + return object : Data { + override val meta: Meta get() = this@cast.meta + override val dependencies: Collection> get() = this@cast.dependencies + override val result: Deferred? get() = this@cast.result as Deferred + override fun startAsync(scope: CoroutineScope): Deferred = this@cast.startAsync(scope) as Deferred + override fun reset() = this@cast.reset() + override val type: KClass = type + } +} + +inline fun Data.cast(): Data = cast(R::class) class DynamicData( override val type: KClass, diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataNode.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataNode.kt index a66cff88..f894c24c 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataNode.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataNode.kt @@ -72,7 +72,7 @@ operator fun DataNode.get(name: Name): DataItem? = when (name.le else -> get(name.first()!!.asName()).node?.get(name.cutFirst()) } -operator fun DataNode.get(name: String): DataItem? = get(name.toString()) +operator fun DataNode.get(name: String): DataItem? = get(name.toName()) /** * Sequence of all children including nodes diff --git a/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/CastDataNode.kt b/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/CastDataNode.kt index c6451518..2ba3319f 100644 --- a/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/CastDataNode.kt +++ b/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/CastDataNode.kt @@ -1,32 +1,21 @@ package hep.dataforge.data -import hep.dataforge.meta.Meta import hep.dataforge.names.NameToken -import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.Deferred import kotlin.reflect.KClass import kotlin.reflect.full.isSubclassOf -@Suppress("UNCHECKED_CAST") -fun Data.safeCast(type: KClass): Data? { - return if (this.type.isSubclassOf(type)) { - return object : Data { - override val meta: Meta get() = this@safeCast.meta - override val dependencies: Collection> get() = this@safeCast.dependencies - override val result: Deferred? get() = this@safeCast.result as Deferred - override fun startAsync(scope: CoroutineScope): Deferred = this@safeCast.startAsync(scope) as Deferred - override fun reset() = this@safeCast.reset() - override val type: KClass = type - } - } else { - null - } -} -inline fun Data<*>.safeCast(): Data? = safeCast(R::class) +fun Data.canCast(type: KClass): Boolean = + this.type.isSubclassOf(type) + +fun Data.safeCast(type: KClass): Data? = + if (canCast(type)) cast(type) else null + + +//inline fun Data<*>.safeCast(): Data? = safeCast(R::class) class CastDataNode(val origin: DataNode, override val type: KClass) : DataNode { - override val items: Map> by lazy { + override val items: Map> by lazy { origin.items.mapNotNull { (key, item) -> when (item) { is DataItem.Leaf -> { @@ -35,7 +24,7 @@ class CastDataNode(val origin: DataNode, override val type: KC } } is DataItem.Node -> { - key to DataItem.Node(item.value.cast(type)) + key to DataItem.Node(item.value.safeCast(type)) } } }.associate { it } diff --git a/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/dataJVM.kt b/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/dataJVM.kt index 32bcec5f..bf803cf0 100644 --- a/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/dataJVM.kt +++ b/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/dataJVM.kt @@ -12,7 +12,7 @@ fun Data.get(): T = runBlocking { await() } /** * Check that node is compatible with given type meaning that each element could be cast to the type */ -actual fun DataNode<*>.checkType(type: KClass) { +actual fun DataNode<*>.checkType(type: KClass) { if (!type.isSuperclassOf(type)) { error("$type expected, but $type received") } @@ -22,12 +22,20 @@ actual fun DataNode<*>.checkType(type: KClass) { * Filter a node by data and node type. Resulting node and its subnodes is guaranteed to have border type [type], * but could contain empty nodes */ -fun DataNode.cast(type: KClass): DataNode { +fun DataNode<*>.safeCast(type: KClass): DataNode { return if (this is CastDataNode) { - origin.cast(type) + origin.safeCast(type) } else { CastDataNode(this, type) } } -inline fun DataNode.cast(): DataNode = cast(R::class) \ No newline at end of file +inline fun DataNode<*>.cast(): DataNode = safeCast(R::class) + +fun DataItem<*>?.safeCast(type: KClass): DataItem? = when (this) { + null -> null + is DataItem.Node -> DataItem.Node(this.value.safeCast(type)) + is DataItem.Leaf -> DataItem.Leaf( + this.value.safeCast(type) ?: error("Can't cast data with type ${this.value.type} to $type") + ) +} \ No newline at end of file diff --git a/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/tcp/EnvelopeServerTest.kt b/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/tcp/EnvelopeServerTest.kt index f00a3754..d48b42ef 100644 --- a/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/tcp/EnvelopeServerTest.kt +++ b/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/tcp/EnvelopeServerTest.kt @@ -43,10 +43,9 @@ class EnvelopeServerTest { } } - @Test fun doEchoTest() { - val client = EnvelopeClient(Global, host = "localhost", port = 7778) + val request = Envelope.build { type = "test.echo" meta { @@ -56,6 +55,7 @@ class EnvelopeServerTest { writeDouble(22.7) } } + val client = EnvelopeClient(Global, host = "localhost", port = 7778) runBlocking { val response = client.respond(request) diff --git a/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/TaskBuilder.kt b/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/TaskBuilder.kt index afe9ddd1..b029079c 100644 --- a/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/TaskBuilder.kt +++ b/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/TaskBuilder.kt @@ -49,7 +49,7 @@ class TaskBuilder(val name: String) { dataTransforms += DataTransformation(from, to) { context, model, data -> data.checkType(inputType) val env = TaskEnv(EmptyName, model.meta, context) - env.block(data.cast(inputType)) + env.block(data.safeCast(inputType)) } } From c239abe6e8cc26e0414dea58fa5a826e319b7826 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Wed, 11 Sep 2019 23:03:56 +0300 Subject: [PATCH 09/48] Added unsafe data casts for all platforms and safeCast for JVM --- .../kotlin/hep/dataforge/data/Data.kt | 29 -------- .../kotlin/hep/dataforge/data/DataNode.kt | 9 +-- .../kotlin/hep/dataforge/data/JoinAction.kt | 2 +- .../kotlin/hep/dataforge/data/PipeAction.kt | 2 +- .../kotlin/hep/dataforge/data/SplitAction.kt | 2 +- .../kotlin/hep/dataforge/data/dataCast.kt | 72 +++++++++++++++++++ .../kotlin/hep/dataforge/data/dataJS.kt | 14 ++-- .../kotlin/hep/dataforge/data/CastDataNode.kt | 10 --- .../kotlin/hep/dataforge/data/dataJVM.kt | 21 +++--- .../hep/dataforge/workspace/TaskBuilder.kt | 2 +- 10 files changed, 96 insertions(+), 67 deletions(-) create mode 100644 dataforge-data/src/commonMain/kotlin/hep/dataforge/data/dataCast.kt diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/Data.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/Data.kt index 098c5c4c..fb56ff19 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/Data.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/Data.kt @@ -4,7 +4,6 @@ import hep.dataforge.meta.EmptyMeta import hep.dataforge.meta.Meta import hep.dataforge.meta.MetaRepr import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.Deferred import kotlin.coroutines.CoroutineContext import kotlin.coroutines.EmptyCoroutineContext import kotlin.reflect.KClass @@ -66,34 +65,6 @@ interface Data : Goal, MetaRepr { } -fun Data.upcast(type: KClass): Data { - return object : Data by this { - override val type: KClass = type - } -} - -/** - * Upcast a [Data] to a supertype - */ -inline fun Data.upcast(): Data = upcast(R::class) - -/** - * Unsafe cast of data node - */ -@Suppress("UNCHECKED_CAST") -fun Data<*>.cast(type: KClass): Data{ - return object : Data { - override val meta: Meta get() = this@cast.meta - override val dependencies: Collection> get() = this@cast.dependencies - override val result: Deferred? get() = this@cast.result as Deferred - override fun startAsync(scope: CoroutineScope): Deferred = this@cast.startAsync(scope) as Deferred - override fun reset() = this@cast.reset() - override val type: KClass = type - } -} - -inline fun Data.cast(): Data = cast(R::class) - class DynamicData( override val type: KClass, override val meta: Meta = EmptyMeta, diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataNode.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataNode.kt index f894c24c..011bbf79 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataNode.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataNode.kt @@ -224,11 +224,4 @@ fun DataNode.filter(predicate: (Name, Data) -> Boolean): DataNod } } -fun DataNode.first(): Data? = dataSequence().first().second - -/** - * Check that node is compatible with given type meaning that each element could be cast to the type - */ -expect fun DataNode<*>.checkType(type: KClass) - -//expect fun DataNode.cast(type: KClass): DataNode \ No newline at end of file +fun DataNode.first(): Data? = dataSequence().first().second \ No newline at end of file diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/JoinAction.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/JoinAction.kt index 4acae87f..31a54a74 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/JoinAction.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/JoinAction.kt @@ -80,7 +80,7 @@ class JoinAction( ) : Action { override fun invoke(node: DataNode, meta: Meta): DataNode { - node.checkType(inputType) + node.ensureType(inputType) return DataNode.build(outputType) { JoinGroupBuilder(meta).apply(action).buildGroups(node).forEach { group -> diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/PipeAction.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/PipeAction.kt index c84e5a13..59162179 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/PipeAction.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/PipeAction.kt @@ -29,7 +29,7 @@ class PipeAction( ) : Action { override fun invoke(node: DataNode, meta: Meta): DataNode { - node.checkType(inputType) + node.ensureType(inputType) return DataNode.build(outputType) { node.dataSequence().forEach { (name, data) -> diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/SplitAction.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/SplitAction.kt index be9764a6..1e7e6c7c 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/SplitAction.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/SplitAction.kt @@ -39,7 +39,7 @@ class SplitAction( ) : Action { override fun invoke(node: DataNode, meta: Meta): DataNode { - node.checkType(inputType) + node.ensureType(inputType) return DataNode.build(outputType) { node.dataSequence().forEach { (name, data) -> diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/dataCast.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/dataCast.kt new file mode 100644 index 00000000..556b77fc --- /dev/null +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/dataCast.kt @@ -0,0 +1,72 @@ +package hep.dataforge.data + +import hep.dataforge.meta.Meta +import hep.dataforge.names.NameToken +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Deferred +import kotlin.reflect.KClass + +fun Data.upcast(type: KClass): Data { + return object : Data by this { + override val type: KClass = type + } +} + +/** + * Safe upcast a [Data] to a supertype + */ +inline fun Data.upcast(): Data = upcast(R::class) + +/** + * Check if node could be safely cast to given class + */ +expect fun DataNode<*>.canCast(type: KClass): Boolean + +/** + * Check if data could be safely cast to given class + */ +expect fun Data<*>.canCast(type: KClass): Boolean + +fun DataItem<*>.canCast(type: KClass): Boolean = when (this) { + is DataItem.Node -> value.canCast(type) + is DataItem.Leaf -> value.canCast(type) +} + +/** + * Unsafe cast of data node + */ +@Suppress("UNCHECKED_CAST") +fun Data<*>.cast(type: KClass): Data { + return object : Data { + override val meta: Meta get() = this@cast.meta + override val dependencies: Collection> get() = this@cast.dependencies + override val result: Deferred? get() = this@cast.result as Deferred + override fun startAsync(scope: CoroutineScope): Deferred = this@cast.startAsync(scope) as Deferred + override fun reset() = this@cast.reset() + override val type: KClass = type + } +} + +inline fun Data<*>.cast(): Data = cast(R::class) + +@Suppress("UNCHECKED_CAST") +fun DataNode<*>.cast(type: KClass): DataNode { + return object : DataNode { + override val type: KClass = type + override val items: Map> get() = this@cast.items as Map> + } +} + +inline fun DataNode<*>.cast(): DataNode = cast(R::class) + +/** + * Check that node is compatible with given type meaning that each element could be cast to the type + */ +fun DataNode<*>.ensureType(type: KClass) { + if (!canCast(type)) { + error("$type expected, but $type received") + } +} + + +//expect fun DataNode.cast(type: KClass): DataNode \ No newline at end of file diff --git a/dataforge-data/src/jsMain/kotlin/hep/dataforge/data/dataJS.kt b/dataforge-data/src/jsMain/kotlin/hep/dataforge/data/dataJS.kt index 7706efad..54148bc5 100644 --- a/dataforge-data/src/jsMain/kotlin/hep/dataforge/data/dataJS.kt +++ b/dataforge-data/src/jsMain/kotlin/hep/dataforge/data/dataJS.kt @@ -5,14 +5,12 @@ import kotlin.reflect.KClass /** * Check that node is compatible with given type meaning that each element could be cast to the type */ -actual fun DataNode<*>.checkType(type: KClass) { +actual fun DataNode<*>.canCast(type: KClass): Boolean { //Not supported in js yet + return true } -///** -// * Performing -// */ -//@Suppress("UNCHECKED_CAST") -//actual fun DataNode.cast(type: KClass): DataNode{ -// return this as DataNode -//} \ No newline at end of file +actual fun Data<*>.canCast(type: KClass): Boolean { + //Not supported in js yet + return true +} diff --git a/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/CastDataNode.kt b/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/CastDataNode.kt index 2ba3319f..79a00075 100644 --- a/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/CastDataNode.kt +++ b/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/CastDataNode.kt @@ -2,18 +2,8 @@ package hep.dataforge.data import hep.dataforge.names.NameToken import kotlin.reflect.KClass -import kotlin.reflect.full.isSubclassOf -fun Data.canCast(type: KClass): Boolean = - this.type.isSubclassOf(type) - -fun Data.safeCast(type: KClass): Data? = - if (canCast(type)) cast(type) else null - - -//inline fun Data<*>.safeCast(): Data? = safeCast(R::class) - class CastDataNode(val origin: DataNode, override val type: KClass) : DataNode { override val items: Map> by lazy { origin.items.mapNotNull { (key, item) -> diff --git a/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/dataJVM.kt b/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/dataJVM.kt index bf803cf0..b2ce7bb6 100644 --- a/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/dataJVM.kt +++ b/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/dataJVM.kt @@ -2,6 +2,7 @@ package hep.dataforge.data import kotlinx.coroutines.runBlocking import kotlin.reflect.KClass +import kotlin.reflect.full.isSubclassOf import kotlin.reflect.full.isSuperclassOf /** @@ -12,11 +13,15 @@ fun Data.get(): T = runBlocking { await() } /** * Check that node is compatible with given type meaning that each element could be cast to the type */ -actual fun DataNode<*>.checkType(type: KClass) { - if (!type.isSuperclassOf(type)) { - error("$type expected, but $type received") - } -} +actual fun DataNode<*>.canCast(type: KClass): Boolean = + type.isSuperclassOf(type) + +actual fun Data<*>.canCast(type: KClass): Boolean = + this.type.isSubclassOf(type) + + +fun Data<*>.safeCast(type: KClass): Data? = + if (canCast(type)) cast(type) else null /** * Filter a node by data and node type. Resulting node and its subnodes is guaranteed to have border type [type], @@ -30,12 +35,12 @@ fun DataNode<*>.safeCast(type: KClass): DataNode { } } -inline fun DataNode<*>.cast(): DataNode = safeCast(R::class) - fun DataItem<*>?.safeCast(type: KClass): DataItem? = when (this) { null -> null is DataItem.Node -> DataItem.Node(this.value.safeCast(type)) is DataItem.Leaf -> DataItem.Leaf( this.value.safeCast(type) ?: error("Can't cast data with type ${this.value.type} to $type") ) -} \ No newline at end of file +} + +inline fun DataItem<*>?.safeCast(): DataItem? = safeCast(R::class) \ No newline at end of file diff --git a/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/TaskBuilder.kt b/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/TaskBuilder.kt index b029079c..ac068a56 100644 --- a/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/TaskBuilder.kt +++ b/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/TaskBuilder.kt @@ -47,7 +47,7 @@ class TaskBuilder(val name: String) { block: TaskEnv.(DataNode) -> DataNode ) { dataTransforms += DataTransformation(from, to) { context, model, data -> - data.checkType(inputType) + data.ensureType(inputType) val env = TaskEnv(EmptyName, model.meta, context) env.block(data.safeCast(inputType)) } From 61c2530c9184b752f723835de22aee48b070b27a Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Thu, 12 Sep 2019 10:39:19 +0300 Subject: [PATCH 10/48] Moved TaskBuilder to common. --- build.gradle.kts | 6 ++-- ...astDataNode.kt => TypeFilteredDataNode.kt} | 9 ++++-- .../kotlin/hep/dataforge/data/dataJVM.kt | 29 +++++++++++-------- .../hep/dataforge/io/TaggedEnvelopeFormat.kt | 1 - .../hep/dataforge/workspace/TaskBuilder.kt | 4 +-- 5 files changed, 28 insertions(+), 21 deletions(-) rename dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/{CastDataNode.kt => TypeFilteredDataNode.kt} (58%) rename dataforge-workspace/src/{jvmMain => commonMain}/kotlin/hep/dataforge/workspace/TaskBuilder.kt (98%) diff --git a/build.gradle.kts b/build.gradle.kts index 495d9a74..41cf03f2 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -1,9 +1,9 @@ plugins { - id("scientifik.mpp") version "0.1.6" apply false - id("scientifik.publish") version "0.1.6" apply false + id("scientifik.mpp") version "0.1.7-dev" apply false + id("scientifik.publish") version "0.1.7-dev" apply false } -val dataforgeVersion by extra("0.1.4-dev-1") +val dataforgeVersion by extra("0.1.4-dev-2") val bintrayRepo by extra("dataforge") val githubProject by extra("dataforge-core") diff --git a/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/CastDataNode.kt b/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/TypeFilteredDataNode.kt similarity index 58% rename from dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/CastDataNode.kt rename to dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/TypeFilteredDataNode.kt index 79a00075..5e90c4ed 100644 --- a/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/CastDataNode.kt +++ b/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/TypeFilteredDataNode.kt @@ -4,17 +4,20 @@ import hep.dataforge.names.NameToken import kotlin.reflect.KClass -class CastDataNode(val origin: DataNode, override val type: KClass) : DataNode { +/** + * A zero-copy data node wrapper that returns only children with appropriate type. + */ +class TypeFilteredDataNode(val origin: DataNode<*>, override val type: KClass) : DataNode { override val items: Map> by lazy { origin.items.mapNotNull { (key, item) -> when (item) { is DataItem.Leaf -> { - (item.value.safeCast(type))?.let { + (item.value.withType(type))?.let { key to DataItem.Leaf(it) } } is DataItem.Node -> { - key to DataItem.Node(item.value.safeCast(type)) + key to DataItem.Node(item.value.withType(type)) } } }.associate { it } diff --git a/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/dataJVM.kt b/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/dataJVM.kt index b2ce7bb6..395f92b1 100644 --- a/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/dataJVM.kt +++ b/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/dataJVM.kt @@ -19,28 +19,33 @@ actual fun DataNode<*>.canCast(type: KClass): Boolean = actual fun Data<*>.canCast(type: KClass): Boolean = this.type.isSubclassOf(type) - -fun Data<*>.safeCast(type: KClass): Data? = +/** + * Cast the node to given type if the cast is possible or return null + */ +fun Data<*>.withType(type: KClass): Data? = if (canCast(type)) cast(type) else null /** * Filter a node by data and node type. Resulting node and its subnodes is guaranteed to have border type [type], * but could contain empty nodes */ -fun DataNode<*>.safeCast(type: KClass): DataNode { - return if (this is CastDataNode) { - origin.safeCast(type) +fun DataNode<*>.withType(type: KClass): DataNode { + return if (canCast(type)) { + cast(type) + } else if (this is TypeFilteredDataNode) { + origin.withType(type) } else { - CastDataNode(this, type) + TypeFilteredDataNode(this, type) } } -fun DataItem<*>?.safeCast(type: KClass): DataItem? = when (this) { +/** + * Filter all elements of given data item that could be cast to given type. If no elements are available, return null. + */ +fun DataItem<*>?.withType(type: KClass): DataItem? = when (this) { null -> null - is DataItem.Node -> DataItem.Node(this.value.safeCast(type)) - is DataItem.Leaf -> DataItem.Leaf( - this.value.safeCast(type) ?: error("Can't cast data with type ${this.value.type} to $type") - ) + is DataItem.Node -> DataItem.Node(this.value.withType(type)) + is DataItem.Leaf -> this.value.withType(type)?.let { DataItem.Leaf(it) } } -inline fun DataItem<*>?.safeCast(): DataItem? = safeCast(R::class) \ No newline at end of file +inline fun DataItem<*>?.withType(): DataItem? = this@withType.withType(R::class) \ No newline at end of file diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt index 7bb04ec4..20fc8632 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt @@ -4,7 +4,6 @@ import hep.dataforge.names.Name import hep.dataforge.names.plus import kotlinx.io.core.* - @ExperimentalUnsignedTypes object TaggedEnvelopeFormat : EnvelopeFormat { const val VERSION = "DF03" diff --git a/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/TaskBuilder.kt b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskBuilder.kt similarity index 98% rename from dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/TaskBuilder.kt rename to dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskBuilder.kt index ac068a56..f999260c 100644 --- a/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/TaskBuilder.kt +++ b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskBuilder.kt @@ -41,7 +41,7 @@ class TaskBuilder(val name: String) { } fun transform( - inputType: KClass, + inputType: KClass, from: String = "", to: String = "", block: TaskEnv.(DataNode) -> DataNode @@ -49,7 +49,7 @@ class TaskBuilder(val name: String) { dataTransforms += DataTransformation(from, to) { context, model, data -> data.ensureType(inputType) val env = TaskEnv(EmptyName, model.meta, context) - env.block(data.safeCast(inputType)) + env.block(data.cast(inputType)) } } From 530b1c1b7603fcbec8309400d525ac2b95ded9fa Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Thu, 12 Sep 2019 11:10:15 +0300 Subject: [PATCH 11/48] Moved TaskBuilder to common. --- .../kotlin/hep/dataforge/workspace/Dependency.kt | 2 +- .../hep/dataforge/workspace/TaskBuilder.kt | 16 +++++++++++++++- .../hep/dataforge/workspace/taskTemplates.kt | 5 +++++ 3 files changed, 21 insertions(+), 2 deletions(-) create mode 100644 dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/taskTemplates.kt diff --git a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/Dependency.kt b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/Dependency.kt index cb3e790e..1e1a9c3f 100644 --- a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/Dependency.kt +++ b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/Dependency.kt @@ -39,7 +39,7 @@ class AllDataDependency(val placement: Name = EmptyName) : Dependency() { } override fun toMeta() = buildMeta { - "data" to "*" + "data" to "@all" "to" to placement } } diff --git a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskBuilder.kt b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskBuilder.kt index f999260c..41b90681 100644 --- a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskBuilder.kt +++ b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskBuilder.kt @@ -14,7 +14,7 @@ import kotlin.reflect.KClass @TaskBuildScope class TaskBuilder(val name: String) { - private var modelTransform: TaskModelBuilder.(Meta) -> Unit = { data("*") } + private var modelTransform: TaskModelBuilder.(Meta) -> Unit = { allData() } var descriptor: NodeDescriptor? = null private val dataTransforms: MutableList = ArrayList() @@ -40,6 +40,20 @@ class TaskBuilder(val name: String) { this.modelTransform = modelTransform } + /** + * Add a transformation on untyped data + */ + fun rawTransform( + from: String = "", + to: String = "", + block: TaskEnv.(DataNode<*>) -> DataNode<*> + ) { + dataTransforms += DataTransformation(from, to){context, model, data-> + val env = TaskEnv(EmptyName, model.meta, context) + env.block(data) + } + } + fun transform( inputType: KClass, from: String = "", diff --git a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/taskTemplates.kt b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/taskTemplates.kt new file mode 100644 index 00000000..ab282015 --- /dev/null +++ b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/taskTemplates.kt @@ -0,0 +1,5 @@ +package hep.dataforge.workspace + +//fun TaskBuilder.zip( +// val firstNo +//) = rawTransform { } \ No newline at end of file From 74b5a1ac5036c96e6c2aa4bd55fbfe5325ec6a12 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Thu, 12 Sep 2019 16:04:22 +0300 Subject: [PATCH 12/48] Test for node builder. Fixed bug with node content resolve by name --- .../kotlin/hep/dataforge/data/Data.kt | 13 +++++--- .../kotlin/hep/dataforge/data/DataNode.kt | 26 ++++++++++++--- .../hep/dataforge/data/DataTreeBuilderTest.kt | 32 +++++++++++++++++++ 3 files changed, 62 insertions(+), 9 deletions(-) create mode 100644 dataforge-data/src/commonTest/kotlin/hep/dataforge/data/DataTreeBuilderTest.kt diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/Data.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/Data.kt index fb56ff19..bd93ddfc 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/Data.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/Data.kt @@ -1,8 +1,6 @@ package hep.dataforge.data -import hep.dataforge.meta.EmptyMeta -import hep.dataforge.meta.Meta -import hep.dataforge.meta.MetaRepr +import hep.dataforge.meta.* import kotlinx.coroutines.CoroutineScope import kotlin.coroutines.CoroutineContext import kotlin.coroutines.EmptyCoroutineContext @@ -11,7 +9,7 @@ import kotlin.reflect.KClass /** * A data element characterized by its meta */ -interface Data : Goal, MetaRepr { +interface Data : Goal, MetaRepr{ /** * Type marker for the data. The type is known before the calculation takes place so it could be checked. */ @@ -21,7 +19,12 @@ interface Data : Goal, MetaRepr { */ val meta: Meta - override fun toMeta(): Meta = meta + override fun toMeta(): Meta = buildMeta { + "type" to (type.simpleName?:"undefined") + if(!meta.isEmpty()) { + "meta" to meta + } + } companion object { const val TYPE = "data" diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataNode.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataNode.kt index 011bbf79..4cc5aaf0 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataNode.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataNode.kt @@ -1,5 +1,8 @@ package hep.dataforge.data +import hep.dataforge.meta.Meta +import hep.dataforge.meta.MetaRepr +import hep.dataforge.meta.buildMeta import hep.dataforge.names.* import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Job @@ -9,22 +12,26 @@ import kotlin.collections.component2 import kotlin.collections.set import kotlin.reflect.KClass -sealed class DataItem { +sealed class DataItem : MetaRepr { abstract val type: KClass class Node(val value: DataNode) : DataItem() { override val type: KClass get() = value.type + + override fun toMeta(): Meta = value.toMeta() } class Leaf(val value: Data) : DataItem() { override val type: KClass get() = value.type + + override fun toMeta(): Meta = value.toMeta() } } /** * A tree-like data structure grouped into the node. All data inside the node must inherit its type */ -interface DataNode { +interface DataNode: MetaRepr { /** * The minimal common ancestor to all data in the node @@ -33,6 +40,15 @@ interface DataNode { val items: Map> + override fun toMeta(): Meta = buildMeta { + "type" to (type.simpleName?:"undefined") + "items" to { + this@DataNode.items.forEach { + it.key.toString() to it.value.toMeta() + } + } + } + companion object { const val TYPE = "dataNode" @@ -68,7 +84,7 @@ fun DataNode<*>.joinAll(scope: CoroutineScope): Job = scope.launch { operator fun DataNode.get(name: Name): DataItem? = when (name.length) { 0 -> error("Empty name") - 1 -> (items[name.first()] as? DataItem.Leaf) + 1 -> items[name.first()] else -> get(name.first()!!.asName()).node?.get(name.cutFirst()) } @@ -110,7 +126,9 @@ class DataTree internal constructor( override val type: KClass, override val items: Map> ) : DataNode { - //TODO add node-level meta? + override fun toString(): String { + return super.toString() + } } private sealed class DataTreeBuilderItem { diff --git a/dataforge-data/src/commonTest/kotlin/hep/dataforge/data/DataTreeBuilderTest.kt b/dataforge-data/src/commonTest/kotlin/hep/dataforge/data/DataTreeBuilderTest.kt new file mode 100644 index 00000000..a3bb509b --- /dev/null +++ b/dataforge-data/src/commonTest/kotlin/hep/dataforge/data/DataTreeBuilderTest.kt @@ -0,0 +1,32 @@ +package hep.dataforge.data + +import kotlin.test.Test +import kotlin.test.assertTrue + + +internal class DataTreeBuilderTest{ + @Test + fun testDataUpdate(){ + val updateData = DataNode.build(Any::class){ + "update" to { + "a" to Data.static("a") + "b" to Data.static("b") + } + } + + val node = DataNode.build(Any::class){ + "primary" to { + "a" to Data.static("a") + "b" to Data.static("b") + } + "root" to Data.static("root") + update(updateData) + } + + println(node.toMeta()) + + assertTrue { node["update.a"] != null } + assertTrue { node["primary.a"] != null } + + } +} \ No newline at end of file From a0abb99d8851e620b6b652132319daa35ed7df72 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Thu, 12 Sep 2019 16:53:57 +0300 Subject: [PATCH 13/48] A test for complex task logic --- .../workspace/SimpleWorkspaceTest.kt | 47 +++++++++++++++++-- 1 file changed, 42 insertions(+), 5 deletions(-) diff --git a/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/SimpleWorkspaceTest.kt b/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/SimpleWorkspaceTest.kt index b64ee467..95233251 100644 --- a/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/SimpleWorkspaceTest.kt +++ b/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/SimpleWorkspaceTest.kt @@ -1,11 +1,11 @@ package hep.dataforge.workspace import hep.dataforge.context.PluginTag -import hep.dataforge.data.first -import hep.dataforge.data.get +import hep.dataforge.data.* import hep.dataforge.meta.boolean import hep.dataforge.meta.get -import org.junit.Test +import hep.dataforge.names.asName +import kotlin.test.Test import kotlin.test.assertEquals import kotlin.test.assertTrue @@ -33,7 +33,7 @@ class SimpleWorkspaceTest { } - task("square") { + val square = task("square") { model { allData() } @@ -46,6 +46,37 @@ class SimpleWorkspaceTest { } } + val linear = task("linear") { + model { + allData() + } + pipe { data -> + context.logger.info { "Starting linear on $data" } + data * 2 + 1 + } + } + + val fullSquare = task("fullsquare") { + model { + dependsOn("square", placement = "square".asName()) + dependsOn("linear", placement = "linear".asName()) + } + transform { data -> + val squareNode = data["square"].withType().node!! + val linearNode = data["linear"].withType().node!! + return@transform DataNode.build(Int::class) { + squareNode.dataSequence().forEach { (name, _) -> + val newData = Data{ + val squareValue = squareNode[name].data!!.get() + val linearValue = linearNode[name].data!!.get() + squareValue+linearValue + } + set(name,newData) + } + } + } + } + task("sum") { model { dependsOn("square") @@ -60,7 +91,7 @@ class SimpleWorkspaceTest { model { allData() } - joinByGroup {env-> + joinByGroup { env -> group("even", filter = { name, _ -> name.toString().toInt() % 2 == 0 }) { result { data -> env.context.logger.info { "Starting even" } @@ -107,4 +138,10 @@ class SimpleWorkspaceTest { assertTrue { tasks["test.test"] != null } //val node = workspace.run("test.test", "empty") } + + @Test + fun testFullSquare(){ + val node = workspace.run("fullsquare") + println(node.toMeta()) + } } \ No newline at end of file From c175dc7de4d6ff490b9150dcf6a4eddfcceccfc7 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Sat, 14 Sep 2019 10:36:47 +0300 Subject: [PATCH 14/48] Added strong typing to tasks and task dependencies --- .../kotlin/hep/dataforge/data/DataNode.kt | 65 ++++++++++++++---- .../kotlin/hep/dataforge/data/JoinAction.kt | 6 +- .../kotlin/hep/dataforge/data/PipeAction.kt | 10 +-- .../kotlin/hep/dataforge/data/SplitAction.kt | 6 +- .../dataforge/data/TypeFilteredDataNode.kt | 4 +- .../kotlin/hep/dataforge/data/dataJVM.kt | 14 ++-- .../hep/dataforge/workspace/Dependency.kt | 25 ++++--- .../hep/dataforge/workspace/GenericTask.kt | 2 +- .../hep/dataforge/workspace/TaskBuilder.kt | 60 +++++++++-------- .../hep/dataforge/workspace/TaskModel.kt | 2 +- .../dataforge/workspace/WorkspaceBuilder.kt | 66 +++++++++++-------- .../hep/dataforge/workspace/taskTemplates.kt | 4 +- .../workspace/SimpleWorkspaceTest.kt | 38 ++++++----- 13 files changed, 180 insertions(+), 122 deletions(-) diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataNode.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataNode.kt index 4cc5aaf0..d7158dab 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataNode.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataNode.kt @@ -1,8 +1,6 @@ package hep.dataforge.data -import hep.dataforge.meta.Meta -import hep.dataforge.meta.MetaRepr -import hep.dataforge.meta.buildMeta +import hep.dataforge.meta.* import hep.dataforge.names.* import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Job @@ -18,20 +16,20 @@ sealed class DataItem : MetaRepr { class Node(val value: DataNode) : DataItem() { override val type: KClass get() = value.type - override fun toMeta(): Meta = value.toMeta() + override fun toMeta(): Meta = value.toMeta() } class Leaf(val value: Data) : DataItem() { override val type: KClass get() = value.type - override fun toMeta(): Meta = value.toMeta() + override fun toMeta(): Meta = value.toMeta() } } /** * A tree-like data structure grouped into the node. All data inside the node must inherit its type */ -interface DataNode: MetaRepr { +interface DataNode : MetaRepr { /** * The minimal common ancestor to all data in the node @@ -41,7 +39,7 @@ interface DataNode: MetaRepr { val items: Map> override fun toMeta(): Meta = buildMeta { - "type" to (type.simpleName?:"undefined") + "type" to (type.simpleName ?: "undefined") "items" to { this@DataNode.items.forEach { it.key.toString() to it.value.toMeta() @@ -52,9 +50,12 @@ interface DataNode: MetaRepr { companion object { const val TYPE = "dataNode" - fun build(type: KClass, block: DataTreeBuilder.() -> Unit) = + operator fun invoke(type: KClass, block: DataTreeBuilder.() -> Unit) = DataTreeBuilder(type).apply(block).build() + inline operator fun invoke(noinline block: DataTreeBuilder.() -> Unit) = + DataTreeBuilder(T::class).apply(block).build() + fun builder(type: KClass) = DataTreeBuilder(type) } } @@ -142,7 +143,7 @@ private sealed class DataTreeBuilderItem { class DataTreeBuilder(private val type: KClass) { private val map = HashMap>() - operator fun set(token: NameToken, node: DataTreeBuilder) { + operator fun set(token: NameToken, node: DataTreeBuilder) { if (map.containsKey(token)) error("Tree entry with name $token is not empty") map[token] = DataTreeBuilderItem.Node(node) } @@ -154,9 +155,9 @@ class DataTreeBuilder(private val type: KClass) { private fun buildNode(token: NameToken): DataTreeBuilder { return if (!map.containsKey(token)) { - DataTreeBuilder(type).also { map[token] = DataTreeBuilderItem.Node(it) } + DataTreeBuilder(type).also { map[token] = DataTreeBuilderItem.Node(it) } } else { - (map[token] as? DataTreeBuilderItem.Node ?: error("The node with name $token is occupied by leaf")).tree + (map[token] as? DataTreeBuilderItem.Node ?: error("The node with name $token is occupied by leaf")).tree } } @@ -176,7 +177,7 @@ class DataTreeBuilder(private val type: KClass) { } } - operator fun set(name: Name, node: DataTreeBuilder) { + operator fun set(name: Name, node: DataTreeBuilder) { when (name.length) { 0 -> error("Can't add data with empty name") 1 -> set(name.first()!!, node) @@ -206,7 +207,7 @@ class DataTreeBuilder(private val type: KClass) { /** * Build and append node */ - infix fun String.to(block: DataTreeBuilder.() -> Unit) = set(toName(), DataTreeBuilder(type).apply(block)) + infix fun String.to(block: DataTreeBuilder.() -> Unit) = set(toName(), DataTreeBuilder(type).apply(block)) fun update(node: DataNode) { @@ -227,6 +228,42 @@ class DataTreeBuilder(private val type: KClass) { } } +fun DataTreeBuilder.datum(name: Name, data: Data) { + this[name] = data +} + +fun DataTreeBuilder.datum(name: String, data: Data) { + this[name.toName()] = data +} + +fun DataTreeBuilder.static(name: Name, data: T, meta: Meta = EmptyMeta) { + this[name] = Data.static(data, meta) +} + +fun DataTreeBuilder.static(name: Name, data: T, block: MetaBuilder.() -> Unit = {}) { + this[name] = Data.static(data, buildMeta(block)) +} + +fun DataTreeBuilder.static(name: String, data: T, block: MetaBuilder.() -> Unit = {}) { + this[name.toName()] = Data.static(data, buildMeta(block)) +} + +fun DataTreeBuilder.node(name: Name, node: DataNode) { + this[name] = node +} + +fun DataTreeBuilder.node(name: String, node: DataNode) { + this[name.toName()] = node +} + +inline fun DataTreeBuilder.node(name: Name, noinline block: DataTreeBuilder.() -> Unit) { + this[name] = DataNode(T::class, block) +} + +inline fun DataTreeBuilder.node(name: String, noinline block: DataTreeBuilder.() -> Unit) { + this[name.toName()] = DataNode(T::class, block) +} + /** * Generate a mutable builder from this node. Node content is not changed */ @@ -234,7 +271,7 @@ fun DataNode.builder(): DataTreeBuilder = DataTreeBuilder(type). dataSequence().forEach { (name, data) -> this[name] = data } } -fun DataNode.filter(predicate: (Name, Data) -> Boolean): DataNode = DataNode.build(type) { +fun DataNode.filter(predicate: (Name, Data) -> Boolean): DataNode = DataNode.invoke(type) { dataSequence().forEach { (name, data) -> if (predicate(name, data)) { this[name] = data diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/JoinAction.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/JoinAction.kt index 31a54a74..7ad4055e 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/JoinAction.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/JoinAction.kt @@ -74,14 +74,14 @@ class JoinGroupBuilder(val actionMeta: Meta) { * The same rules as for KPipe */ class JoinAction( - val inputType: KClass, - val outputType: KClass, + val inputType: KClass, + val outputType: KClass, private val action: JoinGroupBuilder.() -> Unit ) : Action { override fun invoke(node: DataNode, meta: Meta): DataNode { node.ensureType(inputType) - return DataNode.build(outputType) { + return DataNode.invoke(outputType) { JoinGroupBuilder(meta).apply(action).buildGroups(node).forEach { group -> val laminate = Laminate(group.meta, meta) diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/PipeAction.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/PipeAction.kt index 59162179..91a204d8 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/PipeAction.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/PipeAction.kt @@ -22,16 +22,16 @@ class PipeBuilder(var name: Name, var meta: MetaBuilder) { } -class PipeAction( - val inputType: KClass, - val outputType: KClass, +class PipeAction( + val inputType: KClass, + val outputType: KClass, private val block: PipeBuilder.() -> Unit ) : Action { override fun invoke(node: DataNode, meta: Meta): DataNode { node.ensureType(inputType) - return DataNode.build(outputType) { + return DataNode.invoke(outputType) { node.dataSequence().forEach { (name, data) -> //merging data meta with action meta (data meta is primary) val oldMeta = meta.builder().apply { update(data.meta) } @@ -53,7 +53,7 @@ class PipeAction( inline fun DataNode.pipe( meta: Meta, - noinline action: PipeBuilder.() -> Unit + noinline action: PipeBuilder.() -> Unit ): DataNode = PipeAction(T::class, R::class, action).invoke(this, meta) diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/SplitAction.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/SplitAction.kt index 1e7e6c7c..637baeeb 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/SplitAction.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/SplitAction.kt @@ -33,15 +33,15 @@ class SplitBuilder(val name: Name, val meta: Meta) { } class SplitAction( - val inputType: KClass, - val outputType: KClass, + val inputType: KClass, + val outputType: KClass, private val action: SplitBuilder.() -> Unit ) : Action { override fun invoke(node: DataNode, meta: Meta): DataNode { node.ensureType(inputType) - return DataNode.build(outputType) { + return DataNode.invoke(outputType) { node.dataSequence().forEach { (name, data) -> val laminate = Laminate(data.meta, meta) diff --git a/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/TypeFilteredDataNode.kt b/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/TypeFilteredDataNode.kt index 5e90c4ed..d24de964 100644 --- a/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/TypeFilteredDataNode.kt +++ b/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/TypeFilteredDataNode.kt @@ -12,12 +12,12 @@ class TypeFilteredDataNode(val origin: DataNode<*>, override val ty origin.items.mapNotNull { (key, item) -> when (item) { is DataItem.Leaf -> { - (item.value.withType(type))?.let { + (item.value.filterIsInstance(type))?.let { key to DataItem.Leaf(it) } } is DataItem.Node -> { - key to DataItem.Node(item.value.withType(type)) + key to DataItem.Node(item.value.filterIsInstance(type)) } } }.associate { it } diff --git a/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/dataJVM.kt b/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/dataJVM.kt index 395f92b1..5b5507b2 100644 --- a/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/dataJVM.kt +++ b/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/dataJVM.kt @@ -22,18 +22,18 @@ actual fun Data<*>.canCast(type: KClass): Boolean = /** * Cast the node to given type if the cast is possible or return null */ -fun Data<*>.withType(type: KClass): Data? = +fun Data<*>.filterIsInstance(type: KClass): Data? = if (canCast(type)) cast(type) else null /** * Filter a node by data and node type. Resulting node and its subnodes is guaranteed to have border type [type], * but could contain empty nodes */ -fun DataNode<*>.withType(type: KClass): DataNode { +fun DataNode<*>.filterIsInstance(type: KClass): DataNode { return if (canCast(type)) { cast(type) } else if (this is TypeFilteredDataNode) { - origin.withType(type) + origin.filterIsInstance(type) } else { TypeFilteredDataNode(this, type) } @@ -42,10 +42,10 @@ fun DataNode<*>.withType(type: KClass): DataNode { /** * Filter all elements of given data item that could be cast to given type. If no elements are available, return null. */ -fun DataItem<*>?.withType(type: KClass): DataItem? = when (this) { +fun DataItem<*>?.filterIsInstance(type: KClass): DataItem? = when (this) { null -> null - is DataItem.Node -> DataItem.Node(this.value.withType(type)) - is DataItem.Leaf -> this.value.withType(type)?.let { DataItem.Leaf(it) } + is DataItem.Node -> DataItem.Node(this.value.filterIsInstance(type)) + is DataItem.Leaf -> this.value.filterIsInstance(type)?.let { DataItem.Leaf(it) } } -inline fun DataItem<*>?.withType(): DataItem? = this@withType.withType(R::class) \ No newline at end of file +inline fun DataItem<*>?.filterIsInstance(): DataItem? = this@filterIsInstance.filterIsInstance(R::class) \ No newline at end of file diff --git a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/Dependency.kt b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/Dependency.kt index 1e1a9c3f..1f184095 100644 --- a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/Dependency.kt +++ b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/Dependency.kt @@ -21,7 +21,7 @@ class DataDependency(val filter: DataFilter, val placement: Name = EmptyName) : return if (placement.isEmpty()) { result } else { - DataNode.build(Any::class) { this[placement] = result } + DataNode.invoke(Any::class) { this[placement] = result } } } @@ -35,7 +35,7 @@ class AllDataDependency(val placement: Name = EmptyName) : Dependency() { override fun apply(workspace: Workspace): DataNode = if (placement.isEmpty()) { workspace.data } else { - DataNode.build(Any::class) { this[placement] = workspace.data } + DataNode.invoke(Any::class) { this[placement] = workspace.data } } override fun toMeta() = buildMeta { @@ -44,22 +44,25 @@ class AllDataDependency(val placement: Name = EmptyName) : Dependency() { } } -abstract class TaskDependency(val meta: Meta, val placement: Name = EmptyName) : Dependency() { - abstract fun resolveTask(workspace: Workspace): Task<*> +abstract class TaskDependency( + val meta: Meta, + val placement: Name = EmptyName +) : Dependency() { + abstract fun resolveTask(workspace: Workspace): Task /** * A name of the dependency for logging and serialization */ abstract val name: Name - override fun apply(workspace: Workspace): DataNode { + override fun apply(workspace: Workspace): DataNode { val task = resolveTask(workspace) if (task.isTerminal) TODO("Support terminal task") val result = workspace.run(task, meta) return if (placement.isEmpty()) { result } else { - DataNode.build(Any::class) { this[placement] = result } + DataNode(task.type) { this[placement] = result } } } @@ -70,8 +73,12 @@ abstract class TaskDependency(val meta: Meta, val placement: Name = EmptyName) : } } -class DirectTaskDependency(val task: Task<*>, meta: Meta, placement: Name) : TaskDependency(meta, placement) { - override fun resolveTask(workspace: Workspace): Task<*> = task +class DirectTaskDependency( + val task: Task, + meta: Meta, + placement: Name +) : TaskDependency(meta, placement) { + override fun resolveTask(workspace: Workspace): Task = task override val name: Name get() = DIRECT_TASK_NAME + task.name @@ -80,7 +87,7 @@ class DirectTaskDependency(val task: Task<*>, meta: Meta, placement: Name) : Tas } } -class WorkspaceTaskDependency(override val name: Name, meta: Meta, placement: Name) : TaskDependency(meta, placement) { +class WorkspaceTaskDependency(override val name: Name, meta: Meta, placement: Name) : TaskDependency(meta, placement) { override fun resolveTask(workspace: Workspace): Task<*> = workspace.tasks[name] ?: error("Task with name $name is not found in the workspace") } \ No newline at end of file diff --git a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/GenericTask.kt b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/GenericTask.kt index a46862a0..408f1745 100644 --- a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/GenericTask.kt +++ b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/GenericTask.kt @@ -20,7 +20,7 @@ class GenericTask( ) : Task { private fun gather(workspace: Workspace, model: TaskModel): DataNode { - return DataNode.build(Any::class) { + return DataNode.invoke(Any::class) { model.dependencies.forEach { dep -> update(dep.apply(workspace)) } diff --git a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskBuilder.kt b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskBuilder.kt index 41b90681..bdacb711 100644 --- a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskBuilder.kt +++ b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskBuilder.kt @@ -13,7 +13,7 @@ import hep.dataforge.names.toName import kotlin.reflect.KClass @TaskBuildScope -class TaskBuilder(val name: String) { +class TaskBuilder(val name: String, val type: KClass) { private var modelTransform: TaskModelBuilder.(Meta) -> Unit = { allData() } var descriptor: NodeDescriptor? = null private val dataTransforms: MutableList = ArrayList() @@ -21,12 +21,12 @@ class TaskBuilder(val name: String) { /** * TODO will look better as extension class */ - private class DataTransformation( + private inner class DataTransformation( val from: String = "", val to: String = "", - val transform: (Context, TaskModel, DataNode) -> DataNode + val transform: (Context, TaskModel, DataNode) -> DataNode ) { - operator fun invoke(workspace: Workspace, model: TaskModel, node: DataNode): DataNode? { + operator fun invoke(workspace: Workspace, model: TaskModel, node: DataNode): DataNode? { val localData = if (from.isEmpty()) { node } else { @@ -46,9 +46,9 @@ class TaskBuilder(val name: String) { fun rawTransform( from: String = "", to: String = "", - block: TaskEnv.(DataNode<*>) -> DataNode<*> + block: TaskEnv.(DataNode<*>) -> DataNode ) { - dataTransforms += DataTransformation(from, to){context, model, data-> + dataTransforms += DataTransformation(from, to) { context, model, data -> val env = TaskEnv(EmptyName, model.meta, context) env.block(data) } @@ -58,7 +58,7 @@ class TaskBuilder(val name: String) { inputType: KClass, from: String = "", to: String = "", - block: TaskEnv.(DataNode) -> DataNode + block: TaskEnv.(DataNode) -> DataNode ) { dataTransforms += DataTransformation(from, to) { context, model, data -> data.ensureType(inputType) @@ -70,7 +70,7 @@ class TaskBuilder(val name: String) { inline fun transform( from: String = "", to: String = "", - noinline block: TaskEnv.(DataNode) -> DataNode + noinline block: TaskEnv.(DataNode) -> DataNode ) { transform(T::class, from, to, block) } @@ -78,7 +78,7 @@ class TaskBuilder(val name: String) { /** * Perform given action on data elements in `from` node in input and put the result to `to` node */ - inline fun action( + inline fun action( from: String = "", to: String = "", crossinline block: TaskEnv.() -> Action @@ -93,7 +93,7 @@ class TaskBuilder(val name: String) { /** * A customized pipe action with ability to change meta and name */ - inline fun customPipe( + inline fun customPipe( from: String = "", to: String = "", crossinline block: PipeBuilder.(TaskEnv) -> Unit @@ -101,7 +101,7 @@ class TaskBuilder(val name: String) { action(from, to) { PipeAction( inputType = T::class, - outputType = R::class + outputType = type ) { block(this@action) } } } @@ -109,7 +109,7 @@ class TaskBuilder(val name: String) { /** * A simple pipe action without changing meta or name */ - inline fun pipe( + inline fun pipe( from: String = "", to: String = "", crossinline block: suspend TaskEnv.(T) -> R @@ -117,7 +117,7 @@ class TaskBuilder(val name: String) { action(from, to) { PipeAction( inputType = T::class, - outputType = R::class + outputType = type ) { //TODO automatically append task meta result = { data -> @@ -130,7 +130,7 @@ class TaskBuilder(val name: String) { /** * Join elements in gathered data by multiple groups */ - inline fun joinByGroup( + inline fun joinByGroup( from: String = "", to: String = "", crossinline block: JoinGroupBuilder.(TaskEnv) -> Unit @@ -138,7 +138,7 @@ class TaskBuilder(val name: String) { action(from, to) { JoinAction( inputType = T::class, - outputType = R::class + outputType = type ) { block(this@action) } } } @@ -146,7 +146,7 @@ class TaskBuilder(val name: String) { /** * Join all elemlents in gathered data matching input type */ - inline fun join( + inline fun join( from: String = "", to: String = "", crossinline block: suspend TaskEnv.(Map) -> R @@ -154,7 +154,7 @@ class TaskBuilder(val name: String) { action(from, to) { JoinAction( inputType = T::class, - outputType = R::class, + outputType = type, action = { result( actionMeta[TaskModel.MODEL_TARGET_KEY]?.string ?: "@anonymous" @@ -169,7 +169,7 @@ class TaskBuilder(val name: String) { /** * Split each element in gathered data into fixed number of fragments */ - inline fun split( + inline fun split( from: String = "", to: String = "", crossinline block: SplitBuilder.(TaskEnv) -> Unit @@ -177,7 +177,7 @@ class TaskBuilder(val name: String) { action(from, to) { SplitAction( inputType = T::class, - outputType = R::class + outputType = type ) { block(this@action) } } } @@ -189,22 +189,23 @@ class TaskBuilder(val name: String) { this.descriptor = NodeDescriptor.build(transform) } - internal fun build(): GenericTask = + internal fun build(): GenericTask = GenericTask( name.asName(), - Any::class, + type, descriptor ?: NodeDescriptor.empty(), modelTransform ) { val workspace = this - { data -> + return@GenericTask { data -> val model = this if (dataTransforms.isEmpty()) { //return data node as is logger.warn { "No transformation present, returning input data" } - data + data.ensureType(type) + data.cast(type) } else { - val builder = DataTreeBuilder(Any::class) + val builder = DataTreeBuilder(type) dataTransforms.forEach { transformation -> val res = transformation(workspace, model, data) if (res != null) { @@ -221,12 +222,13 @@ class TaskBuilder(val name: String) { } } -fun Workspace.Companion.task(name: String, builder: TaskBuilder.() -> Unit): GenericTask { - return TaskBuilder(name).apply(builder).build() +fun Workspace.Companion.task( + name: String, + type: KClass, + builder: TaskBuilder.() -> Unit +): GenericTask { + return TaskBuilder(name, type).apply(builder).build() } -fun WorkspaceBuilder.task(name: String, builder: TaskBuilder.() -> Unit) { - task(TaskBuilder(name).apply(builder).build()) -} //TODO add delegates to build gradle-like tasks \ No newline at end of file diff --git a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskModel.kt b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskModel.kt index e7443807..f9562438 100644 --- a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskModel.kt +++ b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskModel.kt @@ -36,7 +36,7 @@ data class TaskModel( "meta" to meta "dependsOn" to { val dataDependencies = dependencies.filterIsInstance() - val taskDependencies = dependencies.filterIsInstance() + val taskDependencies = dependencies.filterIsInstance>() setIndexed("data".toName(), dataDependencies.map { it.toMeta() }) setIndexed("task".toName(), taskDependencies.map { it.toMeta() }) { taskDependencies[it].name.toString() } //TODO ensure all dependencies are listed diff --git a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/WorkspaceBuilder.kt b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/WorkspaceBuilder.kt index 999ee50c..6bc4900a 100644 --- a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/WorkspaceBuilder.kt +++ b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/WorkspaceBuilder.kt @@ -2,12 +2,14 @@ package hep.dataforge.workspace import hep.dataforge.context.Context import hep.dataforge.context.ContextBuilder -import hep.dataforge.data.Data import hep.dataforge.data.DataNode import hep.dataforge.data.DataTreeBuilder import hep.dataforge.meta.* +import hep.dataforge.names.EmptyName import hep.dataforge.names.Name -import hep.dataforge.names.toName +import hep.dataforge.names.isEmpty +import kotlin.jvm.JvmName +import kotlin.reflect.KClass @TaskBuildScope interface WorkspaceBuilder { @@ -28,32 +30,26 @@ fun WorkspaceBuilder.context(name: String = "WORKSPACE", block: ContextBuilder.( context = ContextBuilder(name, parentContext).apply(block).build() } -fun WorkspaceBuilder.data(name: Name, data: Data) { - this.data[name] = data +inline fun WorkspaceBuilder.data( + name: Name = EmptyName, + noinline block: DataTreeBuilder.() -> Unit +): DataNode { + val node = DataTreeBuilder(T::class).apply(block) + if (name.isEmpty()) { + @Suppress("UNCHECKED_CAST") + data = node as DataTreeBuilder + } else { + data[name] = node + } + return node.build() } -fun WorkspaceBuilder.data(name: String, data: Data) = data(name.toName(), data) +@JvmName("rawData") +fun WorkspaceBuilder.data( + name: Name = EmptyName, + block: DataTreeBuilder.() -> Unit +): DataNode = data(name, block) -fun WorkspaceBuilder.static(name: Name, data: Any, meta: Meta = EmptyMeta) = - data(name, Data.static(data, meta)) - -fun WorkspaceBuilder.static(name: Name, data: Any, block: MetaBuilder.() -> Unit = {}) = - data(name, Data.static(data, buildMeta(block))) - -fun WorkspaceBuilder.static(name: String, data: Any, block: MetaBuilder.() -> Unit = {}) = - data(name, Data.static(data, buildMeta(block))) - -fun WorkspaceBuilder.data(name: Name, node: DataNode) { - this.data[name] = node -} - -fun WorkspaceBuilder.data(name: String, node: DataNode) = data(name.toName(), node) - -fun WorkspaceBuilder.data(name: Name, block: DataTreeBuilder.() -> Unit) { - this.data[name] = DataNode.build(Any::class, block) -} - -fun WorkspaceBuilder.data(name: String, block: DataTreeBuilder.() -> Unit) = data(name.toName(), block) fun WorkspaceBuilder.target(name: String, block: MetaBuilder.() -> Unit) { targets[name] = buildMeta(block).seal() @@ -70,17 +66,31 @@ fun WorkspaceBuilder.target(name: String, base: String, block: MetaBuilder.() -> .seal() } -fun WorkspaceBuilder.task(task: Task) { - this.tasks.add(task) +fun WorkspaceBuilder.task( + name: String, + type: KClass, + builder: TaskBuilder.() -> Unit +): Task { + return TaskBuilder(name, type).apply(builder).build().also { tasks.add(it) } } +inline fun WorkspaceBuilder.task( + name: String, + noinline builder: TaskBuilder.() -> Unit +): Task = task(name, T::class, builder) + +@JvmName("rawTask") +fun WorkspaceBuilder.task( + name: String, + builder: TaskBuilder.() -> Unit +): Task = task(name, Any::class, builder) /** * A builder for a simple workspace */ class SimpleWorkspaceBuilder(override val parentContext: Context) : WorkspaceBuilder { override var context: Context = parentContext - override var data = DataTreeBuilder(Any::class) + override var data: DataTreeBuilder = DataTreeBuilder(Any::class) override var tasks: MutableSet> = HashSet() override var targets: MutableMap = HashMap() diff --git a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/taskTemplates.kt b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/taskTemplates.kt index ab282015..a2602da3 100644 --- a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/taskTemplates.kt +++ b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/taskTemplates.kt @@ -1,5 +1,5 @@ package hep.dataforge.workspace //fun TaskBuilder.zip( -// val firstNo -//) = rawTransform { } \ No newline at end of file +//// val firstNo +////) = rawTransform { } \ No newline at end of file diff --git a/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/SimpleWorkspaceTest.kt b/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/SimpleWorkspaceTest.kt index 95233251..2adc27b0 100644 --- a/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/SimpleWorkspaceTest.kt +++ b/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/SimpleWorkspaceTest.kt @@ -5,7 +5,7 @@ import hep.dataforge.data.* import hep.dataforge.meta.boolean import hep.dataforge.meta.get import hep.dataforge.names.asName -import kotlin.test.Test +import org.junit.Test import kotlin.test.assertEquals import kotlin.test.assertTrue @@ -14,8 +14,8 @@ class SimpleWorkspaceTest { val testPlugin = object : WorkspacePlugin() { override val tag: PluginTag = PluginTag("test") - val contextTask = Workspace.task("test") { - pipe { + val contextTask = Workspace.task("test", Any::class) { + pipe { context.logger.info { "Test: $it" } } } @@ -28,8 +28,10 @@ class SimpleWorkspaceTest { plugin(testPlugin) } - repeat(100) { - static("myData[$it]", it) + data { + repeat(100) { + static("myData[$it]", it) + } } @@ -37,7 +39,7 @@ class SimpleWorkspaceTest { model { allData() } - pipe { data -> + pipe { data -> if (meta["testFlag"].boolean == true) { println("flag") } @@ -50,7 +52,7 @@ class SimpleWorkspaceTest { model { allData() } - pipe { data -> + pipe { data -> context.logger.info { "Starting linear on $data" } data * 2 + 1 } @@ -62,16 +64,16 @@ class SimpleWorkspaceTest { dependsOn("linear", placement = "linear".asName()) } transform { data -> - val squareNode = data["square"].withType().node!! - val linearNode = data["linear"].withType().node!! - return@transform DataNode.build(Int::class) { + val squareNode = data["square"].filterIsInstance().node!! + val linearNode = data["linear"].filterIsInstance().node!! + return@transform DataNode(Int::class) { squareNode.dataSequence().forEach { (name, _) -> - val newData = Data{ + val newData = Data { val squareValue = squareNode[name].data!!.get() val linearValue = linearNode[name].data!!.get() - squareValue+linearValue + squareValue + linearValue } - set(name,newData) + set(name, newData) } } } @@ -81,17 +83,17 @@ class SimpleWorkspaceTest { model { dependsOn("square") } - join { data -> + join { data -> context.logger.info { "Starting sum" } data.values.sum() } } - task("average") { + task("average") { model { allData() } - joinByGroup { env -> + joinByGroup { env -> group("even", filter = { name, _ -> name.toString().toInt() % 2 == 0 }) { result { data -> env.context.logger.info { "Starting even" } @@ -111,7 +113,7 @@ class SimpleWorkspaceTest { model { dependsOn("average") } - join { data -> + join { data -> data["even"]!! - data["odd"]!! } } @@ -140,7 +142,7 @@ class SimpleWorkspaceTest { } @Test - fun testFullSquare(){ + fun testFullSquare() { val node = workspace.run("fullsquare") println(node.toMeta()) } From 6c1c49d15e17106f385df3ef5eb04e4e581371fc Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Sat, 14 Sep 2019 16:50:25 +0300 Subject: [PATCH 15/48] Added strong typing to tasks and task dependencies --- .../hep/dataforge/workspace/Dependency.kt | 6 +- .../hep/dataforge/workspace/GenericTask.kt | 2 +- .../dataforge/workspace/SimpleWorkspace.kt | 4 +- .../hep/dataforge/workspace/TaskBuilder.kt | 52 +++++--- .../hep/dataforge/workspace/TaskModel.kt | 112 +++++++++++------- .../hep/dataforge/workspace/Workspace.kt | 7 ++ .../dataforge/workspace/WorkspaceBuilder.kt | 5 +- .../hep/dataforge/workspace/taskTemplates.kt | 5 - .../workspace/SimpleWorkspaceTest.kt | 36 ++---- 9 files changed, 135 insertions(+), 94 deletions(-) delete mode 100644 dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/taskTemplates.kt diff --git a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/Dependency.kt b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/Dependency.kt index 1f184095..33b7c8d2 100644 --- a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/Dependency.kt +++ b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/Dependency.kt @@ -87,7 +87,11 @@ class DirectTaskDependency( } } -class WorkspaceTaskDependency(override val name: Name, meta: Meta, placement: Name) : TaskDependency(meta, placement) { +class WorkspaceTaskDependency( + override val name: Name, + meta: Meta, + placement: Name +) : TaskDependency(meta, placement) { override fun resolveTask(workspace: Workspace): Task<*> = workspace.tasks[name] ?: error("Task with name $name is not found in the workspace") } \ No newline at end of file diff --git a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/GenericTask.kt b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/GenericTask.kt index 408f1745..f59c3d24 100644 --- a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/GenericTask.kt +++ b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/GenericTask.kt @@ -55,7 +55,7 @@ class GenericTask( override fun build(workspace: Workspace, taskConfig: Meta): TaskModel { val taskMeta = taskConfig[name]?.node ?: taskConfig val builder = TaskModelBuilder(name, taskMeta) - modelTransform.invoke(builder, taskMeta) + builder.modelTransform(taskMeta) return builder.build() } //TODO add validation diff --git a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/SimpleWorkspace.kt b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/SimpleWorkspace.kt index cb621963..c94fd8ca 100644 --- a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/SimpleWorkspace.kt +++ b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/SimpleWorkspace.kt @@ -1,7 +1,6 @@ package hep.dataforge.workspace import hep.dataforge.context.Context -import hep.dataforge.context.Global import hep.dataforge.context.content import hep.dataforge.context.toMap import hep.dataforge.data.DataNode @@ -24,7 +23,6 @@ class SimpleWorkspace( } companion object { - fun build(parent: Context = Global, block: SimpleWorkspaceBuilder.() -> Unit): SimpleWorkspace = - SimpleWorkspaceBuilder(parent).apply(block).build() + } } \ No newline at end of file diff --git a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskBuilder.kt b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskBuilder.kt index bdacb711..9c0f6958 100644 --- a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskBuilder.kt +++ b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskBuilder.kt @@ -8,13 +8,15 @@ import hep.dataforge.meta.get import hep.dataforge.meta.string import hep.dataforge.names.EmptyName import hep.dataforge.names.Name -import hep.dataforge.names.asName +import hep.dataforge.names.isEmpty import hep.dataforge.names.toName +import kotlin.jvm.JvmName import kotlin.reflect.KClass @TaskBuildScope -class TaskBuilder(val name: String, val type: KClass) { +class TaskBuilder(val name: Name, val type: KClass) { private var modelTransform: TaskModelBuilder.(Meta) -> Unit = { allData() } +// private val additionalDependencies = HashSet() var descriptor: NodeDescriptor? = null private val dataTransforms: MutableList = ArrayList() @@ -30,12 +32,16 @@ class TaskBuilder(val name: String, val type: KClass) { val localData = if (from.isEmpty()) { node } else { - node[from.toName()].node ?: return null + node[from].node ?: return null } return transform(workspace.context, model, localData) } } +// override fun add(dependency: Dependency) { +// additionalDependencies.add(dependency) +// } + fun model(modelTransform: TaskModelBuilder.(Meta) -> Unit) { this.modelTransform = modelTransform } @@ -43,13 +49,14 @@ class TaskBuilder(val name: String, val type: KClass) { /** * Add a transformation on untyped data */ - fun rawTransform( + @JvmName("rawTransform") + fun transform( from: String = "", to: String = "", block: TaskEnv.(DataNode<*>) -> DataNode ) { dataTransforms += DataTransformation(from, to) { context, model, data -> - val env = TaskEnv(EmptyName, model.meta, context) + val env = TaskEnv(EmptyName, model.meta, context, data) env.block(data) } } @@ -62,7 +69,7 @@ class TaskBuilder(val name: String, val type: KClass) { ) { dataTransforms += DataTransformation(from, to) { context, model, data -> data.ensureType(inputType) - val env = TaskEnv(EmptyName, model.meta, context) + val env = TaskEnv(EmptyName, model.meta, context, data) env.block(data.cast(inputType)) } } @@ -88,7 +95,14 @@ class TaskBuilder(val name: String, val type: KClass) { } } - class TaskEnv(val name: Name, val meta: Meta, val context: Context) + class TaskEnv(val name: Name, val meta: Meta, val context: Context, val data: DataNode) { + operator fun DirectTaskDependency.invoke(): DataNode = if(placement.isEmpty()){ + data.cast(task.type) + } else { + data[placement].node?.cast(task.type) + ?: error("Could not find results of direct task dependency $this at \"$placement\"") + } + } /** * A customized pipe action with ability to change meta and name @@ -121,7 +135,7 @@ class TaskBuilder(val name: String, val type: KClass) { ) { //TODO automatically append task meta result = { data -> - TaskEnv(name, meta, context).block(data) + block(data) } } } @@ -133,7 +147,7 @@ class TaskBuilder(val name: String, val type: KClass) { inline fun joinByGroup( from: String = "", to: String = "", - crossinline block: JoinGroupBuilder.(TaskEnv) -> Unit + crossinline block: JoinGroupBuilder.(TaskEnv) -> Unit //TODO needs KEEP-176 ) { action(from, to) { JoinAction( @@ -159,7 +173,7 @@ class TaskBuilder(val name: String, val type: KClass) { result( actionMeta[TaskModel.MODEL_TARGET_KEY]?.string ?: "@anonymous" ) { data -> - TaskEnv(name, meta, context).block(data) + block(data) } } ) @@ -172,7 +186,7 @@ class TaskBuilder(val name: String, val type: KClass) { inline fun split( from: String = "", to: String = "", - crossinline block: SplitBuilder.(TaskEnv) -> Unit + crossinline block: SplitBuilder.(TaskEnv) -> Unit //TODO needs KEEP-176 ) { action(from, to) { SplitAction( @@ -189,9 +203,14 @@ class TaskBuilder(val name: String, val type: KClass) { this.descriptor = NodeDescriptor.build(transform) } - internal fun build(): GenericTask = - GenericTask( - name.asName(), + internal fun build(): GenericTask { +// val actualTransform: TaskModelBuilder.(Meta) -> Unit = { +// modelTransform +// dependencies.addAll(additionalDependencies) +// } + + return GenericTask( + name, type, descriptor ?: NodeDescriptor.empty(), modelTransform @@ -220,15 +239,14 @@ class TaskBuilder(val name: String, val type: KClass) { } } } + } } fun Workspace.Companion.task( name: String, type: KClass, builder: TaskBuilder.() -> Unit -): GenericTask { - return TaskBuilder(name, type).apply(builder).build() -} +): GenericTask = TaskBuilder(name.toName(), type).apply(builder).build() //TODO add delegates to build gradle-like tasks \ No newline at end of file diff --git a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskModel.kt b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskModel.kt index f9562438..7f9cd976 100644 --- a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskModel.kt +++ b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskModel.kt @@ -63,58 +63,88 @@ fun TaskModel.buildInput(workspace: Workspace): DataTree { @DslMarker annotation class TaskBuildScope +interface TaskDependencyContainer { + val defaultMeta: Meta + fun add(dependency: Dependency) +} + +/** + * Add dependency for a task defined in a workspace and resolved by + */ +fun TaskDependencyContainer.dependsOn( + name: Name, + placement: Name = EmptyName, + meta: Meta = defaultMeta +): WorkspaceTaskDependency = + WorkspaceTaskDependency(name, meta, placement).also { add(it) } + +fun TaskDependencyContainer.dependsOn( + name: String, + placement: Name = EmptyName, + meta: Meta = defaultMeta +): WorkspaceTaskDependency = + dependsOn(name.toName(), placement, meta) + +fun TaskDependencyContainer.dependsOn( + task: Task, + placement: Name = EmptyName, + meta: Meta = defaultMeta +): DirectTaskDependency = + DirectTaskDependency(task, meta, placement).also { add(it) } + +fun TaskDependencyContainer.dependsOn( + task: Task, + placement: String, + meta: Meta = defaultMeta +): DirectTaskDependency = + DirectTaskDependency(task, meta, placement.toName()).also { add(it) } + +fun TaskDependencyContainer.dependsOn( + task: Task, + placement: Name = EmptyName, + metaBuilder: MetaBuilder.() -> Unit +): DirectTaskDependency = + dependsOn(task, placement, buildMeta(metaBuilder)) + +/** + * Add custom data dependency + */ +fun TaskDependencyContainer.data(action: DataFilter.() -> Unit): DataDependency = + DataDependency(DataFilter.build(action)).also { add(it) } + +/** + * User-friendly way to add data dependency + */ +fun TaskDependencyContainer.data(pattern: String? = null, from: String? = null, to: String? = null): DataDependency = + data { + pattern?.let { this.pattern = it } + from?.let { this.from = it } + to?.let { this.to = it } + } + +/** + * Add all data as root node + */ +fun TaskDependencyContainer.allData(to: Name = EmptyName) = AllDataDependency(to).also { add(it) } + /** * A builder for [TaskModel] */ -@TaskBuildScope -class TaskModelBuilder(val name: Name, meta: Meta = EmptyMeta) { +class TaskModelBuilder(val name: Name, meta: Meta = EmptyMeta) : TaskDependencyContainer { /** * Meta for current task. By default uses the whole input meta */ var meta: MetaBuilder = meta.builder() val dependencies = HashSet() + override val defaultMeta: Meta get() = meta + + override fun add(dependency: Dependency) { + dependencies.add(dependency) + } + var target: String by this.meta.string(key = MODEL_TARGET_KEY, default = "") - /** - * Add dependency for a task defined in a workspace and resolved by - */ - fun dependsOn(name: Name, meta: Meta = this.meta, placement: Name = EmptyName) { - dependencies.add(WorkspaceTaskDependency(name, meta, placement)) - } - - fun dependsOn(name: String, meta: Meta = this.meta, placement: Name = EmptyName) = - dependsOn(name.toName(), meta, placement) - - fun dependsOn(task: Task<*>, meta: Meta = this.meta, placement: Name = EmptyName) { - dependencies.add(DirectTaskDependency(task, meta, placement)) - } - - fun dependsOn(task: Task<*>, placement: Name = EmptyName, metaBuilder: MetaBuilder.() -> Unit) = - dependsOn(task.name, buildMeta(metaBuilder), placement) - - /** - * Add custom data dependency - */ - fun data(action: DataFilter.() -> Unit) { - dependencies.add(DataDependency(DataFilter.build(action))) - } - - /** - * User-friendly way to add data dependency - */ - fun data(pattern: String? = null, from: String? = null, to: String? = null) = data { - pattern?.let { this.pattern = it } - from?.let { this.from = it } - to?.let { this.to = it } - } - - /** - * Add all data as root node - */ - fun allData(to: Name = EmptyName) { - dependencies.add(AllDataDependency(to)) - } fun build(): TaskModel = TaskModel(name, meta.seal(), dependencies) } diff --git a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/Workspace.kt b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/Workspace.kt index f5f0f3a6..31da6c56 100644 --- a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/Workspace.kt +++ b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/Workspace.kt @@ -1,6 +1,8 @@ package hep.dataforge.workspace +import hep.dataforge.context.Context import hep.dataforge.context.ContextAware +import hep.dataforge.context.Global import hep.dataforge.data.Data import hep.dataforge.data.DataNode import hep.dataforge.data.dataSequence @@ -56,6 +58,8 @@ interface Workspace : ContextAware, Provider { companion object { const val TYPE = "workspace" + operator fun invoke(parent: Context = Global, block: SimpleWorkspaceBuilder.() -> Unit): SimpleWorkspace = + SimpleWorkspaceBuilder(parent).apply(block).build() } } @@ -73,3 +77,6 @@ fun Workspace.run(task: String, meta: Meta) = fun Workspace.run(task: String, block: MetaBuilder.() -> Unit = {}) = run(task, buildMeta(block)) + +fun Workspace.run(task: Task, metaBuilder: MetaBuilder.() -> Unit = {}): DataNode = + run(task, buildMeta(metaBuilder)) \ No newline at end of file diff --git a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/WorkspaceBuilder.kt b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/WorkspaceBuilder.kt index 6bc4900a..75a660cd 100644 --- a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/WorkspaceBuilder.kt +++ b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/WorkspaceBuilder.kt @@ -8,6 +8,7 @@ import hep.dataforge.meta.* import hep.dataforge.names.EmptyName import hep.dataforge.names.Name import hep.dataforge.names.isEmpty +import hep.dataforge.names.toName import kotlin.jvm.JvmName import kotlin.reflect.KClass @@ -70,9 +71,7 @@ fun WorkspaceBuilder.task( name: String, type: KClass, builder: TaskBuilder.() -> Unit -): Task { - return TaskBuilder(name, type).apply(builder).build().also { tasks.add(it) } -} +): Task = TaskBuilder(name.toName(), type).apply(builder).build().also { tasks.add(it) } inline fun WorkspaceBuilder.task( name: String, diff --git a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/taskTemplates.kt b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/taskTemplates.kt deleted file mode 100644 index a2602da3..00000000 --- a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/taskTemplates.kt +++ /dev/null @@ -1,5 +0,0 @@ -package hep.dataforge.workspace - -//fun TaskBuilder.zip( -//// val firstNo -////) = rawTransform { } \ No newline at end of file diff --git a/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/SimpleWorkspaceTest.kt b/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/SimpleWorkspaceTest.kt index 2adc27b0..dd72a905 100644 --- a/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/SimpleWorkspaceTest.kt +++ b/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/SimpleWorkspaceTest.kt @@ -4,7 +4,6 @@ import hep.dataforge.context.PluginTag import hep.dataforge.data.* import hep.dataforge.meta.boolean import hep.dataforge.meta.get -import hep.dataforge.names.asName import org.junit.Test import kotlin.test.assertEquals import kotlin.test.assertTrue @@ -22,7 +21,7 @@ class SimpleWorkspaceTest { override val tasks: Collection> = listOf(contextTask) } - val workspace = SimpleWorkspace.build { + val workspace = Workspace { context { plugin(testPlugin) @@ -35,10 +34,7 @@ class SimpleWorkspaceTest { } - val square = task("square") { - model { - allData() - } + val square = task("square") { pipe { data -> if (meta["testFlag"].boolean == true) { println("flag") @@ -48,24 +44,21 @@ class SimpleWorkspaceTest { } } - val linear = task("linear") { - model { - allData() - } + val linear = task("linear") { pipe { data -> context.logger.info { "Starting linear on $data" } data * 2 + 1 } } - val fullSquare = task("fullsquare") { + val fullSquare = task("fullsquare") { model { - dependsOn("square", placement = "square".asName()) - dependsOn("linear", placement = "linear".asName()) + val squareDep = dependsOn(square, placement = "square") + val linearDep = dependsOn(linear, placement = "linear") } - transform { data -> - val squareNode = data["square"].filterIsInstance().node!! - val linearNode = data["linear"].filterIsInstance().node!! + transform { data -> + val squareNode = data["square"].node!!.cast()//squareDep() + val linearNode = data["linear"].node!!.cast()//linearDep() return@transform DataNode(Int::class) { squareNode.dataSequence().forEach { (name, _) -> val newData = Data { @@ -79,9 +72,9 @@ class SimpleWorkspaceTest { } } - task("sum") { + task("sum") { model { - dependsOn("square") + dependsOn(square) } join { data -> context.logger.info { "Starting sum" } @@ -89,10 +82,7 @@ class SimpleWorkspaceTest { } } - task("average") { - model { - allData() - } + val average = task("average") { joinByGroup { env -> group("even", filter = { name, _ -> name.toString().toInt() % 2 == 0 }) { result { data -> @@ -111,7 +101,7 @@ class SimpleWorkspaceTest { task("delta") { model { - dependsOn("average") + dependsOn(average) } join { data -> data["even"]!! - data["odd"]!! From 71536051aae9b4991abfcd39ecd33cb0a4174473 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Wed, 18 Sep 2019 16:36:35 +0300 Subject: [PATCH 16/48] Data and DataNode file io update --- build.gradle.kts | 6 +- .../kotlin/hep/dataforge/data/DataNode.kt | 2 +- dataforge-io/build.gradle.kts | 4 +- .../kotlin/hep/dataforge/io/Envelope.kt | 3 +- .../io/functions/RemoteFunctionClient.kt | 4 +- .../io/functions/RemoteFunctionServer.kt | 2 +- .../hep/dataforge/io/EnvelopeFormatTest.kt | 2 +- .../kotlin/hep/dataforge/io/FileBinary.kt | 4 +- .../kotlin/hep/dataforge/io/FileEnvelope.kt | 2 +- .../kotlin/hep/dataforge/io/ioFormatsJVM.kt | 16 +++ .../hep/dataforge/io/tcp/EnvelopeClient.kt | 2 +- .../hep/dataforge/io/FileEnvelopeTest.kt | 30 ++++++ .../dataforge/io/tcp/EnvelopeServerTest.kt | 2 +- .../hep/dataforge/workspace/fileData.kt | 99 ++++++++++--------- 14 files changed, 117 insertions(+), 61 deletions(-) create mode 100644 dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/FileEnvelopeTest.kt diff --git a/build.gradle.kts b/build.gradle.kts index 41cf03f2..b0adec0e 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -1,9 +1,9 @@ plugins { - id("scientifik.mpp") version "0.1.7-dev" apply false - id("scientifik.publish") version "0.1.7-dev" apply false + id("scientifik.mpp") version "0.1.7" apply false + id("scientifik.publish") version "0.1.7" apply false } -val dataforgeVersion by extra("0.1.4-dev-2") +val dataforgeVersion by extra("0.1.4-dev-3") val bintrayRepo by extra("dataforge") val githubProject by extra("dataforge-core") diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataNode.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataNode.kt index d7158dab..5e220758 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataNode.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataNode.kt @@ -140,7 +140,7 @@ private sealed class DataTreeBuilderItem { /** * A builder for a DataTree. */ -class DataTreeBuilder(private val type: KClass) { +class DataTreeBuilder(val type: KClass) { private val map = HashMap>() operator fun set(token: NameToken, node: DataTreeBuilder) { diff --git a/dataforge-io/build.gradle.kts b/dataforge-io/build.gradle.kts index 8a997017..083e9d53 100644 --- a/dataforge-io/build.gradle.kts +++ b/dataforge-io/build.gradle.kts @@ -5,8 +5,8 @@ plugins { description = "IO module" scientifik{ - serialization = true - io = true + withSerialization() + withIO() } diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Envelope.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Envelope.kt index 6c3b45b9..80e07b56 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Envelope.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Envelope.kt @@ -26,7 +26,7 @@ interface Envelope { /** * Build a static envelope using provided builder */ - fun build(block: EnvelopeBuilder.() -> Unit) = EnvelopeBuilder().apply(block).build() + operator fun invoke(block: EnvelopeBuilder.() -> Unit) = EnvelopeBuilder().apply(block).build() } } @@ -97,6 +97,7 @@ class EnvelopeBuilder { var type by metaBuilder.string(key = Envelope.ENVELOPE_TYPE_KEY) var dataType by metaBuilder.string(key = Envelope.ENVELOPE_DATA_TYPE_KEY) + var dataID by metaBuilder.string(key = Envelope.ENVELOPE_DATA_ID_KEY) var description by metaBuilder.string(key = Envelope.ENVELOPE_DESCRIPTION_KEY) /** diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/functions/RemoteFunctionClient.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/functions/RemoteFunctionClient.kt index 0ba90282..770e5578 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/functions/RemoteFunctionClient.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/functions/RemoteFunctionClient.kt @@ -12,7 +12,7 @@ class RemoteFunctionClient(override val context: Context, val responder: Respond private fun IOPlugin.encodeOne( meta: Meta, value: T - ): Envelope = Envelope.build { + ): Envelope = Envelope.invoke { meta(meta) type = REQUEST_TYPE data { @@ -26,7 +26,7 @@ class RemoteFunctionClient(override val context: Context, val responder: Respond private fun IOPlugin.encodeMany( meta: Meta, values: List - ): Envelope = Envelope.build { + ): Envelope = Envelope.invoke { meta(meta) type = REQUEST_TYPE meta { diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/functions/RemoteFunctionServer.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/functions/RemoteFunctionServer.kt index 41a2a271..b2fb6cd0 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/functions/RemoteFunctionServer.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/functions/RemoteFunctionServer.kt @@ -40,7 +40,7 @@ class RemoteFunctionServer( input ) - return Envelope.build { + return Envelope.invoke { meta { meta(request.meta) } diff --git a/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/EnvelopeFormatTest.kt b/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/EnvelopeFormatTest.kt index 059c0f19..5c1d82f9 100644 --- a/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/EnvelopeFormatTest.kt +++ b/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/EnvelopeFormatTest.kt @@ -5,7 +5,7 @@ import kotlin.test.assertEquals class EnvelopeFormatTest { - val envelope = Envelope.build { + val envelope = Envelope.invoke { type = "test.format" meta{ "d" to 22.2 diff --git a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileBinary.kt b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileBinary.kt index b69969ed..2c357a47 100644 --- a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileBinary.kt +++ b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileBinary.kt @@ -20,4 +20,6 @@ class FileBinary(val path: Path, private val offset: UInt = 0u, size: ULong? = n return ByteReadPacket(buffer).block() } } -} \ No newline at end of file +} + +fun Path.asBinary(offset: UInt = 0u, size: ULong? = null): FileBinary = FileBinary(this, offset, size) \ No newline at end of file diff --git a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileEnvelope.kt b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileEnvelope.kt index 0c54012c..88ebf285 100644 --- a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileEnvelope.kt +++ b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileEnvelope.kt @@ -22,7 +22,7 @@ class FileEnvelope internal constructor(val path: Path, val format: EnvelopeForm override val data: Binary? = FileBinary(path, partialEnvelope.dataOffset, partialEnvelope.dataSize) } -fun Path.readEnvelope(format: EnvelopeFormat) = FileEnvelope(this, format) +fun Path.readEnvelope(format: EnvelopeFormat = TaggedEnvelopeFormat) = FileEnvelope(this, format) fun Path.writeEnvelope( envelope: Envelope, diff --git a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/ioFormatsJVM.kt b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/ioFormatsJVM.kt index 8ba97fc3..6d075284 100644 --- a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/ioFormatsJVM.kt +++ b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/ioFormatsJVM.kt @@ -1,5 +1,6 @@ package hep.dataforge.io +import hep.dataforge.descriptors.NodeDescriptor import hep.dataforge.io.functions.FunctionServer import hep.dataforge.io.functions.FunctionServer.Companion.FUNCTION_NAME_KEY import hep.dataforge.io.functions.FunctionServer.Companion.INPUT_FORMAT_KEY @@ -7,6 +8,10 @@ import hep.dataforge.io.functions.FunctionServer.Companion.OUTPUT_FORMAT_KEY import hep.dataforge.meta.Meta import hep.dataforge.meta.buildMeta import hep.dataforge.names.Name +import kotlinx.io.nio.asOutput +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.StandardOpenOption import kotlin.reflect.KClass import kotlin.reflect.full.isSuperclassOf @@ -31,4 +36,15 @@ inline fun FunctionServer.function( val plugin = context.plugins.get() ?: error("IO plugin not loaded") val meta = plugin.generateFunctionMeta(functionName) return function(meta) +} + +/** + * Write meta to file in a given [format] + */ +fun Meta.write(path: Path, format: MetaFormat, descriptor: NodeDescriptor? = null) { + format.run { + Files.newByteChannel(path, StandardOpenOption.WRITE, StandardOpenOption.CREATE_NEW) + .asOutput() + .writeMeta(this@write, descriptor) + } } \ No newline at end of file diff --git a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/tcp/EnvelopeClient.kt b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/tcp/EnvelopeClient.kt index 17b0429d..9011ae1e 100644 --- a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/tcp/EnvelopeClient.kt +++ b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/tcp/EnvelopeClient.kt @@ -34,7 +34,7 @@ class EnvelopeClient( suspend fun close() { try { respond( - Envelope.build { + Envelope.invoke { type = EnvelopeServer.SHUTDOWN_ENVELOPE_TYPE } ) diff --git a/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/FileEnvelopeTest.kt b/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/FileEnvelopeTest.kt new file mode 100644 index 00000000..c76b446e --- /dev/null +++ b/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/FileEnvelopeTest.kt @@ -0,0 +1,30 @@ +package hep.dataforge.io + +import java.nio.file.Files +import kotlin.test.Test +import kotlin.test.assertTrue + + +class FileEnvelopeTest { + val envelope = Envelope { + meta { + "a" to "AAA" + "b" to 22.2 + } + dataType = "hep.dataforge.test" + dataID = "myData" // добавил только что + data { + writeDouble(16.7) + + } + } + + @Test + fun testFileWriteRead() { + val tmpPath = Files.createTempFile("dataforge_test", ".df") + tmpPath.writeEnvelope(envelope) + println(tmpPath.toUri()) + val restored: Envelope = tmpPath.readEnvelope() + assertTrue { envelope.contentEquals(restored) } + } +} \ No newline at end of file diff --git a/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/tcp/EnvelopeServerTest.kt b/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/tcp/EnvelopeServerTest.kt index d48b42ef..5691f0a0 100644 --- a/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/tcp/EnvelopeServerTest.kt +++ b/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/tcp/EnvelopeServerTest.kt @@ -46,7 +46,7 @@ class EnvelopeServerTest { @Test fun doEchoTest() { - val request = Envelope.build { + val request = Envelope.invoke { type = "test.echo" meta { "test.value" to 22 diff --git a/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/fileData.kt b/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/fileData.kt index f20e18cf..08959298 100644 --- a/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/fileData.kt +++ b/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/fileData.kt @@ -1,15 +1,16 @@ package hep.dataforge.workspace import hep.dataforge.data.Data +import hep.dataforge.data.DataNode +import hep.dataforge.data.DataTreeBuilder +import hep.dataforge.data.datum import hep.dataforge.descriptors.NodeDescriptor import hep.dataforge.io.* import hep.dataforge.meta.EmptyMeta import hep.dataforge.meta.Meta import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.coroutineScope import kotlinx.coroutines.withContext import kotlinx.io.nio.asInput -import kotlinx.io.nio.asOutput import java.nio.file.Files import java.nio.file.Path import java.nio.file.StandardOpenOption @@ -18,69 +19,75 @@ import kotlin.reflect.KClass /** * Read meta from file in a given [format] */ -suspend fun Path.readMeta(format: MetaFormat, descriptor: NodeDescriptor? = null): Meta { - return withContext(Dispatchers.IO) { - format.run { - Files.newByteChannel(this@readMeta, StandardOpenOption.READ) - .asInput() - .readMeta(descriptor) - } - } -} - -/** - * Write meta to file in a given [format] - */ -suspend fun Meta.write(path: Path, format: MetaFormat, descriptor: NodeDescriptor? = null) { - withContext(Dispatchers.IO) { - format.run { - Files.newByteChannel(path, StandardOpenOption.WRITE, StandardOpenOption.CREATE_NEW) - .asOutput() - .writeMeta(this@write, descriptor) - } +fun Path.readMeta(format: MetaFormat, descriptor: NodeDescriptor? = null): Meta { + return format.run { + Files.newByteChannel(this@readMeta, StandardOpenOption.READ) + .asInput() + .readMeta(descriptor) } } /** * Read data with supported envelope format and binary format. If envelope format is null, then read binary directly from file. + * The operation is blocking since it must read meta header. The reading of envelope body is lazy * @param type explicit type of data read * @param format binary format * @param envelopeFormat the format of envelope. If null, file is read directly * @param metaFile the relative file for optional meta override * @param metaFileFormat the meta format for override */ -suspend fun Path.readData( +fun Path.readData( type: KClass, format: IOFormat, envelopeFormat: EnvelopeFormat? = null, metaFile: Path = resolveSibling("$fileName.meta"), metaFileFormat: MetaFormat = JsonMetaFormat ): Data { - return coroutineScope { - val externalMeta = if (Files.exists(metaFile)) { - metaFile.readMeta(metaFileFormat) - } else { - null - } - if (envelopeFormat == null) { - Data(type, externalMeta ?: EmptyMeta) { - withContext(Dispatchers.IO) { - format.run { - Files.newByteChannel(this@readData, StandardOpenOption.READ) - .asInput() - .readThis() - } + val externalMeta = if (Files.exists(metaFile)) { + metaFile.readMeta(metaFileFormat) + } else { + null + } + return if (envelopeFormat == null) { + Data(type, externalMeta ?: EmptyMeta) { + withContext(Dispatchers.IO) { + format.run { + Files.newByteChannel(this@readData, StandardOpenOption.READ) + .asInput() + .readThis() } } - } else { - withContext(Dispatchers.IO) { - readEnvelope(envelopeFormat).let { - if (externalMeta == null) { - it - } else { - it.withMetaLayers(externalMeta) - } - }.toData(type, format) + } + } else { + readEnvelope(envelopeFormat).let { + if (externalMeta == null) { + it + } else { + it.withMetaLayers(externalMeta) + } + }.toData(type, format) + } +} + +fun DataTreeBuilder.file(path: Path, format: IOFormat, envelopeFormat: EnvelopeFormat? = null) { + val data = path.readData(type, format, envelopeFormat) + val name = path.fileName.toString().replace(".df", "") + datum(name, data) +} + +/** + * Read the directory as a data node + */ +fun Path.readDataNode( + type: KClass, + format: IOFormat, + envelopeFormat: EnvelopeFormat? = null +): DataNode { + if (!Files.isDirectory(this)) error("Provided path $this is not a directory") + return DataNode(type) { + Files.list(this@readDataNode).forEach { path -> + if (!path.fileName.toString().endsWith(".meta")) { + file(path, format, envelopeFormat) } } } From 337401f6e05572344f4141ddaa5b7868bd13f30b Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Tue, 24 Sep 2019 17:35:06 +0300 Subject: [PATCH 17/48] Task builder for WorkspacePlugin --- .../kotlin/hep/dataforge/meta/MetaBuilder.kt | 5 ++++ .../hep/dataforge/workspace/TaskBuilder.kt | 8 ------ .../dataforge/workspace/WorkspacePlugin.kt | 27 +++++++++++++++++-- .../workspace/SimpleWorkspaceTest.kt | 18 +++++++++++-- 4 files changed, 46 insertions(+), 12 deletions(-) diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MetaBuilder.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MetaBuilder.kt index 322b660a..6179f2d9 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MetaBuilder.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MetaBuilder.kt @@ -65,6 +65,11 @@ fun Meta.builder(): MetaBuilder { } } +/** + * Create a deep copy of this meta and apply builder to it + */ +fun Meta.edit(builder: MetaBuilder.() -> Unit): MetaBuilder = builder().apply(builder) + /** * Build a [MetaBuilder] using given transformation */ diff --git a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskBuilder.kt b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskBuilder.kt index 9c0f6958..150d56a6 100644 --- a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskBuilder.kt +++ b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskBuilder.kt @@ -242,11 +242,3 @@ class TaskBuilder(val name: Name, val type: KClass) { } } -fun Workspace.Companion.task( - name: String, - type: KClass, - builder: TaskBuilder.() -> Unit -): GenericTask = TaskBuilder(name.toName(), type).apply(builder).build() - - -//TODO add delegates to build gradle-like tasks \ No newline at end of file diff --git a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/WorkspacePlugin.kt b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/WorkspacePlugin.kt index 1ddd291b..a377c023 100644 --- a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/WorkspacePlugin.kt +++ b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/WorkspacePlugin.kt @@ -3,17 +3,40 @@ package hep.dataforge.workspace import hep.dataforge.context.AbstractPlugin import hep.dataforge.context.toMap import hep.dataforge.names.Name +import hep.dataforge.names.toName +import kotlin.reflect.KClass /** * An abstract plugin with some additional boilerplate to effectively work with workspace context */ abstract class WorkspacePlugin : AbstractPlugin() { - abstract val tasks: Collection> + private val _tasks = HashSet>() + val tasks: Collection> get() = _tasks override fun provideTop(target: String): Map { - return when(target){ + return when (target) { Task.TYPE -> tasks.toMap() else -> emptyMap() } } + + fun task(task: Task<*>){ + _tasks.add(task) + } + + fun task( + name: String, + type: KClass, + builder: TaskBuilder.() -> Unit + ): GenericTask = TaskBuilder(name.toName(), type).apply(builder).build().also { + _tasks.add(it) + } + + inline fun task( + name: String, + noinline builder: TaskBuilder.() -> Unit + ) = task(name, T::class, builder) + +// +////TODO add delegates to build gradle-like tasks } \ No newline at end of file diff --git a/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/SimpleWorkspaceTest.kt b/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/SimpleWorkspaceTest.kt index dd72a905..f45f8720 100644 --- a/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/SimpleWorkspaceTest.kt +++ b/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/SimpleWorkspaceTest.kt @@ -3,7 +3,10 @@ package hep.dataforge.workspace import hep.dataforge.context.PluginTag import hep.dataforge.data.* import hep.dataforge.meta.boolean +import hep.dataforge.meta.builder import hep.dataforge.meta.get +import hep.dataforge.meta.int +import hep.dataforge.names.plus import org.junit.Test import kotlin.test.assertEquals import kotlin.test.assertTrue @@ -13,12 +16,11 @@ class SimpleWorkspaceTest { val testPlugin = object : WorkspacePlugin() { override val tag: PluginTag = PluginTag("test") - val contextTask = Workspace.task("test", Any::class) { + val contextTask = task("test", Any::class) { pipe { context.logger.info { "Test: $it" } } } - override val tasks: Collection> = listOf(contextTask) } val workspace = Workspace { @@ -108,6 +110,18 @@ class SimpleWorkspaceTest { } } + val customPipeTask = task("custom") { + customPipe { + meta = meta.builder().apply { + "newValue" to 22 + } + name += "new" + result { + meta["value"].int ?: 0 + it + } + } + } + target("empty") {} } From 1b118986f53a4d6b39b95ca48ef5522f53192888 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Wed, 2 Oct 2019 13:48:04 +0300 Subject: [PATCH 18/48] Value extensions to access primitives --- .../kotlin/hep/dataforge/meta/Meta.kt | 2 - .../kotlin/hep/dataforge/names/Name.kt | 4 +- .../kotlin/hep/dataforge/values/Value.kt | 38 +++++++------------ .../hep/dataforge/values/valueEstensions.kt | 37 ++++++++++++++++++ 4 files changed, 54 insertions(+), 27 deletions(-) create mode 100644 dataforge-meta/src/commonMain/kotlin/hep/dataforge/values/valueEstensions.kt diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Meta.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Meta.kt index 45a27250..6b255a63 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Meta.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Meta.kt @@ -263,6 +263,4 @@ interface Metoid { val meta: Meta } -fun Value.toMeta() = buildMeta { Meta.VALUE_KEY to this } - fun Meta.isEmpty() = this === EmptyMeta || this.items.isEmpty() diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/names/Name.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/names/Name.kt index 0531a818..41680eb0 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/names/Name.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/names/Name.kt @@ -139,7 +139,7 @@ fun String.toName(): Name { * Convert the [String] to a [Name] by simply wrapping it in a single name token without parsing. * The input string could contain dots and braces, but they are just escaped, not parsed. */ -fun String.asName(): Name = if(isBlank()) EmptyName else NameToken(this).asName() +fun String.asName(): Name = if (isBlank()) EmptyName else NameToken(this).asName() operator fun NameToken.plus(other: Name): Name = Name(listOf(this) + other.tokens) @@ -147,6 +147,8 @@ operator fun Name.plus(other: Name): Name = Name(this.tokens + other.tokens) operator fun Name.plus(other: String): Name = this + other.toName() +operator fun Name.plus(other: NameToken): Name = Name(tokens + other) + fun Name.appendLeft(other: String): Name = NameToken(other) + this fun NameToken.asName() = Name(listOf(this)) diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/values/Value.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/values/Value.kt index cd68e040..2fce145f 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/values/Value.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/values/Value.kt @@ -45,6 +45,8 @@ interface Value { override fun equals(other: Any?): Boolean + override fun hashCode(): Int + companion object { const val TYPE = "value" @@ -86,14 +88,9 @@ object Null : Value { override fun toString(): String = value.toString() override fun equals(other: Any?): Boolean = other === Null + override fun hashCode(): Int = 0 } -/** - * Check if value is null - */ -fun Value.isNull(): Boolean = this == Null - - /** * Singleton true value */ @@ -106,7 +103,7 @@ object True : Value { override fun toString(): String = value.toString() override fun equals(other: Any?): Boolean = other === True - + override fun hashCode(): Int = 1 } /** @@ -121,10 +118,9 @@ object False : Value { override fun toString(): String = True.value.toString() override fun equals(other: Any?): Boolean = other === False + override fun hashCode(): Int = -1 } -val Value.boolean get() = this == True || this.list.firstOrNull() == True || (type == ValueType.STRING && string.toBoolean()) - class NumberValue(override val number: Number) : Value { override val value: Any? get() = number override val type: ValueType get() = ValueType.NUMBER @@ -183,18 +179,18 @@ class ListValue(override val list: List) : Value { } } - override val value: Any? get() = list + override val value: List get() = list override val type: ValueType get() = list.first().type override val number: Number get() = list.first().number override val string: String get() = list.first().string - override fun toString(): String = list.joinToString (prefix = "[", postfix = "]") + override fun toString(): String = list.joinToString(prefix = "[", postfix = "]") override fun equals(other: Any?): Boolean { if (this === other) return true if (other !is Value) return false - if( other is DoubleArrayValue){ - + if (other is DoubleArrayValue) { + return DoubleArray(list.size) { list[it].number.toDouble() }.contentEquals(other.value) } return list == other.list } @@ -206,12 +202,6 @@ class ListValue(override val list: List) : Value { } -/** - * Check if value is list - */ -fun Value.isList(): Boolean = this.list.size > 1 - - fun Number.asValue(): Value = NumberValue(this) fun Boolean.asValue(): Value = if (this) True else False @@ -220,15 +210,15 @@ fun String.asValue(): Value = StringValue(this) fun Iterable.asValue(): Value = ListValue(this.toList()) -fun IntArray.asValue(): Value = ListValue(map{NumberValue(it)}) +fun IntArray.asValue(): Value = ListValue(map { NumberValue(it) }) -fun LongArray.asValue(): Value = ListValue(map{NumberValue(it)}) +fun LongArray.asValue(): Value = ListValue(map { NumberValue(it) }) -fun ShortArray.asValue(): Value = ListValue(map{NumberValue(it)}) +fun ShortArray.asValue(): Value = ListValue(map { NumberValue(it) }) -fun FloatArray.asValue(): Value = ListValue(map{NumberValue(it)}) +fun FloatArray.asValue(): Value = ListValue(map { NumberValue(it) }) -fun ByteArray.asValue(): Value = ListValue(map{NumberValue(it)}) +fun ByteArray.asValue(): Value = ListValue(map { NumberValue(it) }) /** diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/values/valueEstensions.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/values/valueEstensions.kt new file mode 100644 index 00000000..41a5bda3 --- /dev/null +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/values/valueEstensions.kt @@ -0,0 +1,37 @@ +package hep.dataforge.values + +import hep.dataforge.meta.Meta +import hep.dataforge.meta.buildMeta + +/** + * Check if value is null + */ +fun Value.isNull(): Boolean = this == Null + +/** + * Check if value is list + */ +fun Value.isList(): Boolean = this.list.size > 1 + +val Value.boolean + get() = this == True + || this.list.firstOrNull() == True + || (type == ValueType.STRING && string.toBoolean()) + + +val Value.int get() = number.toInt() +val Value.double get() = number.toDouble() +val Value.float get() = number.toFloat() +val Value.long get() = number.toLong() + +val Value.stringList: List get() = list.map { it.string } + +val Value.doubleArray: DoubleArray + get() = if (this is DoubleArrayValue) { + value + } else { + DoubleArray(list.size) { list[it].double } + } + + +fun Value.toMeta() = buildMeta { Meta.VALUE_KEY to this } \ No newline at end of file From e68e184fb2ec288727ae271166df67fdb093b5bb Mon Sep 17 00:00:00 2001 From: Mikhail Zelenyy Date: Fri, 4 Oct 2019 20:16:34 +0300 Subject: [PATCH 19/48] Two test for checking Meta.enum extension. Test `testEnum` failed. --- .../hep/dataforge/meta/MetaExtensionTest.kt | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 dataforge-meta/src/jvmTest/kotlin/hep/dataforge/meta/MetaExtensionTest.kt diff --git a/dataforge-meta/src/jvmTest/kotlin/hep/dataforge/meta/MetaExtensionTest.kt b/dataforge-meta/src/jvmTest/kotlin/hep/dataforge/meta/MetaExtensionTest.kt new file mode 100644 index 00000000..5f82df9a --- /dev/null +++ b/dataforge-meta/src/jvmTest/kotlin/hep/dataforge/meta/MetaExtensionTest.kt @@ -0,0 +1,22 @@ +package hep.dataforge.meta + +import org.junit.Test + +class MetaExtensionTest { + + enum class TestEnum{ + test + } + + @Test + fun testEnum(){ + val meta = buildMeta{"enum" to TestEnum.test} + meta["enum"].enum() + } + @Test + fun testEnumByString(){ + val meta = buildMeta{"enum" to TestEnum.test.toString()} + println(meta["enum"].enum()) + } + +} \ No newline at end of file From e3e9412cb6482107c91310971f91306d064de23a Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Fri, 4 Oct 2019 20:23:02 +0300 Subject: [PATCH 20/48] Fixed build and nasty bug in data builder --- build.gradle.kts | 4 ++-- .../commonMain/kotlin/hep/dataforge/data/DataNode.kt | 3 ++- .../kotlin/hep/dataforge/data/DataTreeBuilderTest.kt | 12 ++++++------ .../kotlin/hep/dataforge/io/JsonMetaFormat.kt | 5 +++++ .../kotlin/hep/dataforge/io/MetaFormatTest.kt | 2 ++ .../kotlin/hep/dataforge/io/FileEnvelopeTest.kt | 2 ++ .../hep/dataforge/io/tcp/EnvelopeServerTest.kt | 2 ++ .../commonMain/kotlin/hep/dataforge/meta/Builder.kt | 7 +++++++ .../kotlin/hep/dataforge/meta/MetaBuilder.kt | 1 + .../kotlin/hep/dataforge/values/exoticValues.kt | 2 ++ .../kotlin/hep/dataforge/meta/DynamicMetaTest.kt | 4 ++-- 11 files changed, 33 insertions(+), 11 deletions(-) create mode 100644 dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Builder.kt diff --git a/build.gradle.kts b/build.gradle.kts index b0adec0e..8dec72d0 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -1,6 +1,6 @@ plugins { - id("scientifik.mpp") version "0.1.7" apply false - id("scientifik.publish") version "0.1.7" apply false + id("scientifik.mpp") version "0.2.0" apply false + id("scientifik.publish") version "0.2.0" apply false } val dataforgeVersion by extra("0.1.4-dev-3") diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataNode.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataNode.kt index 5e220758..53ba081d 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataNode.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataNode.kt @@ -140,6 +140,7 @@ private sealed class DataTreeBuilderItem { /** * A builder for a DataTree. */ +@DFBuilder class DataTreeBuilder(val type: KClass) { private val map = HashMap>() @@ -207,7 +208,7 @@ class DataTreeBuilder(val type: KClass) { /** * Build and append node */ - infix fun String.to(block: DataTreeBuilder.() -> Unit) = set(toName(), DataTreeBuilder(type).apply(block)) + infix fun String.to(block: DataTreeBuilder.() -> Unit) = set(toName(), DataTreeBuilder(type).apply(block)) fun update(node: DataNode) { diff --git a/dataforge-data/src/commonTest/kotlin/hep/dataforge/data/DataTreeBuilderTest.kt b/dataforge-data/src/commonTest/kotlin/hep/dataforge/data/DataTreeBuilderTest.kt index a3bb509b..c27974bd 100644 --- a/dataforge-data/src/commonTest/kotlin/hep/dataforge/data/DataTreeBuilderTest.kt +++ b/dataforge-data/src/commonTest/kotlin/hep/dataforge/data/DataTreeBuilderTest.kt @@ -7,19 +7,19 @@ import kotlin.test.assertTrue internal class DataTreeBuilderTest{ @Test fun testDataUpdate(){ - val updateData = DataNode.build(Any::class){ + val updateData = DataNode{ "update" to { "a" to Data.static("a") "b" to Data.static("b") } } - val node = DataNode.build(Any::class){ - "primary" to { - "a" to Data.static("a") - "b" to Data.static("b") + val node = DataNode{ + node("primary"){ + static("a","a") + static("b","b") } - "root" to Data.static("root") + static("root","root") update(updateData) } diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/JsonMetaFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/JsonMetaFormat.kt index 41946768..e9ec443c 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/JsonMetaFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/JsonMetaFormat.kt @@ -1,3 +1,5 @@ +@file:Suppress("UNUSED_PARAMETER") + package hep.dataforge.io import hep.dataforge.descriptors.ItemDescriptor @@ -43,6 +45,9 @@ object JsonMetaFormat : MetaFormat { } } +/** + * @param descriptor reserved for custom serialization in future + */ fun Value.toJson(descriptor: ValueDescriptor? = null): JsonElement { return if (isList()) { JsonArray(list.map { it.toJson() }) diff --git a/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/MetaFormatTest.kt b/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/MetaFormatTest.kt index 16d946e3..3c7c29cb 100644 --- a/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/MetaFormatTest.kt +++ b/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/MetaFormatTest.kt @@ -4,11 +4,13 @@ import hep.dataforge.meta.* import kotlinx.serialization.json.JsonPrimitive import kotlinx.serialization.json.json import kotlinx.serialization.json.jsonArray +import kotlin.test.Ignore import kotlin.test.Test import kotlin.test.assertEquals class MetaFormatTest { @Test + @Ignore fun testBinaryMetaFormat() { val meta = buildMeta { "a" to 22 diff --git a/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/FileEnvelopeTest.kt b/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/FileEnvelopeTest.kt index c76b446e..acd427d4 100644 --- a/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/FileEnvelopeTest.kt +++ b/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/FileEnvelopeTest.kt @@ -1,6 +1,7 @@ package hep.dataforge.io import java.nio.file.Files +import kotlin.test.Ignore import kotlin.test.Test import kotlin.test.assertTrue @@ -20,6 +21,7 @@ class FileEnvelopeTest { } @Test + @Ignore fun testFileWriteRead() { val tmpPath = Files.createTempFile("dataforge_test", ".df") tmpPath.writeEnvelope(envelope) diff --git a/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/tcp/EnvelopeServerTest.kt b/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/tcp/EnvelopeServerTest.kt index 5691f0a0..38ff434d 100644 --- a/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/tcp/EnvelopeServerTest.kt +++ b/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/tcp/EnvelopeServerTest.kt @@ -10,6 +10,7 @@ import kotlinx.coroutines.runBlocking import org.junit.AfterClass import org.junit.BeforeClass import org.junit.Test +import kotlin.test.Ignore import kotlin.test.assertEquals import kotlin.time.ExperimentalTime @@ -44,6 +45,7 @@ class EnvelopeServerTest { } @Test + @Ignore fun doEchoTest() { val request = Envelope.invoke { diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Builder.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Builder.kt new file mode 100644 index 00000000..c9878c5f --- /dev/null +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Builder.kt @@ -0,0 +1,7 @@ +package hep.dataforge.meta + +/** + * General marker for dataforge builders + */ +@DslMarker +annotation class DFBuilder \ No newline at end of file diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MetaBuilder.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MetaBuilder.kt index 6179f2d9..3792491d 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MetaBuilder.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MetaBuilder.kt @@ -7,6 +7,7 @@ import hep.dataforge.values.Value /** * DSL builder for meta. Is not intended to store mutable state */ +@DFBuilder class MetaBuilder : AbstractMutableMeta() { override fun wrapNode(meta: Meta): MetaBuilder = if (meta is MetaBuilder) meta else meta.builder() override fun empty(): MetaBuilder = MetaBuilder() diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/values/exoticValues.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/values/exoticValues.kt index f6122dbb..fa69e8da 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/values/exoticValues.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/values/exoticValues.kt @@ -14,6 +14,8 @@ class LazyParsedValue(override val string: String) : Value { override fun toString(): String = string override fun equals(other: Any?): Boolean = other is Value && this.parsedValue == other + + override fun hashCode(): Int = string.hashCode() } fun String.lazyParseValue(): LazyParsedValue = LazyParsedValue(this) diff --git a/dataforge-meta/src/jsTest/kotlin/hep/dataforge/meta/DynamicMetaTest.kt b/dataforge-meta/src/jsTest/kotlin/hep/dataforge/meta/DynamicMetaTest.kt index 56033171..ce46a740 100644 --- a/dataforge-meta/src/jsTest/kotlin/hep/dataforge/meta/DynamicMetaTest.kt +++ b/dataforge-meta/src/jsTest/kotlin/hep/dataforge/meta/DynamicMetaTest.kt @@ -10,7 +10,7 @@ class DynamicMetaTest { fun testDynamicMeta() { val d = js("{}") d.a = 22 - d.array = arrayOf(1,2,3) + d.array = arrayOf(1, 2, 3) d.b = "myString" d.ob = js("{}") d.ob.childNode = 18 @@ -18,7 +18,7 @@ class DynamicMetaTest { val meta = DynamicMeta(d) assertEquals(true, meta["ob.booleanNode"].boolean) - assertEquals(2,meta["array[1]"].int) + assertEquals(2, meta["array[1]"].int) } } \ No newline at end of file From 3ff3a5d09eacf3150fdb5220c73fda52ebbdf4db Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Mon, 7 Oct 2019 13:22:44 +0300 Subject: [PATCH 21/48] Fixed function server --- build.gradle.kts | 2 +- .../kotlin/hep/dataforge/io/IOPlugin.kt | 8 +++- .../dataforge/io/functions/FunctionServer.kt | 38 ++++++++++++++----- .../io/functions/RemoteFunctionClient.kt | 29 ++++++++------ .../io/functions/RemoteFunctionServer.kt | 2 +- .../kotlin/hep/dataforge/io/ioFormatsJVM.kt | 1 + .../kotlin/hep/dataforge/meta/Specific.kt | 8 +++- 7 files changed, 63 insertions(+), 25 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index 8dec72d0..06d80994 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -3,7 +3,7 @@ plugins { id("scientifik.publish") version "0.2.0" apply false } -val dataforgeVersion by extra("0.1.4-dev-3") +val dataforgeVersion by extra("0.1.4-dev-4") val bintrayRepo by extra("dataforge") val githubProject by extra("dataforge-core") diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOPlugin.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOPlugin.kt index 2dbd2b88..d835c634 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOPlugin.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOPlugin.kt @@ -31,9 +31,13 @@ class IOPlugin(meta: Meta) : AbstractPlugin(meta) { context.content>(IOFormat.TYPE) } - fun resolveIOFormat(item: MetaItem<*>): IOFormat<*>? { + fun resolveIOFormat(item: MetaItem<*>, type: KClass): IOFormat? { val key = item.string ?: error("Not a string value!") - return ioFormats[key] + return ioFormats[key]?.let { + @Suppress("UNCHECKED_CAST") + if(it.type!= type) error("Format type ${it.type} is not the same as requested type ${type}") + else it as IOFormat + } } companion object : PluginFactory { diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/functions/FunctionServer.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/functions/FunctionServer.kt index 872a5989..2eca605c 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/functions/FunctionServer.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/functions/FunctionServer.kt @@ -7,6 +7,7 @@ import hep.dataforge.meta.Meta import hep.dataforge.meta.get import hep.dataforge.names.asName import hep.dataforge.names.plus +import kotlin.reflect.KClass /** @@ -16,21 +17,25 @@ interface FunctionServer : ContextAware { /** * Call a function with given name and descriptor */ - suspend fun call(meta: Meta, arg: T): R + suspend fun call(meta: Meta, arg: T, inputType: KClass, outputType: KClass): R suspend fun callMany( meta: Meta, - arg: List + arg: List, + inputType: KClass, + outputType: KClass ): List = List(arg.size) { - call(meta, arg[it]) + call(meta, arg[it], inputType, outputType) } /** * Get a generic suspended function with given name and descriptor */ fun function( - meta: Meta - ): (suspend (T) -> R) = { call(meta, it) } + meta: Meta, + inputType: KClass, + outputType: KClass + ): (suspend (T) -> R) = { call(meta, it, inputType, outputType) } companion object { const val FUNCTION_NAME_KEY = "function" @@ -40,16 +45,31 @@ interface FunctionServer : ContextAware { } } -fun IOPlugin.getInputFormat(meta: Meta): IOFormat { +suspend inline fun FunctionServer.call(meta: Meta, arg: T) = + call(meta, arg, T::class, R::class) + +suspend inline fun FunctionServer.callMany(meta: Meta, arg: List) = + callMany(meta, arg, T::class, R::class) + +inline fun FunctionServer.function(meta: Meta) = + function(meta, T::class, R::class) + +fun IOPlugin.getInputFormat(meta: Meta, type: KClass): IOFormat { return meta[FunctionServer.INPUT_FORMAT_KEY]?.let { - resolveIOFormat(it) as IOFormat + resolveIOFormat(it, type) } ?: error("Input format not resolved") } -fun IOPlugin.getOutputFormat(meta: Meta): IOFormat { +fun IOPlugin.getOutputFormat(meta: Meta, type: KClass): IOFormat { return meta[FunctionServer.OUTPUT_FORMAT_KEY]?.let { - resolveIOFormat(it) as IOFormat + resolveIOFormat(it, type) } ?: error("Input format not resolved") } +inline fun IOPlugin.getInputFormat(meta: Meta): IOFormat = + getInputFormat(meta, T::class) + +inline fun IOPlugin.getOutputFormat(meta: Meta): IOFormat = + getOutputFormat(meta, R::class) + diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/functions/RemoteFunctionClient.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/functions/RemoteFunctionClient.kt index 770e5578..4b586ef7 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/functions/RemoteFunctionClient.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/functions/RemoteFunctionClient.kt @@ -6,17 +6,19 @@ import hep.dataforge.io.* import hep.dataforge.meta.Meta import hep.dataforge.meta.get import hep.dataforge.meta.int +import kotlin.reflect.KClass class RemoteFunctionClient(override val context: Context, val responder: Responder) : FunctionServer, ContextAware { private fun IOPlugin.encodeOne( meta: Meta, - value: T + value: T, + valueType: KClass = value::class ): Envelope = Envelope.invoke { meta(meta) type = REQUEST_TYPE data { - val inputFormat: IOFormat = getInputFormat(meta) + val inputFormat: IOFormat = getInputFormat(meta, valueType) inputFormat.run { writeThis(value) } @@ -25,7 +27,8 @@ class RemoteFunctionClient(override val context: Context, val responder: Respond private fun IOPlugin.encodeMany( meta: Meta, - values: List + values: List, + valueType: KClass ): Envelope = Envelope.invoke { meta(meta) type = REQUEST_TYPE @@ -33,7 +36,7 @@ class RemoteFunctionClient(override val context: Context, val responder: Respond SIZE_KEY to values.size } data { - val inputFormat: IOFormat = getInputFormat(meta) + val inputFormat: IOFormat = getInputFormat(meta, valueType) inputFormat.run { values.forEach { writeThis(it) @@ -42,14 +45,14 @@ class RemoteFunctionClient(override val context: Context, val responder: Respond } } - private fun IOPlugin.decode(envelope: Envelope): List { + private fun IOPlugin.decode(envelope: Envelope, valueType: KClass): List { require(envelope.type == RESPONSE_TYPE) { "Unexpected message type: ${envelope.type}" } val size = envelope.meta[SIZE_KEY].int ?: 1 return if (size == 0) { emptyList() } else { - val outputFormat: IOFormat = getOutputFormat(envelope.meta) + val outputFormat: IOFormat = getOutputFormat(envelope.meta, valueType) envelope.data?.read { List(size) { outputFormat.run { @@ -66,20 +69,24 @@ class RemoteFunctionClient(override val context: Context, val responder: Respond override suspend fun call( meta: Meta, - arg: T + arg: T, + inputType: KClass, + outputType: KClass ): R = plugin.run { val request = encodeOne(meta, arg) val response = responder.respond(request) - return decode(response).first() + return decode(response, outputType).first() } override suspend fun callMany( meta: Meta, - arg: List + arg: List, + inputType: KClass, + outputType: KClass ): List = plugin.run { - val request = encodeMany(meta, arg) + val request = encodeMany(meta, arg, inputType) val response = responder.respond(request) - return decode(response) + return decode(response, outputType) } companion object { diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/functions/RemoteFunctionServer.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/functions/RemoteFunctionServer.kt index b2fb6cd0..5bdf3139 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/functions/RemoteFunctionServer.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/functions/RemoteFunctionServer.kt @@ -23,7 +23,7 @@ class RemoteFunctionServer( require(request.type == RemoteFunctionClient.REQUEST_TYPE) { "Unexpected message type: ${request.type}" } val inputFormat = plugin.getInputFormat(request.meta) - val outputFormat = plugin.getOutputFormat(request.meta) + val outputFormat = plugin.getOutputFormat(request.meta) val size = request.meta[RemoteFunctionClient.SIZE_KEY].int ?: 1 diff --git a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/ioFormatsJVM.kt b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/ioFormatsJVM.kt index 6d075284..ca3ca17e 100644 --- a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/ioFormatsJVM.kt +++ b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/ioFormatsJVM.kt @@ -5,6 +5,7 @@ import hep.dataforge.io.functions.FunctionServer import hep.dataforge.io.functions.FunctionServer.Companion.FUNCTION_NAME_KEY import hep.dataforge.io.functions.FunctionServer.Companion.INPUT_FORMAT_KEY import hep.dataforge.io.functions.FunctionServer.Companion.OUTPUT_FORMAT_KEY +import hep.dataforge.io.functions.function import hep.dataforge.meta.Meta import hep.dataforge.meta.buildMeta import hep.dataforge.names.Name diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Specific.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Specific.kt index 571e1ee6..d3ec418a 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Specific.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Specific.kt @@ -1,6 +1,7 @@ package hep.dataforge.meta import hep.dataforge.names.Name +import kotlin.jvm.JvmName /** * Marker interface for classes with specifications @@ -64,4 +65,9 @@ fun > S.createStyle(action: C.() -> Unit): Me fun Specific.spec( spec: Specification, key: Name? = null -): MutableMorphDelegate = MutableMorphDelegate(config, key) { spec.wrap(it) } \ No newline at end of file +): MutableMorphDelegate = MutableMorphDelegate(config, key) { spec.wrap(it) } + +fun MetaItem<*>.spec(spec: Specification): T? = node?.let { spec.wrap(it)} + +@JvmName("configSpec") +fun MetaItem.spec(spec: Specification): T? = node?.let { spec.wrap(it)} \ No newline at end of file From 156a43d4bf91ee070667b5df5785119f7b098373 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Mon, 7 Oct 2019 13:27:54 +0300 Subject: [PATCH 22/48] Fixed enum delegate --- .../src/commonMain/kotlin/hep/dataforge/meta/Meta.kt | 2 +- .../kotlin/hep/dataforge/meta/MetaExtensionTest.kt | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) rename dataforge-meta/src/{jvmTest => commonTest}/kotlin/hep/dataforge/meta/MetaExtensionTest.kt (78%) diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Meta.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Meta.kt index 6b255a63..04c5f27b 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Meta.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Meta.kt @@ -242,7 +242,7 @@ val MetaItem<*>?.long get() = number?.toLong() val MetaItem<*>?.short get() = number?.toShort() inline fun > MetaItem<*>?.enum() = if (this is ValueItem && this.value is EnumValue<*>) { - this.value as E + this.value.value as E } else { string?.let { enumValueOf(it) } } diff --git a/dataforge-meta/src/jvmTest/kotlin/hep/dataforge/meta/MetaExtensionTest.kt b/dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/MetaExtensionTest.kt similarity index 78% rename from dataforge-meta/src/jvmTest/kotlin/hep/dataforge/meta/MetaExtensionTest.kt rename to dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/MetaExtensionTest.kt index 5f82df9a..30b6df18 100644 --- a/dataforge-meta/src/jvmTest/kotlin/hep/dataforge/meta/MetaExtensionTest.kt +++ b/dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/MetaExtensionTest.kt @@ -1,6 +1,7 @@ package hep.dataforge.meta -import org.junit.Test +import kotlin.test.Test + class MetaExtensionTest { @@ -15,7 +16,7 @@ class MetaExtensionTest { } @Test fun testEnumByString(){ - val meta = buildMeta{"enum" to TestEnum.test.toString()} + val meta = buildMeta{"enum" to TestEnum.test.name} println(meta["enum"].enum()) } From 23a2d29e846bdfeefdeb02d9b5c759eb2f2ad0a7 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Wed, 9 Oct 2019 19:10:37 +0300 Subject: [PATCH 23/48] Upgraded plugin and added test for data gathering in workspace. --- build.gradle.kts | 6 +++--- .../dataforge/descriptors/ItemDescriptor.kt | 18 ++++++++---------- .../hep/dataforge/meta/configDelegates.kt | 4 ++-- .../kotlin/hep/dataforge/meta/metaDelegates.kt | 10 +++++----- .../hep/dataforge/workspace/GenericTask.kt | 16 ++++++++-------- .../hep/dataforge/workspace/TaskModel.kt | 6 ++---- .../dataforge/workspace/SimpleWorkspaceTest.kt | 14 ++++++++++++++ 7 files changed, 42 insertions(+), 32 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index 06d80994..a2a7f5b9 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -1,9 +1,9 @@ plugins { - id("scientifik.mpp") version "0.2.0" apply false - id("scientifik.publish") version "0.2.0" apply false + id("scientifik.mpp") version "0.2.1" apply false + id("scientifik.publish") version "0.2.1" apply false } -val dataforgeVersion by extra("0.1.4-dev-4") +val dataforgeVersion by extra("0.1.4-dev-5") val bintrayRepo by extra("dataforge") val githubProject by extra("dataforge-core") diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/descriptors/ItemDescriptor.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/descriptors/ItemDescriptor.kt index 09e4d11e..0a25ff4d 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/descriptors/ItemDescriptor.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/descriptors/ItemDescriptor.kt @@ -32,13 +32,11 @@ sealed class ItemDescriptor(override val config: Config) : Specific { var info: String? by string() /** - * A list of tags for this item. Tags used to customize item usage + * Additional attributes of an item. For example validation and widget parameters * * @return */ - var tags: List by value { value -> - value?.list?.map { it.string } ?: emptyList() - } + var attributes by node() /** * True if the item is required @@ -54,7 +52,7 @@ sealed class ItemDescriptor(override val config: Config) : Specific { * * @author Alexander Nozik */ -class NodeDescriptor(config: Config) : ItemDescriptor(config){ +class NodeDescriptor(config: Config) : ItemDescriptor(config) { /** * True if the node is required @@ -68,7 +66,7 @@ class NodeDescriptor(config: Config) : ItemDescriptor(config){ * * @return */ - var default: Meta? by node() + var default: Config? by node() /** * The list of value descriptors @@ -79,7 +77,7 @@ class NodeDescriptor(config: Config) : ItemDescriptor(config){ } fun value(name: String, descriptor: ValueDescriptor) { - if(items.keys.contains(name)) error("The key $name already exists in descriptor") + if (items.keys.contains(name)) error("The key $name already exists in descriptor") val token = NameToken(VALUE_KEY, name) config[token] = descriptor.config } @@ -101,7 +99,7 @@ class NodeDescriptor(config: Config) : ItemDescriptor(config){ fun node(name: String, descriptor: NodeDescriptor) { - if(items.keys.contains(name)) error("The key $name already exists in descriptor") + if (items.keys.contains(name)) error("The key $name already exists in descriptor") val token = NameToken(NODE_KEY, name) config[token] = descriptor.config } @@ -117,7 +115,7 @@ class NodeDescriptor(config: Config) : ItemDescriptor(config){ companion object : Specification { -// const val ITEM_KEY = "item" + // const val ITEM_KEY = "item" const val NODE_KEY = "node" const val VALUE_KEY = "value" @@ -135,7 +133,7 @@ class NodeDescriptor(config: Config) : ItemDescriptor(config){ * * @author Alexander Nozik */ -class ValueDescriptor(config: Config) : ItemDescriptor(config){ +class ValueDescriptor(config: Config) : ItemDescriptor(config) { /** diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/configDelegates.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/configDelegates.kt index 45b6b4d2..1dabd05b 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/configDelegates.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/configDelegates.kt @@ -105,7 +105,7 @@ inline fun > Configurable.enum(default: E, key: Name? = null /* Node delegates */ -fun Configurable.node(key: Name? = null) = MutableNodeDelegate(config, key) +fun Configurable.node(key: Name? = null): MutableNodeDelegate = MutableNodeDelegate(config, key) fun Configurable.spec(spec: Specification, key: Name? = null) = MutableMorphDelegate(config, key) { spec.wrap(it) } @@ -133,5 +133,5 @@ fun Configurable.doubleArray(key: Name? = null): ReadWriteDelegateWrapper Configurable.child(key: Name? = null, converter: (Meta) -> T) = +fun Configurable.node(key: Name? = null, converter: (Meta) -> T) = MutableMorphDelegate(config, key, converter) diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/metaDelegates.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/metaDelegates.kt index e9c19864..6bed5413 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/metaDelegates.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/metaDelegates.kt @@ -138,7 +138,7 @@ fun Meta.boolean(default: Boolean? = null, key: String? = null) = BooleanDelegat fun Meta.number(default: Number? = null, key: String? = null) = NumberDelegate(this, key, default) -fun Meta.child(key: String? = null) = ChildDelegate(this, key) { it } +fun Meta.node(key: String? = null) = ChildDelegate(this, key) { it } @JvmName("safeString") fun Meta.string(default: String, key: String? = null) = @@ -169,7 +169,7 @@ inline fun > Meta.enum(default: E, key: String? = null) = SafeEnumDelegate(this, key, default) { enumValueOf(it) } -fun Metoid.child(key: String? = null, converter: (Meta) -> T) = ChildDelegate(meta, key, converter) +fun Metoid.node(key: String? = null, converter: (Meta) -> T) = ChildDelegate(meta, key, converter) /* Read-write delegates */ @@ -337,12 +337,12 @@ class MutableSafeEnumvDelegate, E : Enum>( class MutableNodeDelegate>( val meta: M, private val key: Name? = null -) : ReadWriteProperty { - override fun getValue(thisRef: Any?, property: KProperty<*>): Meta? { +) : ReadWriteProperty { + override fun getValue(thisRef: Any?, property: KProperty<*>): M? { return meta[key ?: property.name.asName()]?.node } - override fun setValue(thisRef: Any?, property: KProperty<*>, value: Meta?) { + override fun setValue(thisRef: Any?, property: KProperty<*>, value: M?) { meta[key ?: property.name.asName()] = value } } diff --git a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/GenericTask.kt b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/GenericTask.kt index f59c3d24..ff499888 100644 --- a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/GenericTask.kt +++ b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/GenericTask.kt @@ -19,20 +19,20 @@ class GenericTask( private val dataTransform: Workspace.() -> TaskModel.(DataNode) -> DataNode ) : Task { - private fun gather(workspace: Workspace, model: TaskModel): DataNode { - return DataNode.invoke(Any::class) { - model.dependencies.forEach { dep -> - update(dep.apply(workspace)) - } - } - } +// private fun gather(workspace: Workspace, model: TaskModel): DataNode { +// return DataNode.invoke(Any::class) { +// model.dependencies.forEach { dep -> +// update(dep.apply(workspace)) +// } +// } +// } override fun run(workspace: Workspace, model: TaskModel): DataNode { //validate model validate(model) // gather data - val input = gather(workspace, model) + val input = model.buildInput(workspace)// gather(workspace, model) //execute workspace.context.logger.info{"Starting task '$name' on ${model.target} with meta: \n${model.meta}"} diff --git a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskModel.kt b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskModel.kt index 7f9cd976..f71f01a0 100644 --- a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskModel.kt +++ b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskModel.kt @@ -8,7 +8,6 @@ package hep.dataforge.workspace import hep.dataforge.data.DataFilter import hep.dataforge.data.DataTree import hep.dataforge.data.DataTreeBuilder -import hep.dataforge.data.dataSequence import hep.dataforge.meta.* import hep.dataforge.names.EmptyName import hep.dataforge.names.Name @@ -53,9 +52,8 @@ data class TaskModel( */ fun TaskModel.buildInput(workspace: Workspace): DataTree { return DataTreeBuilder(Any::class).apply { - dependencies.asSequence().flatMap { it.apply(workspace).dataSequence() }.forEach { (name, data) -> - //TODO add concise error on replacement - this[name] = data + dependencies.forEach { dep -> + update(dep.apply(workspace)) } }.build() } diff --git a/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/SimpleWorkspaceTest.kt b/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/SimpleWorkspaceTest.kt index f45f8720..3059cb29 100644 --- a/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/SimpleWorkspaceTest.kt +++ b/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/SimpleWorkspaceTest.kt @@ -35,6 +35,14 @@ class SimpleWorkspaceTest { } } + val filterTask = task("filterOne") { + model { + data("myData\\[12\\]") + } + pipe{ + it + } + } val square = task("square") { pipe { data -> @@ -150,4 +158,10 @@ class SimpleWorkspaceTest { val node = workspace.run("fullsquare") println(node.toMeta()) } + + @Test + fun testGather() { + val node = workspace.run("filterOne") + assertEquals(12, node.first()?.get()) + } } \ No newline at end of file From 44a34eee40270ea1595a5b0900e2c39ae4dfc505 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Sat, 19 Oct 2019 13:35:50 +0300 Subject: [PATCH 24/48] Refactored IO logic --- build.gradle.kts | 2 +- .../kotlin/hep/dataforge/context/Factory.kt | 8 ++ .../hep/dataforge/context/PluginManager.kt | 14 ++-- .../hep/dataforge/context/PluginRepository.kt | 14 ++-- .../hep/dataforge/io/BinaryMetaFormat.kt | 5 +- .../kotlin/hep/dataforge/io/EnvelopeFormat.kt | 27 +++---- .../kotlin/hep/dataforge/io/IOFormat.kt | 42 ++++++---- .../kotlin/hep/dataforge/io/IOPlugin.kt | 53 +++++++------ .../kotlin/hep/dataforge/io/JsonMetaFormat.kt | 25 +++--- .../kotlin/hep/dataforge/io/MetaFormat.kt | 30 ++++--- .../hep/dataforge/io/TaggedEnvelopeFormat.kt | 56 +++++++++---- .../hep/dataforge/io/EnvelopeFormatTest.kt | 2 +- .../kotlin/hep/dataforge/io/FileEnvelope.kt | 23 ++++-- .../hep/dataforge/io/tcp/EnvelopeClient.kt | 7 +- .../hep/dataforge/io/tcp/EnvelopeServer.kt | 9 ++- .../hep/dataforge/io/FileEnvelopeTest.kt | 5 +- .../dataforge/io/tcp/EnvelopeServerTest.kt | 2 +- .../kotlin/hep/dataforge/meta/Meta.kt | 12 +-- .../hep/dataforge/output/OutputManager.kt | 2 +- .../hep/dataforge/workspace/fileData.kt | 78 ++++++++++++------- 20 files changed, 263 insertions(+), 153 deletions(-) create mode 100644 dataforge-context/src/commonMain/kotlin/hep/dataforge/context/Factory.kt diff --git a/build.gradle.kts b/build.gradle.kts index a2a7f5b9..c5d5c73f 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -3,7 +3,7 @@ plugins { id("scientifik.publish") version "0.2.1" apply false } -val dataforgeVersion by extra("0.1.4-dev-5") +val dataforgeVersion by extra("0.1.4-dev-6") val bintrayRepo by extra("dataforge") val githubProject by extra("dataforge-core") diff --git a/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/Factory.kt b/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/Factory.kt new file mode 100644 index 00000000..b20672c2 --- /dev/null +++ b/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/Factory.kt @@ -0,0 +1,8 @@ +package hep.dataforge.context + +import hep.dataforge.meta.EmptyMeta +import hep.dataforge.meta.Meta + +interface Factory { + operator fun invoke(meta: Meta = EmptyMeta, context: Context = Global): T +} \ No newline at end of file diff --git a/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/PluginManager.kt b/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/PluginManager.kt index dad483a8..dd114f79 100644 --- a/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/PluginManager.kt +++ b/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/PluginManager.kt @@ -38,7 +38,7 @@ class PluginManager(override val context: Context) : ContextAware, Iterable Boolean): Plugin? = sequence(recursive).find(predicate) + fun find(recursive: Boolean = true, predicate: (Plugin) -> Boolean): Plugin? = sequence(recursive).find(predicate) /** @@ -47,7 +47,7 @@ class PluginManager(override val context: Context) : ContextAware, Iterable get(type: KClass, tag: PluginTag? = null, recursive: Boolean = true): T? = - get(recursive) { type.isInstance(it) && (tag == null || tag.matches(it.tag)) } as T? + find(recursive) { type.isInstance(it) && (tag == null || tag.matches(it.tag)) } as T? - inline fun get(tag: PluginTag? = null, recursive: Boolean = true): T? = + inline operator fun get(tag: PluginTag? = null, recursive: Boolean = true): T? = get(T::class, tag, recursive) + inline operator fun get(factory: PluginFactory, recursive: Boolean = true): T? = + get(factory.type, factory.tag, recursive) /** * Load given plugin into this manager and return loaded instance. @@ -97,7 +99,7 @@ class PluginManager(override val context: Context) : ContextAware, Iterable load(factory: PluginFactory, meta: Meta = EmptyMeta): T = - load(factory(meta)) + load(factory(meta, context)) fun load(factory: PluginFactory, metaBuilder: MetaBuilder.() -> Unit): T = load(factory, buildMeta(metaBuilder)) @@ -122,7 +124,7 @@ class PluginManager(override val context: Context) : ContextAware, Iterable fetch(factory: PluginFactory, recursive: Boolean = true, meta: Meta = EmptyMeta): T { val loaded = get(factory.type, factory.tag, recursive) return when { - loaded == null -> load(factory(meta)) + loaded == null -> load(factory(meta, context)) loaded.meta == meta -> loaded // if meta is the same, return existing plugin else -> throw RuntimeException("Can't load plugin with tag ${factory.tag}. Plugin with this tag and different configuration already exists in context.") } diff --git a/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/PluginRepository.kt b/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/PluginRepository.kt index eb715f80..06c1132b 100644 --- a/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/PluginRepository.kt +++ b/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/PluginRepository.kt @@ -4,10 +4,9 @@ import hep.dataforge.meta.EmptyMeta import hep.dataforge.meta.Meta import kotlin.reflect.KClass -interface PluginFactory { +interface PluginFactory : Factory { val tag: PluginTag val type: KClass - operator fun invoke(meta: Meta = EmptyMeta): T } expect object PluginRepository { @@ -25,25 +24,26 @@ expect object PluginRepository { * Fetch specific plugin and instantiate it with given meta */ fun PluginRepository.fetch(tag: PluginTag, meta: Meta = EmptyMeta): Plugin = - list().find { it.tag.matches(tag) }?.invoke(meta) ?: error("Plugin with tag $tag not found in the repository") + list().find { it.tag.matches(tag) }?.invoke(meta = meta) + ?: error("Plugin with tag $tag not found in the repository") fun PluginRepository.register( tag: PluginTag, type: KClass, - constructor: (Meta) -> T + constructor: (Context, Meta) -> T ): PluginFactory { val factory = object : PluginFactory { override val tag: PluginTag = tag override val type: KClass = type - override fun invoke(meta: Meta): T = constructor(meta) + override fun invoke(meta: Meta, context: Context): T = constructor(context, meta) } register(factory) return factory } -inline fun PluginRepository.register(tag: PluginTag, noinline constructor: (Meta) -> T) = +inline fun PluginRepository.register(tag: PluginTag, noinline constructor: (Context, Meta) -> T) = register(tag, T::class, constructor) -fun PluginRepository.register(plugin: Plugin) = register(plugin.tag, plugin::class) { plugin } \ No newline at end of file +fun PluginRepository.register(plugin: Plugin) = register(plugin.tag, plugin::class) { _, _ -> plugin } \ No newline at end of file diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/BinaryMetaFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/BinaryMetaFormat.kt index 8774387b..48d84904 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/BinaryMetaFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/BinaryMetaFormat.kt @@ -1,5 +1,6 @@ package hep.dataforge.io +import hep.dataforge.context.Context import hep.dataforge.descriptors.NodeDescriptor import hep.dataforge.meta.* import hep.dataforge.names.Name @@ -10,10 +11,12 @@ import kotlinx.io.core.Output import kotlinx.io.core.readText import kotlinx.io.core.writeText -object BinaryMetaFormat : MetaFormat { +object BinaryMetaFormat : MetaFormat, MetaFormatFactory { override val name: Name = super.name + "bin" override val key: Short = 0x4249//BI + override fun invoke(meta: Meta, context: Context): MetaFormat = this + override fun Input.readMeta(descriptor: NodeDescriptor?): Meta { return (readMetaItem() as MetaItem.NodeItem).node } diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/EnvelopeFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/EnvelopeFormat.kt index a0734c04..3fd4e96a 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/EnvelopeFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/EnvelopeFormat.kt @@ -1,7 +1,7 @@ package hep.dataforge.io -import hep.dataforge.context.Named -import hep.dataforge.io.EnvelopeFormat.Companion.ENVELOPE_FORMAT_TYPE +import hep.dataforge.context.Context +import hep.dataforge.io.EnvelopeFormatFactory.Companion.ENVELOPE_FORMAT_TYPE import hep.dataforge.meta.Meta import hep.dataforge.names.Name import hep.dataforge.names.asName @@ -16,22 +16,23 @@ import kotlin.reflect.KClass @ExperimentalUnsignedTypes data class PartialEnvelope(val meta: Meta, val dataOffset: UInt, val dataSize: ULong?) + +interface EnvelopeFormat : IOFormat { + fun Input.readPartial(): PartialEnvelope + + override fun Input.readThis(): Envelope + + override fun Output.writeThis(obj: Envelope) +} + @Type(ENVELOPE_FORMAT_TYPE) -interface EnvelopeFormat : IOFormat, Named { +interface EnvelopeFormatFactory : IOFormatFactory { override val name: Name get() = "envelope".asName() override val type: KClass get() = Envelope::class - fun Input.readPartial(formats: Collection = IOPlugin.defaultMetaFormats): PartialEnvelope - - fun Input.readEnvelope(formats: Collection = IOPlugin.defaultMetaFormats): Envelope - - override fun Input.readThis(): Envelope = readEnvelope() - - fun Output.writeEnvelope(envelope: Envelope, format: MetaFormat = JsonMetaFormat) - - override fun Output.writeThis(obj: Envelope) = writeEnvelope(obj) + override fun invoke(meta: Meta, context: Context): EnvelopeFormat companion object { - const val ENVELOPE_FORMAT_TYPE = "envelopeFormat" + const val ENVELOPE_FORMAT_TYPE = "io.format.envelope" } } \ No newline at end of file diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOFormat.kt index e2a1f16e..6c5e1244 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOFormat.kt @@ -1,12 +1,13 @@ package hep.dataforge.io +import hep.dataforge.context.Context +import hep.dataforge.context.Factory import hep.dataforge.context.Named -import hep.dataforge.io.IOFormat.Companion.TYPE +import hep.dataforge.io.IOFormatFactory.Companion.IO_FORMAT_TYPE +import hep.dataforge.meta.Meta import hep.dataforge.meta.MetaItem -import hep.dataforge.names.EmptyName import hep.dataforge.names.Name import hep.dataforge.names.asName -import hep.dataforge.names.toName import hep.dataforge.provider.Type import hep.dataforge.values.Value import kotlinx.io.core.* @@ -19,27 +20,33 @@ import kotlin.reflect.KClass /** * And interface for serialization facilities */ -@Type(TYPE) -interface IOFormat : Named { + +interface IOFormat { + fun Output.writeThis(obj: T) + fun Input.readThis(): T + + +} + +fun IOFormat.writePacket(obj: T): ByteReadPacket = buildPacket { writeThis(obj) } +fun IOFormat.writeBytes(obj: T): ByteArray = buildPacket { writeThis(obj) }.readBytes() +fun IOFormat.readBytes(array: ByteArray): T = ByteReadPacket(array).readThis() + +@Type(IO_FORMAT_TYPE) +interface IOFormatFactory : Factory>, Named { /** * Explicit type for dynamic type checks */ val type: KClass - fun Output.writeThis(obj: T) - fun Input.readThis(): T - companion object { - const val TYPE = "ioFormat" + const val IO_FORMAT_TYPE = "io.format" } } -fun IOFormat.writePacket(obj: T): ByteReadPacket = buildPacket { writeThis(obj) } -fun IOFormat.writeBytes(obj: T): ByteArray = buildPacket { writeThis(obj) }.readBytes() -fun IOFormat.readBytes(array: ByteArray): T = ByteReadPacket(array).readThis() - -object DoubleIOFormat : IOFormat { +object DoubleIOFormat : IOFormat, IOFormatFactory { + override fun invoke(meta: Meta, context: Context): IOFormat = this override val name: Name = "double".asName() @@ -52,7 +59,8 @@ object DoubleIOFormat : IOFormat { override fun Input.readThis(): Double = readDouble() } -object ValueIOFormat : IOFormat { +object ValueIOFormat : IOFormat, IOFormatFactory { + override fun invoke(meta: Meta, context: Context): IOFormat = this override val name: Name = "value".asName() @@ -73,11 +81,11 @@ object ValueIOFormat : IOFormat { */ @ImplicitReflectionSerializer class SerializerIOFormat( - override val type: KClass, + type: KClass, val serializer: KSerializer = type.serializer() ) : IOFormat { - override val name: Name = type.simpleName?.toName() ?: EmptyName + //override val name: Name = type.simpleName?.toName() ?: EmptyName override fun Output.writeThis(obj: T) { diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOPlugin.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOPlugin.kt index d835c634..0293daf2 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOPlugin.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOPlugin.kt @@ -1,9 +1,10 @@ package hep.dataforge.io import hep.dataforge.context.* -import hep.dataforge.meta.Meta -import hep.dataforge.meta.MetaItem -import hep.dataforge.meta.string +import hep.dataforge.io.EnvelopeFormatFactory.Companion.ENVELOPE_FORMAT_TYPE +import hep.dataforge.io.IOFormatFactory.Companion.IO_FORMAT_TYPE +import hep.dataforge.io.MetaFormatFactory.Companion.META_FORMAT_TYPE +import hep.dataforge.meta.* import hep.dataforge.names.Name import hep.dataforge.names.get import kotlin.reflect.KClass @@ -11,47 +12,49 @@ import kotlin.reflect.KClass class IOPlugin(meta: Meta) : AbstractPlugin(meta) { override val tag: PluginTag get() = Companion.tag - val metaFormats by lazy { - context.content(MetaFormat.META_FORMAT_TYPE).values + val metaFormatFactories by lazy { + context.content(META_FORMAT_TYPE).values } - fun metaFormat(key: Short): MetaFormat? = metaFormats.find { it.key == key } - fun metaFormat(name: String): MetaFormat? = metaFormats.find { it.name.toString() == name } + fun metaFormat(key: Short, meta: Meta = EmptyMeta): MetaFormat? = + metaFormatFactories.find { it.key == key }?.invoke(meta) + + fun metaFormat(name: String, meta: Meta = EmptyMeta): MetaFormat? = + metaFormatFactories.find { it.name.toString() == name }?.invoke(meta) + + val envelopeFormatFactories by lazy { + context.content(ENVELOPE_FORMAT_TYPE).values + } override fun provideTop(target: String): Map { return when (target) { - MetaFormat.META_FORMAT_TYPE -> defaultMetaFormats.toMap() - EnvelopeFormat.ENVELOPE_FORMAT_TYPE -> defaultEnvelopeFormats.toMap() - IOFormat.TYPE -> defaultIOFormats.toMap() + META_FORMAT_TYPE -> defaultMetaFormats.toMap() + ENVELOPE_FORMAT_TYPE -> defaultEnvelopeFormats.toMap() else -> super.provideTop(target) } } - val ioFormats: Map> by lazy { - context.content>(IOFormat.TYPE) + val ioFormats: Map> by lazy { + context.content>(IO_FORMAT_TYPE) } - fun resolveIOFormat(item: MetaItem<*>, type: KClass): IOFormat? { - val key = item.string ?: error("Not a string value!") + fun resolveIOFormat(item: MetaItem<*>, type: KClass): IOFormat? { + val key = item.string ?: item.node["name"]?.string ?: error("Format name not defined") return ioFormats[key]?.let { @Suppress("UNCHECKED_CAST") - if(it.type!= type) error("Format type ${it.type} is not the same as requested type ${type}") - else it as IOFormat + if (it.type != type) error("Format type ${it.type} is not the same as requested type $type") + else it.invoke(item.node["meta"].node ?: EmptyMeta, context) as IOFormat } } companion object : PluginFactory { - val defaultMetaFormats: List = listOf(JsonMetaFormat, BinaryMetaFormat) + val defaultMetaFormats: List = listOf(JsonMetaFormat, BinaryMetaFormat) val defaultEnvelopeFormats = listOf(TaggedEnvelopeFormat) - val defaultIOFormats = listOf( - DoubleIOFormat, - ValueIOFormat, - BinaryMetaFormat - ) - override val tag: PluginTag = PluginTag("io", group = PluginTag.DATAFORGE_GROUP) override val type: KClass = IOPlugin::class - override fun invoke(meta: Meta): IOPlugin = IOPlugin(meta) + override fun invoke(meta: Meta, context: Context): IOPlugin = IOPlugin(meta) } -} \ No newline at end of file +} + +val Context.io: IOPlugin get() = plugins.fetch(IOPlugin) \ No newline at end of file diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/JsonMetaFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/JsonMetaFormat.kt index e9ec443c..620d1579 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/JsonMetaFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/JsonMetaFormat.kt @@ -2,6 +2,7 @@ package hep.dataforge.io +import hep.dataforge.context.Context import hep.dataforge.descriptors.ItemDescriptor import hep.dataforge.descriptors.NodeDescriptor import hep.dataforge.descriptors.ValueDescriptor @@ -23,26 +24,32 @@ import kotlin.collections.component2 import kotlin.collections.set -object JsonMetaFormat : MetaFormat { - - override val name: Name = super.name + "json" - override val key: Short = 0x4a53//"JS" +class JsonMetaFormat(private val json: Json = Json.plain) : MetaFormat { override fun Output.writeMeta(meta: Meta, descriptor: NodeDescriptor?) { - val json = meta.toJson(descriptor) - writeText(json.toString()) + val jsonObject = meta.toJson(descriptor) + writeText(json.stringify(JsonObjectSerializer, jsonObject)) } override fun Input.readMeta(descriptor: NodeDescriptor?): Meta { val str = readText() - val json = Json.plain.parseJson(str) + val jsonElement = json.parseJson(str) - if (json is JsonObject) { - return json.toMeta() + if (jsonElement is JsonObject) { + return jsonElement.toMeta() } else { TODO("Non-object root not supported") } } + + companion object : MetaFormatFactory { + val default = JsonMetaFormat() + + override fun invoke(meta: Meta, context: Context): MetaFormat = default + + override val name: Name = super.name + "json" + override val key: Short = 0x4a53//"JS" + } } /** diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/MetaFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/MetaFormat.kt index c208701d..baa8864e 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/MetaFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/MetaFormat.kt @@ -1,8 +1,8 @@ package hep.dataforge.io -import hep.dataforge.context.Named +import hep.dataforge.context.Context import hep.dataforge.descriptors.NodeDescriptor -import hep.dataforge.io.MetaFormat.Companion.META_FORMAT_TYPE +import hep.dataforge.io.MetaFormatFactory.Companion.META_FORMAT_TYPE import hep.dataforge.meta.Meta import hep.dataforge.names.Name import hep.dataforge.names.asName @@ -13,12 +13,8 @@ import kotlin.reflect.KClass /** * A format for meta serialization */ -@Type(META_FORMAT_TYPE) -interface MetaFormat : IOFormat, Named { - override val name: Name get() = "meta".asName() - val key: Short - override val type: KClass get() = Meta::class +interface MetaFormat : IOFormat { override fun Output.writeThis(obj: Meta) { writeMeta(obj, null) @@ -28,9 +24,20 @@ interface MetaFormat : IOFormat, Named { fun Output.writeMeta(meta: Meta, descriptor: NodeDescriptor? = null) fun Input.readMeta(descriptor: NodeDescriptor? = null): Meta +} + +@Type(META_FORMAT_TYPE) +interface MetaFormatFactory : IOFormatFactory { + override val name: Name get() = "meta".asName() + + override val type: KClass get() = Meta::class + + val key: Short + + override operator fun invoke(meta: Meta, context: Context): MetaFormat companion object { - const val META_FORMAT_TYPE = "metaFormat" + const val META_FORMAT_TYPE = "io.format.meta" } } @@ -38,15 +45,18 @@ fun Meta.toString(format: MetaFormat): String = buildPacket { format.run { writeThis(this@toString) } }.readText() -fun Meta.toBytes(format: MetaFormat = JsonMetaFormat): ByteReadPacket = buildPacket { +fun Meta.toString(formatFactory: MetaFormatFactory): String = toString(formatFactory()) + +fun Meta.toBytes(format: MetaFormat = JsonMetaFormat.default): ByteReadPacket = buildPacket { format.run { writeThis(this@toBytes) } } - fun MetaFormat.parse(str: String): Meta { return ByteReadPacket(str.toByteArray()).readThis() } +fun MetaFormatFactory.parse(str: String): Meta = invoke().parse(str) + fun MetaFormat.fromBytes(packet: ByteReadPacket): Meta { return packet.readThis() } diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt index 20fc8632..480d08e3 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt @@ -1,17 +1,29 @@ package hep.dataforge.io +import hep.dataforge.context.Context +import hep.dataforge.meta.Meta +import hep.dataforge.meta.get +import hep.dataforge.meta.string import hep.dataforge.names.Name import hep.dataforge.names.plus +import hep.dataforge.names.toName import kotlinx.io.core.* @ExperimentalUnsignedTypes -object TaggedEnvelopeFormat : EnvelopeFormat { - const val VERSION = "DF03" - private const val START_SEQUENCE = "#~" - private const val END_SEQUENCE = "~#\r\n" - private const val TAG_SIZE = 24u +class TaggedEnvelopeFormat(val io: IOPlugin, meta: Meta) : EnvelopeFormat { - override val name: Name = super.name + VERSION + private val metaFormat: MetaFormat + + private val metaFormatKey: Short + + init { + val metaName = meta["name"].string?.toName() ?: JsonMetaFormat.name + val metaFormatFactory = io.metaFormatFactories.find { it.name == metaName } + ?: error("Meta format could not be resolved") + + metaFormat = metaFormatFactory(meta, io.context) + metaFormatKey = metaFormatFactory.key + } private fun Tag.toBytes(): ByteReadPacket = buildPacket(24) { writeText(START_SEQUENCE) @@ -35,12 +47,12 @@ object TaggedEnvelopeFormat : EnvelopeFormat { return Tag(metaFormatKey, metaLength, dataLength) } - override fun Output.writeEnvelope(envelope: Envelope, format: MetaFormat) { - val metaBytes = format.writeBytes(envelope.meta) - val tag = Tag(format.key, metaBytes.size.toUInt(), envelope.data?.size ?: 0.toULong()) + override fun Output.writeThis(obj: Envelope) { + val metaBytes = metaFormat.writeBytes(obj.meta) + val tag = Tag(metaFormatKey, metaBytes.size.toUInt(), obj.data?.size ?: 0.toULong()) writePacket(tag.toBytes()) writeFully(metaBytes) - envelope.data?.read { copyTo(this@writeEnvelope) } + obj.data?.read { copyTo(this@writeThis) } } /** @@ -49,10 +61,10 @@ object TaggedEnvelopeFormat : EnvelopeFormat { * @param input an input to read from * @param formats a collection of meta formats to resolve */ - override fun Input.readEnvelope(formats: Collection): Envelope { + override fun Input.readThis(): Envelope { val tag = readTag() - val metaFormat = formats.find { it.key == tag.metaFormatKey } + val metaFormat = io.metaFormat(tag.metaFormatKey) ?: error("Meta format with key ${tag.metaFormatKey} not found") val metaPacket = ByteReadPacket(readBytes(tag.metaSize.toInt())) @@ -62,10 +74,10 @@ object TaggedEnvelopeFormat : EnvelopeFormat { return SimpleEnvelope(meta, ArrayBinary(dataBytes)) } - override fun Input.readPartial(formats: Collection): PartialEnvelope { + override fun Input.readPartial(): PartialEnvelope { val tag = readTag() - val metaFormat = formats.find { it.key == tag.metaFormatKey } + val metaFormat = io.metaFormat(tag.metaFormatKey) ?: error("Meta format with key ${tag.metaFormatKey} not found") val metaPacket = ByteReadPacket(readBytes(tag.metaSize.toInt())) @@ -80,4 +92,20 @@ object TaggedEnvelopeFormat : EnvelopeFormat { val dataSize: ULong ) + companion object : EnvelopeFormatFactory { + const val VERSION = "DF03" + private const val START_SEQUENCE = "#~" + private const val END_SEQUENCE = "~#\r\n" + private const val TAG_SIZE = 24u + + override val name: Name = super.name + VERSION + + override fun invoke(meta: Meta, context: Context): EnvelopeFormat { + val plugin = context.plugins.fetch(IOPlugin) + return TaggedEnvelopeFormat(plugin, meta) + } + + val default = invoke() + } + } \ No newline at end of file diff --git a/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/EnvelopeFormatTest.kt b/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/EnvelopeFormatTest.kt index 5c1d82f9..3acb852c 100644 --- a/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/EnvelopeFormatTest.kt +++ b/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/EnvelopeFormatTest.kt @@ -18,7 +18,7 @@ class EnvelopeFormatTest { @ExperimentalStdlibApi @Test fun testTaggedFormat(){ - TaggedEnvelopeFormat.run { + TaggedEnvelopeFormat().run { val bytes = writeBytes(envelope) println(bytes.decodeToString()) val res = readBytes(bytes) diff --git a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileEnvelope.kt b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileEnvelope.kt index 88ebf285..6cb3473f 100644 --- a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileEnvelope.kt +++ b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileEnvelope.kt @@ -1,5 +1,6 @@ package hep.dataforge.io +import hep.dataforge.meta.EmptyMeta import hep.dataforge.meta.Meta import kotlinx.io.nio.asInput import kotlinx.io.nio.asOutput @@ -22,22 +23,30 @@ class FileEnvelope internal constructor(val path: Path, val format: EnvelopeForm override val data: Binary? = FileBinary(path, partialEnvelope.dataOffset, partialEnvelope.dataSize) } -fun Path.readEnvelope(format: EnvelopeFormat = TaggedEnvelopeFormat) = FileEnvelope(this, format) +fun IOPlugin.readEnvelopeFile( + path: Path, + formatFactory: EnvelopeFormatFactory = TaggedEnvelopeFormat, + formatMeta: Meta = EmptyMeta +): FileEnvelope { + val format = formatFactory(formatMeta, context) + return FileEnvelope(path, format) +} -fun Path.writeEnvelope( +fun IOPlugin.writeEnvelopeFile( + path: Path, envelope: Envelope, - format: EnvelopeFormat = TaggedEnvelopeFormat, - metaFormat: MetaFormat = JsonMetaFormat + formatFactory: EnvelopeFormatFactory = TaggedEnvelopeFormat, + formatMeta: Meta = EmptyMeta ) { val output = Files.newByteChannel( - this, + path, StandardOpenOption.WRITE, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING ).asOutput() - with(format) { - output.writeEnvelope(envelope, metaFormat) + with(formatFactory(formatMeta, context)) { + output.writeThis(envelope) } } diff --git a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/tcp/EnvelopeClient.kt b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/tcp/EnvelopeClient.kt index 9011ae1e..791a57b5 100644 --- a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/tcp/EnvelopeClient.kt +++ b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/tcp/EnvelopeClient.kt @@ -3,6 +3,8 @@ package hep.dataforge.io.tcp import hep.dataforge.context.Context import hep.dataforge.context.ContextAware import hep.dataforge.io.* +import hep.dataforge.meta.EmptyMeta +import hep.dataforge.meta.Meta import kotlinx.coroutines.asCoroutineDispatcher import kotlinx.coroutines.withContext import kotlinx.io.streams.writePacket @@ -15,11 +17,14 @@ class EnvelopeClient( override val context: Context, val host: String, val port: Int, - val format: EnvelopeFormat = TaggedEnvelopeFormat + formatFactory: EnvelopeFormatFactory = TaggedEnvelopeFormat, + formatMeta: Meta = EmptyMeta ) : Responder, ContextAware { private val dispatcher = Executors.newSingleThreadExecutor().asCoroutineDispatcher() + private val format = formatFactory(formatMeta, context = context) + // private var socket: SocketChannel? = null // // private fun getSocket(): Socket { diff --git a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/tcp/EnvelopeServer.kt b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/tcp/EnvelopeServer.kt index 68da8375..e81a1ea6 100644 --- a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/tcp/EnvelopeServer.kt +++ b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/tcp/EnvelopeServer.kt @@ -2,10 +2,12 @@ package hep.dataforge.io.tcp import hep.dataforge.context.Context import hep.dataforge.context.ContextAware -import hep.dataforge.io.EnvelopeFormat +import hep.dataforge.io.EnvelopeFormatFactory import hep.dataforge.io.Responder import hep.dataforge.io.TaggedEnvelopeFormat import hep.dataforge.io.type +import hep.dataforge.meta.EmptyMeta +import hep.dataforge.meta.Meta import kotlinx.coroutines.* import kotlinx.io.streams.writePacket import java.net.ServerSocket @@ -17,11 +19,14 @@ class EnvelopeServer( val port: Int, val responder: Responder, val scope: CoroutineScope, - val format: EnvelopeFormat = TaggedEnvelopeFormat + formatFactory: EnvelopeFormatFactory = TaggedEnvelopeFormat, + formatMeta: Meta = EmptyMeta ) : ContextAware { private var job: Job? = null + private val format = formatFactory(formatMeta, context = context) + fun start() { if (job == null) { logger.info { "Starting envelope server on port $port" } diff --git a/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/FileEnvelopeTest.kt b/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/FileEnvelopeTest.kt index acd427d4..c362a8c3 100644 --- a/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/FileEnvelopeTest.kt +++ b/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/FileEnvelopeTest.kt @@ -1,5 +1,6 @@ package hep.dataforge.io +import hep.dataforge.context.Global import java.nio.file.Files import kotlin.test.Ignore import kotlin.test.Test @@ -24,9 +25,9 @@ class FileEnvelopeTest { @Ignore fun testFileWriteRead() { val tmpPath = Files.createTempFile("dataforge_test", ".df") - tmpPath.writeEnvelope(envelope) + Global.io.writeEnvelopeFile(tmpPath,envelope) println(tmpPath.toUri()) - val restored: Envelope = tmpPath.readEnvelope() + val restored: Envelope = Global.io.readEnvelopeFile(tmpPath) assertTrue { envelope.contentEquals(restored) } } } \ No newline at end of file diff --git a/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/tcp/EnvelopeServerTest.kt b/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/tcp/EnvelopeServerTest.kt index 38ff434d..17512298 100644 --- a/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/tcp/EnvelopeServerTest.kt +++ b/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/tcp/EnvelopeServerTest.kt @@ -17,7 +17,7 @@ import kotlin.time.ExperimentalTime @ExperimentalStdlibApi object EchoResponder : Responder { override suspend fun respond(request: Envelope): Envelope { - val string = TaggedEnvelopeFormat.run { writeBytes(request).decodeToString() } + val string = TaggedEnvelopeFormat().run { writeBytes(request).decodeToString() } println("ECHO:") println(string) return request diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Meta.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Meta.kt index 04c5f27b..6b7f85e9 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Meta.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Meta.kt @@ -59,6 +59,8 @@ interface Meta : MetaRepr { * A key for single value node */ const val VALUE_KEY = "@value" + + val empty: EmptyMeta = EmptyMeta } } @@ -183,15 +185,15 @@ operator fun > MetaNode?.get(key: NameToken): MetaItem? = /** * Equals, hashcode and to string for any meta */ -abstract class MetaBase: Meta{ +abstract class MetaBase : Meta { - override fun equals(other: Any?): Boolean = if(other is Meta) { + override fun equals(other: Any?): Boolean = if (other is Meta) { this.items == other.items } else { false } - override fun hashCode(): Int = items.hashCode() + override fun hashCode(): Int = items.hashCode() override fun toString(): String = items.toString() } @@ -228,7 +230,7 @@ object EmptyMeta : MetaBase() { * Unsafe methods to access values and nodes directly from [MetaItem] */ -val MetaItem<*>?.value +val MetaItem<*>?.value: Value? get() = (this as? ValueItem)?.value ?: (this?.node?.get(VALUE_KEY) as? ValueItem)?.value @@ -252,7 +254,7 @@ val MetaItem<*>?.stringList get() = value?.list?.map { it.string } ?: emptyList( val MetaItem?.node: M? get() = when (this) { null -> null - is ValueItem -> error("Trying to interpret value meta item as node item") + is ValueItem -> null//error("Trying to interpret value meta item as node item") is NodeItem -> node } diff --git a/dataforge-output/src/commonMain/kotlin/hep/dataforge/output/OutputManager.kt b/dataforge-output/src/commonMain/kotlin/hep/dataforge/output/OutputManager.kt index 448b3adb..d9f7c1b2 100644 --- a/dataforge-output/src/commonMain/kotlin/hep/dataforge/output/OutputManager.kt +++ b/dataforge-output/src/commonMain/kotlin/hep/dataforge/output/OutputManager.kt @@ -70,7 +70,7 @@ class ConsoleOutputManager : AbstractPlugin(), OutputManager { override val type = ConsoleOutputManager::class - override fun invoke(meta:Meta) = ConsoleOutputManager() + override fun invoke(meta: Meta, context: Context) = ConsoleOutputManager() } } diff --git a/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/fileData.kt b/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/fileData.kt index 08959298..cd6aafea 100644 --- a/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/fileData.kt +++ b/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/fileData.kt @@ -11,83 +11,101 @@ import hep.dataforge.meta.Meta import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.withContext import kotlinx.io.nio.asInput +import kotlinx.io.nio.asOutput import java.nio.file.Files import java.nio.file.Path import java.nio.file.StandardOpenOption import kotlin.reflect.KClass /** - * Read meta from file in a given [format] + * Read meta from file in a given [MetaFormat] */ -fun Path.readMeta(format: MetaFormat, descriptor: NodeDescriptor? = null): Meta { - return format.run { - Files.newByteChannel(this@readMeta, StandardOpenOption.READ) - .asInput() - .readMeta(descriptor) - } +fun MetaFormat.readMetaFile(path: Path, descriptor: NodeDescriptor? = null): Meta { + return Files.newByteChannel(path, StandardOpenOption.READ) + .asInput() + .readMeta(descriptor) +} + +/** + * Write meta to file using given [MetaFormat] + */ +fun MetaFormat.writeMetaFile(path: Path, meta: Meta, descriptor: NodeDescriptor? = null) { + return Files.newByteChannel(path, StandardOpenOption.WRITE, StandardOpenOption.CREATE_NEW) + .asOutput() + .writeMeta(meta, descriptor) } /** * Read data with supported envelope format and binary format. If envelope format is null, then read binary directly from file. * The operation is blocking since it must read meta header. The reading of envelope body is lazy * @param type explicit type of data read - * @param format binary format - * @param envelopeFormat the format of envelope. If null, file is read directly + * @param dataFormat binary format + * @param envelopeFormatFactory the format of envelope. If null, file is read directly * @param metaFile the relative file for optional meta override * @param metaFileFormat the meta format for override */ -fun Path.readData( +fun IOPlugin.readData( + path: Path, type: KClass, - format: IOFormat, - envelopeFormat: EnvelopeFormat? = null, - metaFile: Path = resolveSibling("$fileName.meta"), - metaFileFormat: MetaFormat = JsonMetaFormat + dataFormat: IOFormat, + envelopeFormatFactory: EnvelopeFormatFactory? = null, + metaFile: Path = path.resolveSibling("${path.fileName}.meta"), + metaFileFormat: MetaFormat = JsonMetaFormat.default ): Data { val externalMeta = if (Files.exists(metaFile)) { - metaFile.readMeta(metaFileFormat) + metaFileFormat.readMetaFile(metaFile) } else { null } - return if (envelopeFormat == null) { + return if (envelopeFormatFactory == null) { Data(type, externalMeta ?: EmptyMeta) { withContext(Dispatchers.IO) { - format.run { - Files.newByteChannel(this@readData, StandardOpenOption.READ) + dataFormat.run { + Files.newByteChannel(path, StandardOpenOption.READ) .asInput() .readThis() } } } } else { - readEnvelope(envelopeFormat).let { + readEnvelopeFile(path, envelopeFormatFactory).let { if (externalMeta == null) { it } else { it.withMetaLayers(externalMeta) } - }.toData(type, format) + }.toData(type, dataFormat) } } -fun DataTreeBuilder.file(path: Path, format: IOFormat, envelopeFormat: EnvelopeFormat? = null) { - val data = path.readData(type, format, envelopeFormat) - val name = path.fileName.toString().replace(".df", "") - datum(name, data) +//TODO wants multi-receiver +fun DataTreeBuilder.file( + plugin: IOPlugin, + path: Path, + dataFormat: IOFormat, + envelopeFormatFactory: EnvelopeFormatFactory? = null +) { + plugin.run { + val data = readData(path, type, dataFormat, envelopeFormatFactory) + val name = path.fileName.toString().replace(".df", "") + datum(name, data) + } } /** * Read the directory as a data node */ -fun Path.readDataNode( +fun IOPlugin.readDataNode( + path: Path, type: KClass, - format: IOFormat, - envelopeFormat: EnvelopeFormat? = null + dataFormat: IOFormat, + envelopeFormatFactory: EnvelopeFormatFactory? = null ): DataNode { - if (!Files.isDirectory(this)) error("Provided path $this is not a directory") + if (!Files.isDirectory(path)) error("Provided path $this is not a directory") return DataNode(type) { - Files.list(this@readDataNode).forEach { path -> + Files.list(path).forEach { path -> if (!path.fileName.toString().endsWith(".meta")) { - file(path, format, envelopeFormat) + file(this@readDataNode,path, dataFormat, envelopeFormatFactory) } } } From bf787fb5fc76f4a0222bf3c961018e61f2031a2a Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Sat, 19 Oct 2019 14:36:13 +0300 Subject: [PATCH 25/48] Added peek format functionality to EnvelopeFormat --- .../kotlin/hep/dataforge/io/EnvelopeFormat.kt | 6 ++ .../hep/dataforge/io/TaggedEnvelopeFormat.kt | 60 +++++++++++-------- 2 files changed, 40 insertions(+), 26 deletions(-) diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/EnvelopeFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/EnvelopeFormat.kt index 3fd4e96a..ef6326f0 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/EnvelopeFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/EnvelopeFormat.kt @@ -32,6 +32,12 @@ interface EnvelopeFormatFactory : IOFormatFactory { override fun invoke(meta: Meta, context: Context): EnvelopeFormat + /** + * Try to infer specific format from input and return null if the attempt is failed. + * This method does **not** return Input into initial state. + */ + fun peekFormat(io: IOPlugin, input: Input): EnvelopeFormat? + companion object { const val ENVELOPE_FORMAT_TYPE = "io.format.envelope" } diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt index 480d08e3..f8a2fbb8 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt @@ -10,20 +10,14 @@ import hep.dataforge.names.toName import kotlinx.io.core.* @ExperimentalUnsignedTypes -class TaggedEnvelopeFormat(val io: IOPlugin, meta: Meta) : EnvelopeFormat { - - private val metaFormat: MetaFormat - +class TaggedEnvelopeFormat( + val io: IOPlugin, private val metaFormatKey: Short +) : EnvelopeFormat { - init { - val metaName = meta["name"].string?.toName() ?: JsonMetaFormat.name - val metaFormatFactory = io.metaFormatFactories.find { it.name == metaName } - ?: error("Meta format could not be resolved") + private val metaFormat = + io.metaFormat(metaFormatKey) ?: error("Meta format with key $metaFormatKey could not be resolved in $io") - metaFormat = metaFormatFactory(meta, io.context) - metaFormatKey = metaFormatFactory.key - } private fun Tag.toBytes(): ByteReadPacket = buildPacket(24) { writeText(START_SEQUENCE) @@ -34,19 +28,6 @@ class TaggedEnvelopeFormat(val io: IOPlugin, meta: Meta) : EnvelopeFormat { writeText(END_SEQUENCE) } - private fun Input.readTag(): Tag { - val start = readTextExactBytes(2) - if (start != START_SEQUENCE) error("The input is not an envelope") - val version = readTextExactBytes(4) - if (version != VERSION) error("Wrong version of DataForge: expected $VERSION but found $version") - val metaFormatKey = readShort() - val metaLength = readUInt() - val dataLength = readULong() - val end = readTextExactBytes(4) - if (end != END_SEQUENCE) error("The input is not an envelope") - return Tag(metaFormatKey, metaLength, dataLength) - } - override fun Output.writeThis(obj: Envelope) { val metaBytes = metaFormat.writeBytes(obj.meta) val tag = Tag(metaFormatKey, metaBytes.size.toUInt(), obj.data?.size ?: 0.toULong()) @@ -101,8 +82,35 @@ class TaggedEnvelopeFormat(val io: IOPlugin, meta: Meta) : EnvelopeFormat { override val name: Name = super.name + VERSION override fun invoke(meta: Meta, context: Context): EnvelopeFormat { - val plugin = context.plugins.fetch(IOPlugin) - return TaggedEnvelopeFormat(plugin, meta) + val io = context.io + + val metaFormatName = meta["name"].string?.toName() ?: JsonMetaFormat.name + val metaFormatFactory = io.metaFormatFactories.find { it.name == metaFormatName } + ?: error("Meta format could not be resolved") + + return TaggedEnvelopeFormat(io, metaFormatFactory.key) + } + + private fun Input.readTag(): Tag { + val start = readTextExactBytes(2) + if (start != START_SEQUENCE) error("The input is not an envelope") + val version = readTextExactBytes(4) + if (version != VERSION) error("Wrong version of DataForge: expected $VERSION but found $version") + val metaFormatKey = readShort() + val metaLength = readUInt() + val dataLength = readULong() + val end = readTextExactBytes(4) + if (end != END_SEQUENCE) error("The input is not an envelope") + return Tag(metaFormatKey, metaLength, dataLength) + } + + override fun peekFormat(io: IOPlugin, input: Input): EnvelopeFormat? { + return try { + val tag = input.readTag() + TaggedEnvelopeFormat(io, tag.metaFormatKey) + } catch (ex: Exception) { + null + } } val default = invoke() From e686501662ab8493ff428f1c53e29a4143a5e7f2 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Sun, 20 Oct 2019 17:57:43 +0300 Subject: [PATCH 26/48] TaglessEnvelopeFormat implementation --- .../kotlin/hep/dataforge/io/Binary.kt | 2 + .../kotlin/hep/dataforge/io/IOPlugin.kt | 1 + .../hep/dataforge/io/TaggedEnvelopeFormat.kt | 6 +- .../hep/dataforge/io/TaglessEnvelopeFormat.kt | 254 ++++++++++++++++++ .../hep/dataforge/io/EnvelopeFormatTest.kt | 16 +- .../dataforge/io/tcp/EnvelopeServerTest.kt | 2 - 6 files changed, 275 insertions(+), 6 deletions(-) create mode 100644 dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaglessEnvelopeFormat.kt diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Binary.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Binary.kt index 429ab185..a20f6fac 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Binary.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Binary.kt @@ -69,6 +69,8 @@ inline class ArrayBinary(val array: ByteArray) : RandomAccessBinary { } } +fun ByteArray.asBinary() = ArrayBinary(this) + /** * Read given binary as object using given format */ diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOPlugin.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOPlugin.kt index 0293daf2..7e61924f 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOPlugin.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOPlugin.kt @@ -52,6 +52,7 @@ class IOPlugin(meta: Meta) : AbstractPlugin(meta) { val defaultEnvelopeFormats = listOf(TaggedEnvelopeFormat) override val tag: PluginTag = PluginTag("io", group = PluginTag.DATAFORGE_GROUP) + override val type: KClass = IOPlugin::class override fun invoke(meta: Meta, context: Context): IOPlugin = IOPlugin(meta) } diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt index f8a2fbb8..26239601 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt @@ -15,8 +15,8 @@ class TaggedEnvelopeFormat( private val metaFormatKey: Short ) : EnvelopeFormat { - private val metaFormat = - io.metaFormat(metaFormatKey) ?: error("Meta format with key $metaFormatKey could not be resolved in $io") + private val metaFormat = io.metaFormat(metaFormatKey) + ?: error("Meta format with key $metaFormatKey could not be resolved in $io") private fun Tag.toBytes(): ByteReadPacket = buildPacket(24) { @@ -113,7 +113,7 @@ class TaggedEnvelopeFormat( } } - val default = invoke() + val default by lazy { invoke()} } } \ No newline at end of file diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaglessEnvelopeFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaglessEnvelopeFormat.kt new file mode 100644 index 00000000..4bdeca78 --- /dev/null +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaglessEnvelopeFormat.kt @@ -0,0 +1,254 @@ +package hep.dataforge.io + +import hep.dataforge.context.Context +import hep.dataforge.meta.* +import hep.dataforge.names.asName +import kotlinx.io.core.* + +class TaglessEnvelopeFormat( + val io: IOPlugin, + val metaType: String = JsonMetaFormat.name.toString(), + meta: Meta = EmptyMeta +) : EnvelopeFormat { + + val metaFormat = io.metaFormat(metaType, meta) + ?: error("Meta format with type $metaType could not be resolved in $io") + + private fun Output.writeProperty(key: String, value: Any) { + writeText("#? $key: $value;\r\n") + } + + override fun Output.writeThis(obj: Envelope) { + + //printing header + writeText(TAGLESS_ENVELOPE_HEADER + "\r\n") + + //printing all properties + writeProperty(META_TYPE_PROPERTY, metaType) + //TODO add optional metaFormat properties + writeProperty(DATA_LENGTH_PROPERTY, obj.data?.size ?: 0) + + //Printing meta + if (!obj.meta.isEmpty()) { + val metaBytes = metaFormat.writeBytes(obj.meta) + writeProperty(META_LENGTH_PROPERTY, metaBytes.size) + writeText(DEFAULT_META_START + "\r\n") + writeFully(metaBytes) + writeText("\r\n") + } + + //Printing data + obj.data?.let { data -> + writeText(DEFAULT_DATA_START + "\r\n") + writeFully(data.toBytes()) + } + } + + override fun Input.readThis(): Envelope { + var line: String = "" + do { + line = readUTF8Line() ?: error("Input does not contain tagless envelope header") + } while (!line.startsWith(TAGLESS_ENVELOPE_HEADER)) + val properties = HashMap() + + line = "" + while (line.isBlank() || line.startsWith("#?")) { + if (line.startsWith("#?")) { + val match = propertyPattern.find(line) + ?: error("Line $line does not match property declaration pattern") + val (key, value) = match.destructured + properties[key] = value + } + line = readUTF8Line() ?: return SimpleEnvelope(Meta.empty, null) + } + + var meta: Meta = EmptyMeta + + if (line.startsWith(DEFAULT_META_START)) { + val metaFormat = properties[META_TYPE_PROPERTY]?.let { io.metaFormat(it) } ?: JsonMetaFormat.default + meta = if (properties.containsKey(META_LENGTH_PROPERTY)) { + val bytes = ByteArray(properties[META_LENGTH_PROPERTY]!!.toInt()) + readFully(bytes) + metaFormat.readBytes(bytes) + } else { + metaFormat.run { + readThis() + } + } + } + + do { + line = readUTF8Line() ?: return SimpleEnvelope(meta, null) + //returning an Envelope without data if end of input is reached + } while (!line.startsWith(DEFAULT_DATA_START)) + + val data: Binary? = if (properties.containsKey(DATA_LENGTH_PROPERTY)) { + val bytes = ByteArray(properties[DATA_LENGTH_PROPERTY]!!.toInt()) + readFully(bytes) + bytes.asBinary() + } else { + val bytes = readBytes() + bytes.asBinary() + } + + return SimpleEnvelope(meta, data) + } + + override fun Input.readPartial(): PartialEnvelope { + TODO("not implemented") //To change body of created functions use File | Settings | File Templates. + } + +// class TaglessReader(private val override: Map) : EnvelopeReader { +// +// private val BUFFER_SIZE = 1024 +// +// @Throws(IOException::class) +// override fun read(stream: InputStream): Envelope { +// return read(Channels.newChannel(stream)) +// } +// +// override fun read(channel: ReadableByteChannel): Envelope { +// val properties = HashMap(override) +// val buffer = ByteBuffer.allocate(BUFFER_SIZE).apply { position(BUFFER_SIZE) } +// val meta = readMeta(channel, buffer, properties) +// return LazyEnvelope(meta) { BufferedBinary(readData(channel, buffer, properties)) } +// } +// +// +// /** +// * Read lines using provided channel and buffer. Buffer state is changed by this operation +// */ +// private fun readLines(channel: ReadableByteChannel, buffer: ByteBuffer): Sequence { +// return sequence { +// val builder = ByteArrayOutputStream() +// while (true) { +// if (!buffer.hasRemaining()) { +// if (!channel.isOpen) { +// break +// } +// buffer.flip() +// val count = channel.read(buffer) +// buffer.flip() +// if (count < BUFFER_SIZE) { +// channel.close() +// } +// } +// val b = buffer.get() +// builder.write(b.toInt()) +// if (b == '\n'.toByte()) { +// yield(String(builder.toByteArray(), Charsets.UTF_8)) +// builder.reset() +// } +// } +// } +// } +// +// @Throws(IOException::class) +// private fun readMeta( +// channel: ReadableByteChannel, +// buffer: ByteBuffer, +// properties: MutableMap +// ): Meta { +// val sb = StringBuilder() +// val metaEnd = properties.getOrDefault(DATA_START_PROPERTY, DEFAULT_DATA_START) +// readLines(channel, buffer).takeWhile { it.trim { it <= ' ' } != metaEnd }.forEach { line -> +// if (line.startsWith("#?")) { +// readProperty(line.trim(), properties) +// } else if (line.isEmpty() || line.startsWith("#~")) { +// //Ignore headings, do nothing +// } else { +// sb.append(line).append("\r\n") +// } +// } +// +// +// return if (sb.isEmpty()) { +// Meta.empty() +// } else { +// val metaType = MetaType.resolve(properties) +// try { +// metaType.reader.readString(sb.toString()) +// } catch (e: ParseException) { +// throw RuntimeException("Failed to parse meta", e) +// } +// +// } +// } +// +// +// @Throws(IOException::class) +// private fun readData( +// channel: ReadableByteChannel, +// buffer: ByteBuffer, +// properties: Map +// ): ByteBuffer { +// val array = ByteArray(buffer.remaining()); +// buffer.get(array) +// if (properties.containsKey(DATA_LENGTH_PROPERTY)) { +// val result = ByteBuffer.allocate(Integer.parseInt(properties[DATA_LENGTH_PROPERTY])) +// result.put(array)//TODO fix it to not use direct array access +// channel.read(result) +// return result +// } else { +// val baos = ByteArrayOutputStream() +// baos.write(array) +// while (channel.isOpen) { +// val read = channel.read(buffer) +// buffer.flip() +// if (read < BUFFER_SIZE) { +// channel.close() +// } +// +// baos.write(buffer.array()) +// } +// val remainingArray: ByteArray = ByteArray(buffer.remaining()) +// buffer.get(remainingArray) +// baos.write(remainingArray) +// return ByteBuffer.wrap(baos.toByteArray()) +// } +// } +// } + + companion object : EnvelopeFormatFactory { + + private val propertyPattern = "#\\?\\s*(?[\\w.]*)\\s*:\\s*(?[^;]*);?".toRegex() + + const val META_TYPE_PROPERTY = "metaType" + const val META_LENGTH_PROPERTY = "metaLength" + const val DATA_LENGTH_PROPERTY = "dataLength" + + + const val TAGLESS_ENVELOPE_TYPE = "tagless" + + const val TAGLESS_ENVELOPE_HEADER = "#~DFTL~#" + const val META_START_PROPERTY = "metaSeparator" + const val DEFAULT_META_START = "#~META~#" + const val DATA_START_PROPERTY = "dataSeparator" + const val DEFAULT_DATA_START = "#~DATA~#" + + const val code: Int = 0x4446544c //DFTL + + override val name = TAGLESS_ENVELOPE_TYPE.asName() + + override fun invoke(meta: Meta, context: Context): EnvelopeFormat { + val metaFormatName: String = meta["name"].string ?: JsonMetaFormat.name.toString() + return TaglessEnvelopeFormat(context.io, metaFormatName, meta) + } + + val default by lazy { invoke() } + + override fun peekFormat(io: IOPlugin, input: Input): EnvelopeFormat? { + return try { + val buffer = ByteArray(TAGLESS_ENVELOPE_HEADER.length) + input.readFully(buffer) + return if (buffer.toString() == TAGLESS_ENVELOPE_HEADER) { + TaglessEnvelopeFormat(io) + } else { + null + } + } catch (ex: Exception) { + null + } + } + } +} \ No newline at end of file diff --git a/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/EnvelopeFormatTest.kt b/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/EnvelopeFormatTest.kt index 3acb852c..fca95bbb 100644 --- a/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/EnvelopeFormatTest.kt +++ b/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/EnvelopeFormatTest.kt @@ -18,7 +18,21 @@ class EnvelopeFormatTest { @ExperimentalStdlibApi @Test fun testTaggedFormat(){ - TaggedEnvelopeFormat().run { + TaggedEnvelopeFormat.default.run { + val bytes = writeBytes(envelope) + println(bytes.decodeToString()) + val res = readBytes(bytes) + assertEquals(envelope.meta,res.meta) + val double = res.data?.read { + readDouble() + } + assertEquals(22.2, double) + } + } + + @Test + fun testTaglessFormat(){ + TaglessEnvelopeFormat.default.run { val bytes = writeBytes(envelope) println(bytes.decodeToString()) val res = readBytes(bytes) diff --git a/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/tcp/EnvelopeServerTest.kt b/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/tcp/EnvelopeServerTest.kt index 17512298..400d3b54 100644 --- a/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/tcp/EnvelopeServerTest.kt +++ b/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/tcp/EnvelopeServerTest.kt @@ -10,7 +10,6 @@ import kotlinx.coroutines.runBlocking import org.junit.AfterClass import org.junit.BeforeClass import org.junit.Test -import kotlin.test.Ignore import kotlin.test.assertEquals import kotlin.time.ExperimentalTime @@ -45,7 +44,6 @@ class EnvelopeServerTest { } @Test - @Ignore fun doEchoTest() { val request = Envelope.invoke { From 87ae41886b01ab73fbd0846930d96ab59aa1d607 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Sun, 20 Oct 2019 19:30:51 +0300 Subject: [PATCH 27/48] Partial TaglessEnvelopeFormat --- .../hep/dataforge/io/TaglessEnvelopeFormat.kt | 166 ++++++------------ 1 file changed, 50 insertions(+), 116 deletions(-) diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaglessEnvelopeFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaglessEnvelopeFormat.kt index 4bdeca78..af01acc4 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaglessEnvelopeFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaglessEnvelopeFormat.kt @@ -4,6 +4,7 @@ import hep.dataforge.context.Context import hep.dataforge.meta.* import hep.dataforge.names.asName import kotlinx.io.core.* +import kotlinx.serialization.toUtf8Bytes class TaglessEnvelopeFormat( val io: IOPlugin, @@ -11,6 +12,9 @@ class TaglessEnvelopeFormat( meta: Meta = EmptyMeta ) : EnvelopeFormat { + private val metaStart = meta[META_START_PROPERTY].string ?: DEFAULT_META_START + private val dataStart = meta[DATA_START_PROPERTY].string ?: DEFAULT_DATA_START + val metaFormat = io.metaFormat(metaType, meta) ?: error("Meta format with type $metaType could not be resolved in $io") @@ -32,14 +36,14 @@ class TaglessEnvelopeFormat( if (!obj.meta.isEmpty()) { val metaBytes = metaFormat.writeBytes(obj.meta) writeProperty(META_LENGTH_PROPERTY, metaBytes.size) - writeText(DEFAULT_META_START + "\r\n") + writeText(metaStart + "\r\n") writeFully(metaBytes) writeText("\r\n") } //Printing data obj.data?.let { data -> - writeText(DEFAULT_DATA_START + "\r\n") + writeText(dataStart + "\r\n") writeFully(data.toBytes()) } } @@ -64,7 +68,7 @@ class TaglessEnvelopeFormat( var meta: Meta = EmptyMeta - if (line.startsWith(DEFAULT_META_START)) { + if (line.startsWith(metaStart)) { val metaFormat = properties[META_TYPE_PROPERTY]?.let { io.metaFormat(it) } ?: JsonMetaFormat.default meta = if (properties.containsKey(META_LENGTH_PROPERTY)) { val bytes = ByteArray(properties[META_LENGTH_PROPERTY]!!.toInt()) @@ -80,7 +84,7 @@ class TaglessEnvelopeFormat( do { line = readUTF8Line() ?: return SimpleEnvelope(meta, null) //returning an Envelope without data if end of input is reached - } while (!line.startsWith(DEFAULT_DATA_START)) + } while (!line.startsWith(dataStart)) val data: Binary? = if (properties.containsKey(DATA_LENGTH_PROPERTY)) { val bytes = ByteArray(properties[DATA_LENGTH_PROPERTY]!!.toInt()) @@ -95,119 +99,49 @@ class TaglessEnvelopeFormat( } override fun Input.readPartial(): PartialEnvelope { - TODO("not implemented") //To change body of created functions use File | Settings | File Templates. - } + var offset = 0u + var line: String = "" + do { + line = readUTF8Line() ?: error("Input does not contain tagless envelope header") + offset += line.toUtf8Bytes().size.toUInt() + } while (!line.startsWith(TAGLESS_ENVELOPE_HEADER)) + val properties = HashMap() -// class TaglessReader(private val override: Map) : EnvelopeReader { -// -// private val BUFFER_SIZE = 1024 -// -// @Throws(IOException::class) -// override fun read(stream: InputStream): Envelope { -// return read(Channels.newChannel(stream)) -// } -// -// override fun read(channel: ReadableByteChannel): Envelope { -// val properties = HashMap(override) -// val buffer = ByteBuffer.allocate(BUFFER_SIZE).apply { position(BUFFER_SIZE) } -// val meta = readMeta(channel, buffer, properties) -// return LazyEnvelope(meta) { BufferedBinary(readData(channel, buffer, properties)) } -// } -// -// -// /** -// * Read lines using provided channel and buffer. Buffer state is changed by this operation -// */ -// private fun readLines(channel: ReadableByteChannel, buffer: ByteBuffer): Sequence { -// return sequence { -// val builder = ByteArrayOutputStream() -// while (true) { -// if (!buffer.hasRemaining()) { -// if (!channel.isOpen) { -// break -// } -// buffer.flip() -// val count = channel.read(buffer) -// buffer.flip() -// if (count < BUFFER_SIZE) { -// channel.close() -// } -// } -// val b = buffer.get() -// builder.write(b.toInt()) -// if (b == '\n'.toByte()) { -// yield(String(builder.toByteArray(), Charsets.UTF_8)) -// builder.reset() -// } -// } -// } -// } -// -// @Throws(IOException::class) -// private fun readMeta( -// channel: ReadableByteChannel, -// buffer: ByteBuffer, -// properties: MutableMap -// ): Meta { -// val sb = StringBuilder() -// val metaEnd = properties.getOrDefault(DATA_START_PROPERTY, DEFAULT_DATA_START) -// readLines(channel, buffer).takeWhile { it.trim { it <= ' ' } != metaEnd }.forEach { line -> -// if (line.startsWith("#?")) { -// readProperty(line.trim(), properties) -// } else if (line.isEmpty() || line.startsWith("#~")) { -// //Ignore headings, do nothing -// } else { -// sb.append(line).append("\r\n") -// } -// } -// -// -// return if (sb.isEmpty()) { -// Meta.empty() -// } else { -// val metaType = MetaType.resolve(properties) -// try { -// metaType.reader.readString(sb.toString()) -// } catch (e: ParseException) { -// throw RuntimeException("Failed to parse meta", e) -// } -// -// } -// } -// -// -// @Throws(IOException::class) -// private fun readData( -// channel: ReadableByteChannel, -// buffer: ByteBuffer, -// properties: Map -// ): ByteBuffer { -// val array = ByteArray(buffer.remaining()); -// buffer.get(array) -// if (properties.containsKey(DATA_LENGTH_PROPERTY)) { -// val result = ByteBuffer.allocate(Integer.parseInt(properties[DATA_LENGTH_PROPERTY])) -// result.put(array)//TODO fix it to not use direct array access -// channel.read(result) -// return result -// } else { -// val baos = ByteArrayOutputStream() -// baos.write(array) -// while (channel.isOpen) { -// val read = channel.read(buffer) -// buffer.flip() -// if (read < BUFFER_SIZE) { -// channel.close() -// } -// -// baos.write(buffer.array()) -// } -// val remainingArray: ByteArray = ByteArray(buffer.remaining()) -// buffer.get(remainingArray) -// baos.write(remainingArray) -// return ByteBuffer.wrap(baos.toByteArray()) -// } -// } -// } + line = "" + while (line.isBlank() || line.startsWith("#?")) { + if (line.startsWith("#?")) { + val match = propertyPattern.find(line) + ?: error("Line $line does not match property declaration pattern") + val (key, value) = match.destructured + properties[key] = value + } + line = readUTF8Line() ?: return PartialEnvelope(Meta.empty, offset.toUInt(), 0.toULong()) + offset += line.toUtf8Bytes().size.toUInt() + } + + var meta: Meta = EmptyMeta + + if (line.startsWith(metaStart)) { + val metaFormat = properties[META_TYPE_PROPERTY]?.let { io.metaFormat(it) } ?: JsonMetaFormat.default + meta = if (properties.containsKey(META_LENGTH_PROPERTY)) { + val bytes = ByteArray(properties[META_LENGTH_PROPERTY]!!.toInt()) + readFully(bytes) + offset += bytes.size.toUInt() + metaFormat.readBytes(bytes) + } else { + error("Can't partially read an envelope with undefined meta size") + } + } + + do { + line = readUTF8Line() ?: return PartialEnvelope(Meta.empty, offset.toUInt(), 0.toULong()) + offset += line.toUtf8Bytes().size.toUInt() + //returning an Envelope without data if end of input is reached + } while (!line.startsWith(dataStart)) + + val dataSize = properties[DATA_LENGTH_PROPERTY]?.toULong() + return PartialEnvelope(meta, offset, dataSize) + } companion object : EnvelopeFormatFactory { From 26f5114e48eea103a3c730e9cd3390f8d4dbd516 Mon Sep 17 00:00:00 2001 From: Mikhail Zelenyy Date: Mon, 21 Oct 2019 16:26:47 +0300 Subject: [PATCH 28/48] Simple test for selecting data --- .../workspace/DataPropagationTest.kt | 95 +++++++++++++++++++ 1 file changed, 95 insertions(+) create mode 100644 dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/DataPropagationTest.kt diff --git a/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/DataPropagationTest.kt b/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/DataPropagationTest.kt new file mode 100644 index 00000000..c449ffc3 --- /dev/null +++ b/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/DataPropagationTest.kt @@ -0,0 +1,95 @@ +package hep.dataforge.workspace + +import hep.dataforge.context.Context +import hep.dataforge.context.PluginFactory +import hep.dataforge.context.PluginTag +import hep.dataforge.data.* +import hep.dataforge.meta.Meta +import hep.dataforge.names.asName +import org.junit.Test +import kotlin.reflect.KClass +import kotlin.test.assertEquals + + +class DataPropagationTestPlugin : WorkspacePlugin() { + override val tag: PluginTag = Companion.tag + + val testAllData = task("allData", Int::class) { + model { + allData() + } + transform { data -> + return@transform DataNode { + val result = data.dataSequence().map { it.second.get() }.reduce { acc, pair -> acc + pair } + set("result".asName(), Data { result }) + } + } + } + + + val testSingleData = task("singleData", Int::class) { + model { + data("myData\\[12\\]") + } + transform { data -> + return@transform DataNode { + val result = data.dataSequence().map { it.second.get() }.reduce { acc, pair -> acc + pair } + set("result".asName(), Data { result }) + } + } + } + + val testAllRegexData = task("allRegexData", Int::class) { + model { + data(pattern = "myData.*") + } + transform { data -> + return@transform DataNode { + val result = data.dataSequence().map { it.second.get() }.reduce { acc, pair -> acc + pair } + set("result".asName(), Data { result }) + } + } + } + + + companion object : PluginFactory { + + override val type: KClass = DataPropagationTestPlugin::class + + override fun invoke(meta: Meta, context: Context): DataPropagationTestPlugin = + DataPropagationTestPlugin(meta) + + override val tag: PluginTag = PluginTag("Test") + } +} + +class DataPropagationTest { + val testWorkspace = Workspace { + context { + plugin(DataPropagationTestPlugin()) + } + data { + repeat(100) { + static("myData[$it]", it) + } + } + } + + @Test + fun testAllData() { + val node = testWorkspace.run("Test.allData") + assertEquals(4950, node.first()!!.get()) + } + + @Test + fun testAllRegexData() { + val node = testWorkspace.run("Test.allRegexData") + assertEquals(4950, node.first()!!.get()) + } + + @Test + fun testSingleData() { + val node = testWorkspace.run("Test.singleData") + assertEquals(12, node.first()!!.get()) + } +} \ No newline at end of file From 956c07e386dc6285697e5de4898365f96d3ed24d Mon Sep 17 00:00:00 2001 From: Mikhail Zelenyy Date: Tue, 22 Oct 2019 14:24:22 +0300 Subject: [PATCH 29/48] Fixed default regex in DataFilter --- .../src/commonMain/kotlin/hep/dataforge/data/DataFilter.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataFilter.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataFilter.kt index 987d8412..a55aac9d 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataFilter.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataFilter.kt @@ -16,7 +16,7 @@ class DataFilter(override val config: Config) : Specific { /** * A regular expression pattern for the filter */ - var pattern by string("*.") + var pattern by string(".*") // val prefix by string() // val suffix by string() From d3dd15884c58cf2f0d0c58276cfff572175a20c9 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Sun, 27 Oct 2019 16:50:05 +0300 Subject: [PATCH 30/48] Fixed all IO issues (for now) --- build.gradle.kts | 3 +- .../dataforge-io-yaml/build.gradle.kts | 9 ++ .../kotlin/hep/dataforge/io/IOFormat.kt | 101 ++++++++++++- .../kotlin/hep/dataforge/io/JsonMetaFormat.kt | 20 +-- .../kotlin/hep/dataforge/io/MetaFormat.kt | 2 +- .../kotlin/hep/dataforge/io/MetaSerializer.kt | 67 --------- .../hep/dataforge/io/TaglessEnvelopeFormat.kt | 19 +-- .../io/serialization/MetaSerializer.kt | 135 ++++++++++++++++++ .../io/serialization/nameSerializers.kt | 33 +++++ .../io/serialization/serializationUtils.kt | 32 +++++ .../kotlin/hep/dataforge/io/MetaFormatTest.kt | 2 - .../hep/dataforge/io/MetaSerializerTest.kt | 21 +++ .../dataforge/io/tcp/InputStreamAsInput.kt | 1 - .../hep/dataforge/io/FileEnvelopeTest.kt | 2 - .../dataforge/io/tcp/EnvelopeServerTest.kt | 1 - ...{valueEstensions.kt => valueExtensions.kt} | 0 settings.gradle.kts | 3 +- 17 files changed, 349 insertions(+), 102 deletions(-) create mode 100644 dataforge-io/dataforge-io-yaml/build.gradle.kts delete mode 100644 dataforge-io/src/commonMain/kotlin/hep/dataforge/io/MetaSerializer.kt create mode 100644 dataforge-io/src/commonMain/kotlin/hep/dataforge/io/serialization/MetaSerializer.kt create mode 100644 dataforge-io/src/commonMain/kotlin/hep/dataforge/io/serialization/nameSerializers.kt create mode 100644 dataforge-io/src/commonMain/kotlin/hep/dataforge/io/serialization/serializationUtils.kt rename dataforge-meta/src/commonMain/kotlin/hep/dataforge/values/{valueEstensions.kt => valueExtensions.kt} (100%) diff --git a/build.gradle.kts b/build.gradle.kts index c5d5c73f..eb287fa1 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -1,9 +1,10 @@ plugins { id("scientifik.mpp") version "0.2.1" apply false + id("scientifik.jvm") version "0.2.1" apply false id("scientifik.publish") version "0.2.1" apply false } -val dataforgeVersion by extra("0.1.4-dev-6") +val dataforgeVersion by extra("0.1.4-dev-8") val bintrayRepo by extra("dataforge") val githubProject by extra("dataforge-core") diff --git a/dataforge-io/dataforge-io-yaml/build.gradle.kts b/dataforge-io/dataforge-io-yaml/build.gradle.kts new file mode 100644 index 00000000..1a4af603 --- /dev/null +++ b/dataforge-io/dataforge-io-yaml/build.gradle.kts @@ -0,0 +1,9 @@ +plugins { + id("scientifik.jvm") +} + +description = "YAML meta IO" + +dependencies{ + api(project(":dataforge-io")) +} diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOFormat.kt index 6c5e1244..3734b82d 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOFormat.kt @@ -11,26 +11,54 @@ import hep.dataforge.names.asName import hep.dataforge.provider.Type import hep.dataforge.values.Value import kotlinx.io.core.* +import kotlinx.io.pool.ObjectPool import kotlinx.serialization.ImplicitReflectionSerializer import kotlinx.serialization.KSerializer import kotlinx.serialization.cbor.Cbor import kotlinx.serialization.serializer +import kotlin.math.min import kotlin.reflect.KClass /** - * And interface for serialization facilities + * And interface for reading and writing objects into with IO streams */ - interface IOFormat { fun Output.writeThis(obj: T) fun Input.readThis(): T - - } -fun IOFormat.writePacket(obj: T): ByteReadPacket = buildPacket { writeThis(obj) } -fun IOFormat.writeBytes(obj: T): ByteArray = buildPacket { writeThis(obj) }.readBytes() -fun IOFormat.readBytes(array: ByteArray): T = ByteReadPacket(array).readThis() +fun Input.readWith(format: IOFormat): T = format.run { readThis() } +fun Output.writeWith(format: IOFormat, obj: T) = format.run { writeThis(obj) } + +class ListIOFormat(val format: IOFormat) : IOFormat> { + override fun Output.writeThis(obj: List) { + writeInt(obj.size) + format.run { + obj.forEach { + writeThis(it) + } + } + } + + override fun Input.readThis(): List { + val size = readInt() + return format.run { + List(size) { readThis() } + } + } +} + +val IOFormat.list get() = ListIOFormat(this) + +fun ObjectPool.fill(block: IoBuffer.() -> Unit): IoBuffer { + val buffer = borrow() + return try { + buffer.apply(block) + } catch (ex: Exception) { + //recycle(buffer) + throw ex + } +} @Type(IO_FORMAT_TYPE) interface IOFormatFactory : Factory>, Named { @@ -44,6 +72,65 @@ interface IOFormatFactory : Factory>, Named { } } +@Deprecated("To be removed in io-2") +inline fun buildPacketWithoutPool(headerSizeHint: Int = 0, block: BytePacketBuilder.() -> Unit): ByteReadPacket { + val builder = BytePacketBuilder(headerSizeHint, IoBuffer.NoPool) + block(builder) + return builder.build() +} + +//@Suppress("EXPERIMENTAL_API_USAGE", "EXPERIMENTAL_OVERRIDE") +//internal fun Input.useAtMost(most: Int, reader: Input.() -> R): R { +// val limitedInput: Input = object : AbstractInput( +// IoBuffer.Pool.borrow(), +// remaining = most.toLong(), +// pool = IoBuffer.Pool +// ) { +// var read = 0 +// override fun closeSource() { +// this@useAtMost.close() +// } +// +// override fun fill(): IoBuffer? { +// if (read >= most) return null +// return IoBuffer.Pool.fill { +// reserveEndGap(IoBuffer.ReservedSize) +// read += this@useAtMost.peekTo(this, max = most - read) +// } +// } +// +// } +// return limitedInput.reader() +//} + +fun IOFormat.writePacket(obj: T): ByteReadPacket = buildPacket { writeThis(obj) } +fun IOFormat.writeBytes(obj: T): ByteArray = buildPacket { writeThis(obj) }.readBytes() +fun IOFormat.readBytes(array: ByteArray): T { + //= ByteReadPacket(array).readThis() + val byteArrayInput: Input = object : AbstractInput( + IoBuffer.Pool.borrow(), + remaining = array.size.toLong(), + pool = IoBuffer.Pool + ) { + var written = 0 + override fun closeSource() { + // do nothing + } + + override fun fill(): IoBuffer? { + if (array.size - written <= 0) return null + + return IoBuffer.Pool.fill { + reserveEndGap(IoBuffer.ReservedSize) + val toWrite = min(capacity, array.size - written) + writeFully(array, written, toWrite) + written += toWrite + } + } + + } + return byteArrayInput.readThis() +} object DoubleIOFormat : IOFormat, IOFormatFactory { override fun invoke(meta: Meta, context: Context): IOFormat = this diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/JsonMetaFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/JsonMetaFormat.kt index 620d1579..a95cdec4 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/JsonMetaFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/JsonMetaFormat.kt @@ -24,7 +24,7 @@ import kotlin.collections.component2 import kotlin.collections.set -class JsonMetaFormat(private val json: Json = Json.plain) : MetaFormat { +class JsonMetaFormat(private val json: Json = Json.indented) : MetaFormat { override fun Output.writeMeta(meta: Meta, descriptor: NodeDescriptor?) { val jsonObject = meta.toJson(descriptor) @@ -34,12 +34,7 @@ class JsonMetaFormat(private val json: Json = Json.plain) : MetaFormat { override fun Input.readMeta(descriptor: NodeDescriptor?): Meta { val str = readText() val jsonElement = json.parseJson(str) - - if (jsonElement is JsonObject) { - return jsonElement.toMeta() - } else { - TODO("Non-object root not supported") - } + return jsonElement.toMeta() } companion object : MetaFormatFactory { @@ -92,7 +87,12 @@ fun Meta.toJson(descriptor: NodeDescriptor? = null): JsonObject { return JsonObject(map) } -fun JsonObject.toMeta(descriptor: NodeDescriptor? = null): JsonMeta = JsonMeta(this, descriptor) +fun JsonElement.toMeta(descriptor: NodeDescriptor? = null): Meta { + return when (val item = toMetaItem(descriptor)) { + is MetaItem.NodeItem<*> -> item.node + is MetaItem.ValueItem ->item.value.toMeta() + } +} fun JsonPrimitive.toValue(descriptor: ValueDescriptor?): Value { return when (this) { @@ -107,7 +107,7 @@ fun JsonElement.toMetaItem(descriptor: ItemDescriptor? = null): MetaItem { - val meta = toMeta(descriptor as? NodeDescriptor) + val meta = JsonMeta(this, descriptor as? NodeDescriptor) MetaItem.NodeItem(meta) } is JsonArray -> { @@ -143,7 +143,7 @@ class JsonMeta(val json: JsonObject, val descriptor: NodeDescriptor? = null) : M this[name] = MetaItem.ValueItem(value.toValue(itemDescriptor as? ValueDescriptor)) as MetaItem } is JsonObject -> { - this[name] = MetaItem.NodeItem(value.toMeta(itemDescriptor as? NodeDescriptor)) + this[name] = MetaItem.NodeItem(JsonMeta(value, itemDescriptor as? NodeDescriptor)) } is JsonArray -> { when { diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/MetaFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/MetaFormat.kt index baa8864e..83f04957 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/MetaFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/MetaFormat.kt @@ -20,7 +20,7 @@ interface MetaFormat : IOFormat { writeMeta(obj, null) } - override fun Input.readThis(): Meta = readMeta(null) + override fun Input.readThis(): Meta = readMeta() fun Output.writeMeta(meta: Meta, descriptor: NodeDescriptor? = null) fun Input.readMeta(descriptor: NodeDescriptor? = null): Meta diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/MetaSerializer.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/MetaSerializer.kt deleted file mode 100644 index 5f644f47..00000000 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/MetaSerializer.kt +++ /dev/null @@ -1,67 +0,0 @@ -package hep.dataforge.io - -import hep.dataforge.meta.Config -import hep.dataforge.meta.Meta -import hep.dataforge.meta.toConfig -import hep.dataforge.names.Name -import hep.dataforge.names.NameToken -import hep.dataforge.names.toName -import kotlinx.serialization.* -import kotlinx.serialization.internal.StringDescriptor -import kotlinx.serialization.json.JsonObjectSerializer - -@Serializer(Name::class) -object NameSerializer : KSerializer { - override val descriptor: SerialDescriptor = StringDescriptor.withName("Name") - - override fun deserialize(decoder: Decoder): Name { - return decoder.decodeString().toName() - } - - override fun serialize(encoder: Encoder, obj: Name) { - encoder.encodeString(obj.toString()) - } -} - -@Serializer(NameToken::class) -object NameTokenSerializer : KSerializer { - override val descriptor: SerialDescriptor = StringDescriptor.withName("NameToken") - - override fun deserialize(decoder: Decoder): NameToken { - return decoder.decodeString().toName().first()!! - } - - override fun serialize(encoder: Encoder, obj: NameToken) { - encoder.encodeString(obj.toString()) - } -} - -/** - * Serialized for meta - */ -@Serializer(Meta::class) -object MetaSerializer : KSerializer { - override val descriptor: SerialDescriptor = JsonObjectSerializer.descriptor - - override fun deserialize(decoder: Decoder): Meta { - //currently just delegates serialization to json serializer - return JsonObjectSerializer.deserialize(decoder).toMeta() - } - - override fun serialize(encoder: Encoder, obj: Meta) { - JsonObjectSerializer.serialize(encoder, obj.toJson()) - } -} - -@Serializer(Config::class) -object ConfigSerializer : KSerializer { - override val descriptor: SerialDescriptor = JsonObjectSerializer.descriptor - - override fun deserialize(decoder: Decoder): Config { - return JsonObjectSerializer.deserialize(decoder).toMeta().toConfig() - } - - override fun serialize(encoder: Encoder, obj: Config) { - JsonObjectSerializer.serialize(encoder, obj.toJson()) - } -} \ No newline at end of file diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaglessEnvelopeFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaglessEnvelopeFormat.kt index af01acc4..94d56b46 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaglessEnvelopeFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaglessEnvelopeFormat.kt @@ -70,10 +70,10 @@ class TaglessEnvelopeFormat( if (line.startsWith(metaStart)) { val metaFormat = properties[META_TYPE_PROPERTY]?.let { io.metaFormat(it) } ?: JsonMetaFormat.default - meta = if (properties.containsKey(META_LENGTH_PROPERTY)) { - val bytes = ByteArray(properties[META_LENGTH_PROPERTY]!!.toInt()) - readFully(bytes) - metaFormat.readBytes(bytes) + val metaSize = properties.get(META_LENGTH_PROPERTY)?.toInt() + meta = if (metaSize != null) { + val metaPacket = ByteReadPacket(readBytes(metaSize)) + metaFormat.run { metaPacket.readThis() } } else { metaFormat.run { readThis() @@ -123,11 +123,12 @@ class TaglessEnvelopeFormat( if (line.startsWith(metaStart)) { val metaFormat = properties[META_TYPE_PROPERTY]?.let { io.metaFormat(it) } ?: JsonMetaFormat.default - meta = if (properties.containsKey(META_LENGTH_PROPERTY)) { - val bytes = ByteArray(properties[META_LENGTH_PROPERTY]!!.toInt()) - readFully(bytes) - offset += bytes.size.toUInt() - metaFormat.readBytes(bytes) + + val metaSize = properties.get(META_LENGTH_PROPERTY)?.toInt() + meta = if (metaSize != null) { + val metaPacket = ByteReadPacket(readBytes(metaSize)) + offset += metaSize.toUInt() + metaFormat.run { metaPacket.readThis() } } else { error("Can't partially read an envelope with undefined meta size") } diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/serialization/MetaSerializer.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/serialization/MetaSerializer.kt new file mode 100644 index 00000000..2f6a00de --- /dev/null +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/serialization/MetaSerializer.kt @@ -0,0 +1,135 @@ +package hep.dataforge.io + +import hep.dataforge.io.serialization.descriptor +import hep.dataforge.meta.* +import hep.dataforge.names.NameToken +import hep.dataforge.values.* +import kotlinx.serialization.* +import kotlinx.serialization.internal.* +import kotlinx.serialization.json.JsonInput +import kotlinx.serialization.json.JsonObjectSerializer +import kotlinx.serialization.json.JsonOutput + + +@Serializer(Value::class) +object ValueSerializer : KSerializer { + private val valueTypeSerializer = EnumSerializer(ValueType::class) + private val listSerializer by lazy { ArrayListSerializer(ValueSerializer)} + + override val descriptor: SerialDescriptor = descriptor("Value") { + element("isList") + element("valueType") + element("value") + } + + private fun Decoder.decodeValue(): Value { + return when (decode(valueTypeSerializer)) { + ValueType.NULL -> Null + ValueType.NUMBER -> decodeDouble().asValue() //TODO differentiate? + ValueType.BOOLEAN -> decodeBoolean().asValue() + ValueType.STRING -> decodeString().asValue() + else -> decodeString().parseValue() + } + } + + + override fun deserialize(decoder: Decoder): Value { + val isList = decoder.decodeBoolean() + return if (isList) { + listSerializer.deserialize(decoder).asValue() + } else { + decoder.decodeValue() + } + } + + private fun Encoder.encodeValue(value: Value) { + encode(valueTypeSerializer, value.type) + when (value.type) { + ValueType.NULL -> { + // do nothing + } + ValueType.NUMBER -> encodeDouble(value.double) + ValueType.BOOLEAN -> encodeBoolean(value.boolean) + ValueType.STRING -> encodeString(value.string) + else -> encodeString(value.string) + } + } + + override fun serialize(encoder: Encoder, obj: Value) { + encoder.encodeBoolean(obj.isList()) + if (obj.isList()) { + listSerializer.serialize(encoder, obj.list) + } else { + encoder.encodeValue(obj) + } + } +} + +@Serializer(MetaItem::class) +object MetaItemSerializer : KSerializer> { + override val descriptor: SerialDescriptor = descriptor("MetaItem") { + element("isNode") + element("value") + } + + override fun deserialize(decoder: Decoder): MetaItem<*> { + val isNode = decoder.decodeBoolean() + return if (isNode) { + MetaItem.NodeItem(decoder.decode(MetaSerializer)) + } else { + MetaItem.ValueItem(decoder.decode(ValueSerializer)) + } + } + + override fun serialize(encoder: Encoder, obj: MetaItem<*>) { + encoder.encodeBoolean(obj is MetaItem.NodeItem) + when (obj) { + is MetaItem.NodeItem -> MetaSerializer.serialize(encoder, obj.node) + is MetaItem.ValueItem -> ValueSerializer.serialize(encoder, obj.value) + } + } +} + +private class DeserializedMeta(override val items: Map>) : MetaBase() + + +/** + * Serialized for meta + */ +@Serializer(Meta::class) +object MetaSerializer : KSerializer { + private val mapSerializer = + HashMapSerializer(StringSerializer, MetaItemSerializer) + + override val descriptor: SerialDescriptor = + NamedMapClassDescriptor("Meta", StringSerializer.descriptor, MetaItemSerializer.descriptor) + + override fun deserialize(decoder: Decoder): Meta { + return if (decoder is JsonInput) { + JsonObjectSerializer.deserialize(decoder).toMeta() + } else { + DeserializedMeta(mapSerializer.deserialize(decoder).mapKeys { NameToken(it.key) }) + } + } + + override fun serialize(encoder: Encoder, obj: Meta) { + if (encoder is JsonOutput) { + JsonObjectSerializer.serialize(encoder, obj.toJson()) + } else { + mapSerializer.serialize(encoder, obj.items.mapKeys { it.key.toString() }) + } + } +} + +@Serializer(Config::class) +object ConfigSerializer : KSerializer { + override val descriptor: SerialDescriptor = MetaSerializer.descriptor + + override fun deserialize(decoder: Decoder): Config { + return MetaSerializer.deserialize(decoder).toConfig() + } + + override fun serialize(encoder: Encoder, obj: Config) { + MetaSerializer.serialize(encoder, obj) + } +} \ No newline at end of file diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/serialization/nameSerializers.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/serialization/nameSerializers.kt new file mode 100644 index 00000000..c12a6c19 --- /dev/null +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/serialization/nameSerializers.kt @@ -0,0 +1,33 @@ +package hep.dataforge.io.serialization + +import hep.dataforge.names.Name +import hep.dataforge.names.NameToken +import hep.dataforge.names.toName +import kotlinx.serialization.* +import kotlinx.serialization.internal.StringDescriptor + +@Serializer(Name::class) +object NameSerializer : KSerializer { + override val descriptor: SerialDescriptor = StringDescriptor.withName("Name") + + override fun deserialize(decoder: Decoder): Name { + return decoder.decodeString().toName() + } + + override fun serialize(encoder: Encoder, obj: Name) { + encoder.encodeString(obj.toString()) + } +} + +@Serializer(NameToken::class) +object NameTokenSerializer : KSerializer { + override val descriptor: SerialDescriptor = StringDescriptor.withName("NameToken") + + override fun deserialize(decoder: Decoder): NameToken { + return decoder.decodeString().toName().first()!! + } + + override fun serialize(encoder: Encoder, obj: NameToken) { + encoder.encodeString(obj.toString()) + } +} \ No newline at end of file diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/serialization/serializationUtils.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/serialization/serializationUtils.kt new file mode 100644 index 00000000..f83b2197 --- /dev/null +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/serialization/serializationUtils.kt @@ -0,0 +1,32 @@ +package hep.dataforge.io.serialization + +import kotlinx.serialization.CompositeDecoder +import kotlinx.serialization.Decoder +import kotlinx.serialization.KSerializer +import kotlinx.serialization.SerialDescriptor +import kotlinx.serialization.internal.SerialClassDescImpl + +inline class SerialDescriptorBuilder(private val impl: SerialClassDescImpl) { + fun element(name: String, isOptional: Boolean = false) = impl.addElement(name, isOptional) + + fun annotation(a: Annotation) = impl.pushAnnotation(a) + + fun classAnnotation(a: Annotation) = impl.pushClassAnnotation(a) + + fun descriptor(name: String, block: SerialDescriptorBuilder.() -> Unit) = impl.pushDescriptor( + SerialDescriptorBuilder(SerialClassDescImpl(name)).apply(block).build() + ) + + fun build(): SerialDescriptor = impl +} + +inline fun KSerializer<*>.descriptor(name: String, block: SerialDescriptorBuilder.() -> Unit): SerialDescriptor = + SerialDescriptorBuilder(SerialClassDescImpl(name)).apply(block).build() + +fun Decoder.decodeStructure( + desc: SerialDescriptor, + vararg typeParams: KSerializer<*> = emptyArray(), + block: CompositeDecoder.() -> Unit +) { + beginStructure(desc, *typeParams).apply(block).endStructure(desc) +} \ No newline at end of file diff --git a/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/MetaFormatTest.kt b/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/MetaFormatTest.kt index 3c7c29cb..16d946e3 100644 --- a/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/MetaFormatTest.kt +++ b/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/MetaFormatTest.kt @@ -4,13 +4,11 @@ import hep.dataforge.meta.* import kotlinx.serialization.json.JsonPrimitive import kotlinx.serialization.json.json import kotlinx.serialization.json.jsonArray -import kotlin.test.Ignore import kotlin.test.Test import kotlin.test.assertEquals class MetaFormatTest { @Test - @Ignore fun testBinaryMetaFormat() { val meta = buildMeta { "a" to 22 diff --git a/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/MetaSerializerTest.kt b/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/MetaSerializerTest.kt index ea5854d2..fcf9efb4 100644 --- a/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/MetaSerializerTest.kt +++ b/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/MetaSerializerTest.kt @@ -1,7 +1,11 @@ package hep.dataforge.io +import hep.dataforge.io.serialization.NameSerializer import hep.dataforge.meta.buildMeta import hep.dataforge.names.toName +import kotlinx.io.charsets.Charsets +import kotlinx.io.core.String +import kotlinx.serialization.cbor.Cbor import kotlinx.serialization.json.Json import kotlin.test.Test import kotlin.test.assertEquals @@ -23,6 +27,23 @@ class MetaSerializerTest { assertEquals(restored, meta) } + @Test + fun testCborSerialization() { + val meta = buildMeta { + "a" to 22 + "node" to { + "b" to "DDD" + "c" to 11.1 + "array" to doubleArrayOf(1.0, 2.0, 3.0) + } + } + + val bytes = Cbor.dump(MetaSerializer, meta) + println(String(bytes, charset = Charsets.ISO_8859_1)) + val restored = Cbor.load(MetaSerializer, bytes) + assertEquals(restored, meta) + } + @Test fun testNameSerialization() { val name = "a.b.c".toName() diff --git a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/tcp/InputStreamAsInput.kt b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/tcp/InputStreamAsInput.kt index 6dab414d..1c711be0 100644 --- a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/tcp/InputStreamAsInput.kt +++ b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/tcp/InputStreamAsInput.kt @@ -17,7 +17,6 @@ internal class InputStreamAsInput( override fun fill(): IoBuffer? { - val packet = stream.readPacketAtMost(4096) return pool.borrow().apply { resetForWrite(4096) diff --git a/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/FileEnvelopeTest.kt b/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/FileEnvelopeTest.kt index c362a8c3..280bd1bb 100644 --- a/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/FileEnvelopeTest.kt +++ b/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/FileEnvelopeTest.kt @@ -2,7 +2,6 @@ package hep.dataforge.io import hep.dataforge.context.Global import java.nio.file.Files -import kotlin.test.Ignore import kotlin.test.Test import kotlin.test.assertTrue @@ -22,7 +21,6 @@ class FileEnvelopeTest { } @Test - @Ignore fun testFileWriteRead() { val tmpPath = Files.createTempFile("dataforge_test", ".df") Global.io.writeEnvelopeFile(tmpPath,envelope) diff --git a/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/tcp/EnvelopeServerTest.kt b/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/tcp/EnvelopeServerTest.kt index 400d3b54..d41ce5be 100644 --- a/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/tcp/EnvelopeServerTest.kt +++ b/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/tcp/EnvelopeServerTest.kt @@ -45,7 +45,6 @@ class EnvelopeServerTest { @Test fun doEchoTest() { - val request = Envelope.invoke { type = "test.echo" meta { diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/values/valueEstensions.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/values/valueExtensions.kt similarity index 100% rename from dataforge-meta/src/commonMain/kotlin/hep/dataforge/values/valueEstensions.kt rename to dataforge-meta/src/commonMain/kotlin/hep/dataforge/values/valueExtensions.kt diff --git a/settings.gradle.kts b/settings.gradle.kts index 011123b9..b486c03f 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -12,7 +12,7 @@ pluginManagement { eachPlugin { when (requested.id.id) { "kotlinx-atomicfu" -> useModule("org.jetbrains.kotlinx:atomicfu-gradle-plugin:${requested.version}") - "scientifik.mpp", "scientifik.publish" -> useModule("scientifik:gradle-tools:${requested.version}") + "scientifik.mpp", "scientifik.jvm", "scientifik.publish" -> useModule("scientifik:gradle-tools:${requested.version}") } } } @@ -24,6 +24,7 @@ enableFeaturePreview("GRADLE_METADATA") include( ":dataforge-meta", ":dataforge-io", + ":dataforge-io:dataforge-io-yaml", ":dataforge-context", ":dataforge-data", ":dataforge-output", From 3147016bb37b5d46ae229145a9e34d1ed5fdef86 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Mon, 28 Oct 2019 10:31:37 +0300 Subject: [PATCH 31/48] Yaml feature --- dataforge-io/dataforge-io-yaml/build.gradle.kts | 1 + 1 file changed, 1 insertion(+) diff --git a/dataforge-io/dataforge-io-yaml/build.gradle.kts b/dataforge-io/dataforge-io-yaml/build.gradle.kts index 1a4af603..13c0dcfd 100644 --- a/dataforge-io/dataforge-io-yaml/build.gradle.kts +++ b/dataforge-io/dataforge-io-yaml/build.gradle.kts @@ -6,4 +6,5 @@ description = "YAML meta IO" dependencies{ api(project(":dataforge-io")) + api("com.charleskorn.kaml:kaml:0.14.0") } From cd4736ab5ec9fc9f3174281ecc7b7ea638b3c3c7 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Mon, 28 Oct 2019 12:35:19 +0300 Subject: [PATCH 32/48] Transformation of Meta to Map and back --- .../dataforge-io-yaml/build.gradle.kts | 9 ---- .../kotlin/hep/dataforge/meta/Builder.kt | 7 --- .../kotlin/hep/dataforge/meta/Laminate.kt | 2 - .../kotlin/hep/dataforge/meta/Meta.kt | 52 +------------------ .../kotlin/hep/dataforge/meta/annotations.kt | 11 ++++ .../kotlin/hep/dataforge/meta/mapMeta.kt | 36 +++++++++++++ .../hep/dataforge/meta/metaDelegates.kt | 19 ++++--- .../kotlin/hep/dataforge/meta/metaMatcher.kt | 51 ++++++++++++++++++ .../kotlin/hep/dataforge/meta/MetaTest.kt | 19 +++++++ settings.gradle.kts | 1 - 10 files changed, 129 insertions(+), 78 deletions(-) delete mode 100644 dataforge-io/dataforge-io-yaml/build.gradle.kts delete mode 100644 dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Builder.kt create mode 100644 dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/annotations.kt create mode 100644 dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/mapMeta.kt create mode 100644 dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/metaMatcher.kt diff --git a/dataforge-io/dataforge-io-yaml/build.gradle.kts b/dataforge-io/dataforge-io-yaml/build.gradle.kts deleted file mode 100644 index 1a4af603..00000000 --- a/dataforge-io/dataforge-io-yaml/build.gradle.kts +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id("scientifik.jvm") -} - -description = "YAML meta IO" - -dependencies{ - api(project(":dataforge-io")) -} diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Builder.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Builder.kt deleted file mode 100644 index c9878c5f..00000000 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Builder.kt +++ /dev/null @@ -1,7 +0,0 @@ -package hep.dataforge.meta - -/** - * General marker for dataforge builders - */ -@DslMarker -annotation class DFBuilder \ No newline at end of file diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Laminate.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Laminate.kt index b76fd46f..294140ae 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Laminate.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Laminate.kt @@ -4,8 +4,6 @@ import hep.dataforge.names.NameToken /** * A meta laminate consisting of multiple immutable meta layers. For mutable front layer, use [Styled]. - * - * */ class Laminate(layers: List) : MetaBase() { diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Meta.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Meta.kt index 6b7f85e9..e5aa0c45 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Meta.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Meta.kt @@ -80,27 +80,6 @@ operator fun Meta?.get(name: Name): MetaItem<*>? { operator fun Meta?.get(token: NameToken): MetaItem<*>? = this?.items?.get(token) operator fun Meta?.get(key: String): MetaItem<*>? = get(key.toName()) -/** - * Get all items matching given name. - */ -fun Meta.getAll(name: Name): Map> { - val root = when (name.length) { - 0 -> error("Can't use empty name for that") - 1 -> this - else -> (this[name.cutLast()] as? NodeItem<*>)?.node - } - - val (body, index) = name.last()!! - val regex = index.toRegex() - - return root?.items - ?.filter { it.key.body == body && (index.isEmpty() || regex.matches(it.key.index)) } - ?.mapKeys { it.key.index } - ?: emptyMap() -} - -fun Meta.getAll(name: String): Map> = getAll(name.toName()) - /** * Get a sequence of [Name]-[Value] pairs */ @@ -138,27 +117,6 @@ interface MetaNode> : Meta { override val items: Map> } -/** - * Get all items matching given name. - */ -fun > M.getAll(name: Name): Map> { - val root: MetaNode? = when (name.length) { - 0 -> error("Can't use empty name for that") - 1 -> this - else -> (this[name.cutLast()] as? NodeItem)?.node - } - - val (body, index) = name.last()!! - val regex = index.toRegex() - - return root?.items - ?.filter { it.key.body == body && (index.isEmpty() || regex.matches(it.key.index)) } - ?.mapKeys { it.key.index } - ?: emptyMap() -} - -fun > M.getAll(name: String): Map> = getAll(name.toName()) - operator fun > MetaNode?.get(name: Name): MetaItem? { if (this == null) return null return name.first()?.let { token -> @@ -229,7 +187,6 @@ object EmptyMeta : MetaBase() { /** * Unsafe methods to access values and nodes directly from [MetaItem] */ - val MetaItem<*>?.value: Value? get() = (this as? ValueItem)?.value ?: (this?.node?.get(VALUE_KEY) as? ValueItem)?.value @@ -258,11 +215,4 @@ val MetaItem?.node: M? is NodeItem -> node } -/** - * Generic meta-holder object - */ -interface Metoid { - val meta: Meta -} - -fun Meta.isEmpty() = this === EmptyMeta || this.items.isEmpty() +fun Meta.isEmpty() = this === EmptyMeta || this.items.isEmpty() \ No newline at end of file diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/annotations.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/annotations.kt new file mode 100644 index 00000000..865ef800 --- /dev/null +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/annotations.kt @@ -0,0 +1,11 @@ +package hep.dataforge.meta + +/** + * General marker for dataforge builders + */ +@DslMarker +annotation class DFBuilder + +@Experimental(level = Experimental.Level.WARNING) +@Retention(AnnotationRetention.BINARY) +annotation class DFExperimental \ No newline at end of file diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/mapMeta.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/mapMeta.kt new file mode 100644 index 00000000..32cb8abd --- /dev/null +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/mapMeta.kt @@ -0,0 +1,36 @@ +package hep.dataforge.meta + +import hep.dataforge.values.Value + +///** +// * Find all elements with given body +// */ +//private fun Meta.byBody(body: String): Map> = +// items.filter { it.key.body == body }.mapKeys { it.key.index } +// +//private fun Meta.distinctNames() = items.keys.map { it.body }.distinct() + +/** + * Convert meta to map of maps + */ +fun Meta.toMap(): Map { + return items.entries.associate { (token, item) -> + token.toString() to when (item) { + is MetaItem.NodeItem -> item.node.toMap() + is MetaItem.ValueItem -> item.value.value + } + } +} + +/** + * Convert map of maps to meta + */ +fun Map.toMeta(): Meta = buildMeta { + entries.forEach { (key, value) -> + @Suppress("UNCHECKED_CAST") + when (value) { + is Map<*, *> -> setNode(key, (value as Map).toMeta()) + else -> setValue(key, Value.of(value)) + } + } +} \ No newline at end of file diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/metaDelegates.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/metaDelegates.kt index 6bed5413..65e49da2 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/metaDelegates.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/metaDelegates.kt @@ -28,15 +28,21 @@ class StringDelegate(val meta: Meta, private val key: String? = null, private va } } -class BooleanDelegate(val meta: Meta, private val key: String? = null, private val default: Boolean? = null) : - ReadOnlyProperty { - override fun getValue(thisRef: Metoid, property: KProperty<*>): Boolean? { +class BooleanDelegate( + val meta: Meta, + private val key: String? = null, + private val default: Boolean? = null +) : ReadOnlyProperty { + override fun getValue(thisRef: Any?, property: KProperty<*>): Boolean? { return meta[key ?: property.name]?.boolean ?: default } } -class NumberDelegate(val meta: Meta, private val key: String? = null, private val default: Number? = null) : - ReadOnlyProperty { +class NumberDelegate( + val meta: Meta, + private val key: String? = null, + private val default: Number? = null +) : ReadOnlyProperty { override fun getValue(thisRef: Any?, property: KProperty<*>): Number? { return meta[key ?: property.name]?.number ?: default } @@ -168,9 +174,6 @@ fun Meta.number(key: String? = null, default: () -> Number) = inline fun > Meta.enum(default: E, key: String? = null) = SafeEnumDelegate(this, key, default) { enumValueOf(it) } - -fun Metoid.node(key: String? = null, converter: (Meta) -> T) = ChildDelegate(meta, key, converter) - /* Read-write delegates */ class MutableValueDelegate>( diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/metaMatcher.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/metaMatcher.kt new file mode 100644 index 00000000..8e2ea991 --- /dev/null +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/metaMatcher.kt @@ -0,0 +1,51 @@ +package hep.dataforge.meta + +import hep.dataforge.names.Name +import hep.dataforge.names.toName + +/** + * Get all items matching given name. + */ +@DFExperimental +fun Meta.getAll(name: Name): Map> { + val root = when (name.length) { + 0 -> error("Can't use empty name for that") + 1 -> this + else -> (this[name.cutLast()] as? MetaItem.NodeItem<*>)?.node + } + + val (body, index) = name.last()!! + val regex = index.toRegex() + + return root?.items + ?.filter { it.key.body == body && (index.isEmpty() || regex.matches(it.key.index)) } + ?.mapKeys { it.key.index } + ?: emptyMap() +} + +@DFExperimental +fun Meta.getAll(name: String): Map> = getAll(name.toName()) + + +/** + * Get all items matching given name. + */ +@DFExperimental +fun > M.getAll(name: Name): Map> { + val root: MetaNode? = when (name.length) { + 0 -> error("Can't use empty name for that") + 1 -> this + else -> (this[name.cutLast()] as? MetaItem.NodeItem)?.node + } + + val (body, index) = name.last()!! + val regex = index.toRegex() + + return root?.items + ?.filter { it.key.body == body && (index.isEmpty() || regex.matches(it.key.index)) } + ?.mapKeys { it.key.index } + ?: emptyMap() +} + +@DFExperimental +fun > M.getAll(name: String): Map> = getAll(name.toName()) \ No newline at end of file diff --git a/dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/MetaTest.kt b/dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/MetaTest.kt index 7f83b0b2..7528f8d2 100644 --- a/dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/MetaTest.kt +++ b/dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/MetaTest.kt @@ -30,4 +30,23 @@ class MetaTest { }.seal() assertEquals(meta1, meta2) } + + @Test + fun metaToMap(){ + val meta = buildMeta { + "a" to 22 + "b" to { + "c" to "ddd" + } + "list" to (0..4).map { + buildMeta { + "value" to it + } + } + } + val map = meta.toMap() + val reconstructed = map.toMeta() + + assertEquals(meta,reconstructed) + } } \ No newline at end of file diff --git a/settings.gradle.kts b/settings.gradle.kts index b486c03f..abf004a2 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -24,7 +24,6 @@ enableFeaturePreview("GRADLE_METADATA") include( ":dataforge-meta", ":dataforge-io", - ":dataforge-io:dataforge-io-yaml", ":dataforge-context", ":dataforge-data", ":dataforge-output", From 728da96effe86925592686fc27c1cc5cc5af4233 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Mon, 28 Oct 2019 19:37:15 +0300 Subject: [PATCH 33/48] Front matter envelope format --- .../dataforge-io-yaml/build.gradle.kts | 12 +++ .../io/yaml/FrontMatterEnvelopeFormat.kt | 92 +++++++++++++++++++ .../hep/dataforge/io/yaml/YamlMetaFormat.kt | 54 +++++++++++ .../dataforge/io/yaml/YamlMetaFormatTest.kt | 43 +++++++++ .../kotlin/hep/dataforge/io/Binary.kt | 4 +- .../hep/dataforge/io/BinaryMetaFormat.kt | 2 +- .../kotlin/hep/dataforge/io/EnvelopeFormat.kt | 4 +- .../kotlin/hep/dataforge/io/IOFormat.kt | 58 ++++-------- .../kotlin/hep/dataforge/io/MetaFormat.kt | 12 +-- .../hep/dataforge/io/TaggedEnvelopeFormat.kt | 10 +- .../hep/dataforge/io/TaglessEnvelopeFormat.kt | 10 +- .../io/functions/RemoteFunctionClient.kt | 6 +- .../io/functions/RemoteFunctionServer.kt | 4 +- .../kotlin/hep/dataforge/io/FileEnvelope.kt | 2 +- .../hep/dataforge/io/tcp/EnvelopeClient.kt | 4 +- .../hep/dataforge/io/tcp/EnvelopeServer.kt | 4 +- .../kotlin/hep/dataforge/meta/mapMeta.kt | 5 +- .../hep/dataforge/workspace/fileData.kt | 2 +- settings.gradle.kts | 1 + 19 files changed, 254 insertions(+), 75 deletions(-) create mode 100644 dataforge-io/dataforge-io-yaml/build.gradle.kts create mode 100644 dataforge-io/dataforge-io-yaml/src/main/kotlin/hep/dataforge/io/yaml/FrontMatterEnvelopeFormat.kt create mode 100644 dataforge-io/dataforge-io-yaml/src/main/kotlin/hep/dataforge/io/yaml/YamlMetaFormat.kt create mode 100644 dataforge-io/dataforge-io-yaml/src/test/kotlin/hep/dataforge/io/yaml/YamlMetaFormatTest.kt diff --git a/dataforge-io/dataforge-io-yaml/build.gradle.kts b/dataforge-io/dataforge-io-yaml/build.gradle.kts new file mode 100644 index 00000000..74ba43cf --- /dev/null +++ b/dataforge-io/dataforge-io-yaml/build.gradle.kts @@ -0,0 +1,12 @@ +plugins { + id("scientifik.jvm") +} + +description = "YAML meta IO" + +dependencies { + api(project(":dataforge-io")) + api("org.yaml:snakeyaml:1.25") + testImplementation(kotlin("test")) + testImplementation(kotlin("test-junit")) +} diff --git a/dataforge-io/dataforge-io-yaml/src/main/kotlin/hep/dataforge/io/yaml/FrontMatterEnvelopeFormat.kt b/dataforge-io/dataforge-io-yaml/src/main/kotlin/hep/dataforge/io/yaml/FrontMatterEnvelopeFormat.kt new file mode 100644 index 00000000..d12e0a26 --- /dev/null +++ b/dataforge-io/dataforge-io-yaml/src/main/kotlin/hep/dataforge/io/yaml/FrontMatterEnvelopeFormat.kt @@ -0,0 +1,92 @@ +package hep.dataforge.io.yaml + +import hep.dataforge.context.Context +import hep.dataforge.io.* +import hep.dataforge.meta.EmptyMeta +import hep.dataforge.meta.Meta +import hep.dataforge.meta.get +import hep.dataforge.meta.string +import kotlinx.io.core.* +import kotlinx.serialization.toUtf8Bytes + +class FrontMatterEnvelopeFormat( + val io: IOPlugin, + val metaType: String = YamlMetaFormat.name.toString(), + meta: Meta = EmptyMeta +) : EnvelopeFormat { + + val metaFormat = io.metaFormat(metaType, meta) + ?: error("Meta format with type $metaType could not be resolved in $io") + + override fun Input.readPartial(): PartialEnvelope { + var line: String = "" + var offset = 0u + do { + line = readUTF8Line() ?: error("Input does not contain front matter separator") + offset += line.toUtf8Bytes().size.toUInt() + } while (!line.startsWith(SEPARATOR)) + + val readMetaFormat = + metaTypeRegex.matchEntire(line)?.groupValues?.first() + ?.let { io.metaFormat(it) } ?: YamlMetaFormat.default + + val metaBlock = buildPacket { + do { + line = readUTF8Line() ?: error("Input does not contain closing front matter separator") + appendln(line) + offset += line.toUtf8Bytes().size.toUInt() + } while (!line.startsWith(SEPARATOR)) + } + val meta = readMetaFormat.fromBytes(metaBlock) + return PartialEnvelope(meta, offset, null) + } + + override fun Input.readObject(): Envelope { + var line: String = "" + do { + line = readUTF8Line() ?: error("Input does not contain front matter separator") + } while (!line.startsWith(SEPARATOR)) + + val readMetaFormat = + metaTypeRegex.matchEntire(line)?.groupValues?.first() + ?.let { io.metaFormat(it) } ?: YamlMetaFormat.default + + val metaBlock = buildPacket { + do { + appendln(readUTF8Line() ?: error("Input does not contain closing front matter separator")) + } while (!line.startsWith(SEPARATOR)) + } + val meta = readMetaFormat.fromBytes(metaBlock) + val bytes = readBytes() + val data = bytes.asBinary() + return SimpleEnvelope(meta, data) + } + + override fun Output.writeObject(obj: Envelope) { + writeText("$SEPARATOR\r\n") + metaFormat.run { writeObject(obj.meta) } + writeText("$SEPARATOR\r\n") + obj.data?.read { copyTo(this@writeObject) } + } + + companion object : EnvelopeFormatFactory { + const val SEPARATOR = "---" + + private val metaTypeRegex = "---(\\w*)\\s*".toRegex() + + override fun invoke(meta: Meta, context: Context): EnvelopeFormat { + val metaFormatName: String = meta["name"].string ?: YamlMetaFormat.name.toString() + return FrontMatterEnvelopeFormat(context.io, metaFormatName, meta) + } + + override fun peekFormat(io: IOPlugin, input: Input): EnvelopeFormat? { + val line = input.readUTF8Line(3, 30) + return if (line != null && line.startsWith("---")) { + invoke() + } else { + null + } + } + + } +} \ No newline at end of file diff --git a/dataforge-io/dataforge-io-yaml/src/main/kotlin/hep/dataforge/io/yaml/YamlMetaFormat.kt b/dataforge-io/dataforge-io-yaml/src/main/kotlin/hep/dataforge/io/yaml/YamlMetaFormat.kt new file mode 100644 index 00000000..39fa68df --- /dev/null +++ b/dataforge-io/dataforge-io-yaml/src/main/kotlin/hep/dataforge/io/yaml/YamlMetaFormat.kt @@ -0,0 +1,54 @@ +package hep.dataforge.io.yaml + +import hep.dataforge.context.Context +import hep.dataforge.descriptors.NodeDescriptor +import hep.dataforge.io.MetaFormat +import hep.dataforge.io.MetaFormatFactory +import hep.dataforge.meta.Meta +import hep.dataforge.meta.toMap +import hep.dataforge.meta.toMeta +import hep.dataforge.names.Name +import hep.dataforge.names.plus +import kotlinx.io.core.Input +import kotlinx.io.core.Output +import kotlinx.io.core.readUByte +import kotlinx.io.core.writeText +import org.yaml.snakeyaml.Yaml +import java.io.InputStream + +private class InputAsStream(val input: Input) : InputStream() { + override fun read(): Int { + if (input.endOfInput) return -1 + return input.readUByte().toInt() + } + + override fun close() { + input.close() + } +} + +private fun Input.asStream() = InputAsStream(this) + +class YamlMetaFormat(val meta: Meta) : MetaFormat { + private val yaml = Yaml() + + override fun Output.writeMeta(meta: Meta, descriptor: NodeDescriptor?) { + val string = yaml.dump(meta.toMap(descriptor)) + writeText(string) + } + + override fun Input.readMeta(descriptor: NodeDescriptor?): Meta { + val map: Map = yaml.load(asStream()) + return map.toMeta(descriptor) + } + + companion object : MetaFormatFactory { + val default = YamlMetaFormat() + + override fun invoke(meta: Meta, context: Context): MetaFormat = YamlMetaFormat(meta) + + override val name: Name = super.name + "yaml" + + override val key: Short = 0x594d //YM + } +} \ No newline at end of file diff --git a/dataforge-io/dataforge-io-yaml/src/test/kotlin/hep/dataforge/io/yaml/YamlMetaFormatTest.kt b/dataforge-io/dataforge-io-yaml/src/test/kotlin/hep/dataforge/io/yaml/YamlMetaFormatTest.kt new file mode 100644 index 00000000..0827ff1e --- /dev/null +++ b/dataforge-io/dataforge-io-yaml/src/test/kotlin/hep/dataforge/io/yaml/YamlMetaFormatTest.kt @@ -0,0 +1,43 @@ +package hep.dataforge.io.yaml + +import hep.dataforge.io.parse +import hep.dataforge.io.toString +import hep.dataforge.meta.Meta +import hep.dataforge.meta.buildMeta +import hep.dataforge.meta.get +import hep.dataforge.meta.seal +import org.junit.Test +import kotlin.test.assertEquals + + +class YamlMetaFormatTest{ + @Test + fun testYamlMetaFormat(){ + val meta = buildMeta { + "a" to 22 + "node" to { + "b" to "DDD" + "c" to 11.1 + "d" to { + "d1" to { + "d11" to "aaa" + "d12" to "bbb" + } + "d2" to 2 + } + "array" to doubleArrayOf(1.0, 2.0, 3.0) + } + } + val string = meta.toString(YamlMetaFormat) + println(string) + val result = YamlMetaFormat.parse(string) + + assertEquals(meta, meta.seal()) + + meta.items.keys.forEach { + if (meta[it] != result[it]) error("${meta[it]} != ${result[it]}") + } + + assertEquals(meta, result) + } +} \ No newline at end of file diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Binary.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Binary.kt index a20f6fac..252de7bc 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Binary.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Binary.kt @@ -76,13 +76,13 @@ fun ByteArray.asBinary() = ArrayBinary(this) */ fun Binary.readWith(format: IOFormat): T = format.run { read { - readThis() + readObject() } } fun IOFormat.writeBinary(obj: T): Binary { val packet = buildPacket { - writeThis(obj) + writeObject(obj) } return ArrayBinary(packet.readBytes()) } \ No newline at end of file diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/BinaryMetaFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/BinaryMetaFormat.kt index 48d84904..ba9886d8 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/BinaryMetaFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/BinaryMetaFormat.kt @@ -85,7 +85,7 @@ object BinaryMetaFormat : MetaFormat, MetaFormatFactory { writeValue(item.value) } is MetaItem.NodeItem -> { - writeThis(item.node) + writeObject(item.node) } } } diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/EnvelopeFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/EnvelopeFormat.kt index ef6326f0..4c5c5839 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/EnvelopeFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/EnvelopeFormat.kt @@ -20,9 +20,9 @@ data class PartialEnvelope(val meta: Meta, val dataOffset: UInt, val dataSize: U interface EnvelopeFormat : IOFormat { fun Input.readPartial(): PartialEnvelope - override fun Input.readThis(): Envelope + override fun Input.readObject(): Envelope - override fun Output.writeThis(obj: Envelope) + override fun Output.writeObject(obj: Envelope) } @Type(ENVELOPE_FORMAT_TYPE) diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOFormat.kt index 3734b82d..093ffbc8 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOFormat.kt @@ -23,27 +23,27 @@ import kotlin.reflect.KClass * And interface for reading and writing objects into with IO streams */ interface IOFormat { - fun Output.writeThis(obj: T) - fun Input.readThis(): T + fun Output.writeObject(obj: T) + fun Input.readObject(): T } -fun Input.readWith(format: IOFormat): T = format.run { readThis() } -fun Output.writeWith(format: IOFormat, obj: T) = format.run { writeThis(obj) } +fun Input.readWith(format: IOFormat): T = format.run { readObject() } +fun Output.writeWith(format: IOFormat, obj: T) = format.run { writeObject(obj) } class ListIOFormat(val format: IOFormat) : IOFormat> { - override fun Output.writeThis(obj: List) { + override fun Output.writeObject(obj: List) { writeInt(obj.size) format.run { obj.forEach { - writeThis(it) + writeObject(it) } } } - override fun Input.readThis(): List { + override fun Input.readObject(): List { val size = readInt() return format.run { - List(size) { readThis() } + List(size) { readObject() } } } } @@ -79,32 +79,8 @@ inline fun buildPacketWithoutPool(headerSizeHint: Int = 0, block: BytePacketBuil return builder.build() } -//@Suppress("EXPERIMENTAL_API_USAGE", "EXPERIMENTAL_OVERRIDE") -//internal fun Input.useAtMost(most: Int, reader: Input.() -> R): R { -// val limitedInput: Input = object : AbstractInput( -// IoBuffer.Pool.borrow(), -// remaining = most.toLong(), -// pool = IoBuffer.Pool -// ) { -// var read = 0 -// override fun closeSource() { -// this@useAtMost.close() -// } -// -// override fun fill(): IoBuffer? { -// if (read >= most) return null -// return IoBuffer.Pool.fill { -// reserveEndGap(IoBuffer.ReservedSize) -// read += this@useAtMost.peekTo(this, max = most - read) -// } -// } -// -// } -// return limitedInput.reader() -//} - -fun IOFormat.writePacket(obj: T): ByteReadPacket = buildPacket { writeThis(obj) } -fun IOFormat.writeBytes(obj: T): ByteArray = buildPacket { writeThis(obj) }.readBytes() +fun IOFormat.writePacket(obj: T): ByteReadPacket = buildPacket { writeObject(obj) } +fun IOFormat.writeBytes(obj: T): ByteArray = buildPacket { writeObject(obj) }.readBytes() fun IOFormat.readBytes(array: ByteArray): T { //= ByteReadPacket(array).readThis() val byteArrayInput: Input = object : AbstractInput( @@ -129,7 +105,7 @@ fun IOFormat.readBytes(array: ByteArray): T { } } - return byteArrayInput.readThis() + return byteArrayInput.readObject() } object DoubleIOFormat : IOFormat, IOFormatFactory { @@ -139,11 +115,11 @@ object DoubleIOFormat : IOFormat, IOFormatFactory { override val type: KClass get() = Double::class - override fun Output.writeThis(obj: Double) { + override fun Output.writeObject(obj: Double) { writeDouble(obj) } - override fun Input.readThis(): Double = readDouble() + override fun Input.readObject(): Double = readDouble() } object ValueIOFormat : IOFormat, IOFormatFactory { @@ -153,11 +129,11 @@ object ValueIOFormat : IOFormat, IOFormatFactory { override val type: KClass get() = Value::class - override fun Output.writeThis(obj: Value) { + override fun Output.writeObject(obj: Value) { BinaryMetaFormat.run { writeValue(obj) } } - override fun Input.readThis(): Value { + override fun Input.readObject(): Value { return (BinaryMetaFormat.run { readMetaItem() } as? MetaItem.ValueItem)?.value ?: error("The item is not a value") } @@ -175,12 +151,12 @@ class SerializerIOFormat( //override val name: Name = type.simpleName?.toName() ?: EmptyName - override fun Output.writeThis(obj: T) { + override fun Output.writeObject(obj: T) { val bytes = Cbor.plain.dump(serializer, obj) writeFully(bytes) } - override fun Input.readThis(): T { + override fun Input.readObject(): T { //FIXME reads the whole input val bytes = readBytes() return Cbor.plain.load(serializer, bytes) diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/MetaFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/MetaFormat.kt index 83f04957..313d04ae 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/MetaFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/MetaFormat.kt @@ -16,11 +16,11 @@ import kotlin.reflect.KClass interface MetaFormat : IOFormat { - override fun Output.writeThis(obj: Meta) { + override fun Output.writeObject(obj: Meta) { writeMeta(obj, null) } - override fun Input.readThis(): Meta = readMeta() + override fun Input.readObject(): Meta = readMeta() fun Output.writeMeta(meta: Meta, descriptor: NodeDescriptor? = null) fun Input.readMeta(descriptor: NodeDescriptor? = null): Meta @@ -42,23 +42,23 @@ interface MetaFormatFactory : IOFormatFactory { } fun Meta.toString(format: MetaFormat): String = buildPacket { - format.run { writeThis(this@toString) } + format.run { writeObject(this@toString) } }.readText() fun Meta.toString(formatFactory: MetaFormatFactory): String = toString(formatFactory()) fun Meta.toBytes(format: MetaFormat = JsonMetaFormat.default): ByteReadPacket = buildPacket { - format.run { writeThis(this@toBytes) } + format.run { writeObject(this@toBytes) } } fun MetaFormat.parse(str: String): Meta { - return ByteReadPacket(str.toByteArray()).readThis() + return ByteReadPacket(str.toByteArray()).readObject() } fun MetaFormatFactory.parse(str: String): Meta = invoke().parse(str) fun MetaFormat.fromBytes(packet: ByteReadPacket): Meta { - return packet.readThis() + return packet.readObject() } diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt index 26239601..b69a5f19 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt @@ -28,12 +28,12 @@ class TaggedEnvelopeFormat( writeText(END_SEQUENCE) } - override fun Output.writeThis(obj: Envelope) { + override fun Output.writeObject(obj: Envelope) { val metaBytes = metaFormat.writeBytes(obj.meta) val tag = Tag(metaFormatKey, metaBytes.size.toUInt(), obj.data?.size ?: 0.toULong()) writePacket(tag.toBytes()) writeFully(metaBytes) - obj.data?.read { copyTo(this@writeThis) } + obj.data?.read { copyTo(this@writeObject) } } /** @@ -42,7 +42,7 @@ class TaggedEnvelopeFormat( * @param input an input to read from * @param formats a collection of meta formats to resolve */ - override fun Input.readThis(): Envelope { + override fun Input.readObject(): Envelope { val tag = readTag() val metaFormat = io.metaFormat(tag.metaFormatKey) @@ -51,7 +51,7 @@ class TaggedEnvelopeFormat( val metaPacket = ByteReadPacket(readBytes(tag.metaSize.toInt())) val dataBytes = readBytes(tag.dataSize.toInt()) - val meta = metaFormat.run { metaPacket.readThis() } + val meta = metaFormat.run { metaPacket.readObject() } return SimpleEnvelope(meta, ArrayBinary(dataBytes)) } @@ -62,7 +62,7 @@ class TaggedEnvelopeFormat( ?: error("Meta format with key ${tag.metaFormatKey} not found") val metaPacket = ByteReadPacket(readBytes(tag.metaSize.toInt())) - val meta = metaFormat.run { metaPacket.readThis() } + val meta = metaFormat.run { metaPacket.readObject() } return PartialEnvelope(meta, TAG_SIZE + tag.metaSize, tag.dataSize) } diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaglessEnvelopeFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaglessEnvelopeFormat.kt index 94d56b46..83fbe46b 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaglessEnvelopeFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaglessEnvelopeFormat.kt @@ -22,7 +22,7 @@ class TaglessEnvelopeFormat( writeText("#? $key: $value;\r\n") } - override fun Output.writeThis(obj: Envelope) { + override fun Output.writeObject(obj: Envelope) { //printing header writeText(TAGLESS_ENVELOPE_HEADER + "\r\n") @@ -48,7 +48,7 @@ class TaglessEnvelopeFormat( } } - override fun Input.readThis(): Envelope { + override fun Input.readObject(): Envelope { var line: String = "" do { line = readUTF8Line() ?: error("Input does not contain tagless envelope header") @@ -73,10 +73,10 @@ class TaglessEnvelopeFormat( val metaSize = properties.get(META_LENGTH_PROPERTY)?.toInt() meta = if (metaSize != null) { val metaPacket = ByteReadPacket(readBytes(metaSize)) - metaFormat.run { metaPacket.readThis() } + metaFormat.run { metaPacket.readObject() } } else { metaFormat.run { - readThis() + readObject() } } } @@ -128,7 +128,7 @@ class TaglessEnvelopeFormat( meta = if (metaSize != null) { val metaPacket = ByteReadPacket(readBytes(metaSize)) offset += metaSize.toUInt() - metaFormat.run { metaPacket.readThis() } + metaFormat.run { metaPacket.readObject() } } else { error("Can't partially read an envelope with undefined meta size") } diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/functions/RemoteFunctionClient.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/functions/RemoteFunctionClient.kt index 4b586ef7..23df23cc 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/functions/RemoteFunctionClient.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/functions/RemoteFunctionClient.kt @@ -20,7 +20,7 @@ class RemoteFunctionClient(override val context: Context, val responder: Respond data { val inputFormat: IOFormat = getInputFormat(meta, valueType) inputFormat.run { - writeThis(value) + writeObject(value) } } } @@ -39,7 +39,7 @@ class RemoteFunctionClient(override val context: Context, val responder: Respond val inputFormat: IOFormat = getInputFormat(meta, valueType) inputFormat.run { values.forEach { - writeThis(it) + writeObject(it) } } } @@ -56,7 +56,7 @@ class RemoteFunctionClient(override val context: Context, val responder: Respond envelope.data?.read { List(size) { outputFormat.run { - readThis() + readObject() } } } ?: error("Message does not contain data") diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/functions/RemoteFunctionServer.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/functions/RemoteFunctionServer.kt index 5bdf3139..8252b1d3 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/functions/RemoteFunctionServer.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/functions/RemoteFunctionServer.kt @@ -30,7 +30,7 @@ class RemoteFunctionServer( val input = request.data?.read { inputFormat.run { List(size) { - readThis() + readObject() } } } ?: error("Input is empty") @@ -48,7 +48,7 @@ class RemoteFunctionServer( data { outputFormat.run { output.forEach { - writeThis(it) + writeObject(it) } } } diff --git a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileEnvelope.kt b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileEnvelope.kt index 6cb3473f..dcc8685d 100644 --- a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileEnvelope.kt +++ b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileEnvelope.kt @@ -46,7 +46,7 @@ fun IOPlugin.writeEnvelopeFile( ).asOutput() with(formatFactory(formatMeta, context)) { - output.writeThis(envelope) + output.writeObject(envelope) } } diff --git a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/tcp/EnvelopeClient.kt b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/tcp/EnvelopeClient.kt index 791a57b5..b6b85101 100644 --- a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/tcp/EnvelopeClient.kt +++ b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/tcp/EnvelopeClient.kt @@ -56,10 +56,10 @@ class EnvelopeClient( val output = socket.getOutputStream() format.run { output.writePacket { - writeThis(request) + writeObject(request) } logger.debug { "Sent request with type ${request.type} to ${socket.remoteSocketAddress}" } - val res = input.readThis() + val res = input.readObject() logger.debug { "Received response with type ${res.type} from ${socket.remoteSocketAddress}" } return@withContext res } diff --git a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/tcp/EnvelopeServer.kt b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/tcp/EnvelopeServer.kt index e81a1ea6..b733aedd 100644 --- a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/tcp/EnvelopeServer.kt +++ b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/tcp/EnvelopeServer.kt @@ -75,7 +75,7 @@ class EnvelopeServer( val outputStream = socket.getOutputStream() format.run { while (socket.isConnected) { - val request = input.readThis() + val request = input.readObject() logger.debug { "Accepted request with type ${request.type} from ${socket.remoteSocketAddress}" } if (request.type == SHUTDOWN_ENVELOPE_TYPE) { //Echo shutdown command @@ -87,7 +87,7 @@ class EnvelopeServer( runBlocking { val response = responder.respond(request) outputStream.writePacket { - writeThis(response) + writeObject(response) } logger.debug { "Sent response with type ${response.type} to ${socket.remoteSocketAddress}" } } diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/mapMeta.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/mapMeta.kt index 32cb8abd..d146ea1f 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/mapMeta.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/mapMeta.kt @@ -1,5 +1,6 @@ package hep.dataforge.meta +import hep.dataforge.descriptors.NodeDescriptor import hep.dataforge.values.Value ///** @@ -13,7 +14,7 @@ import hep.dataforge.values.Value /** * Convert meta to map of maps */ -fun Meta.toMap(): Map { +fun Meta.toMap(descriptor: NodeDescriptor? = null): Map { return items.entries.associate { (token, item) -> token.toString() to when (item) { is MetaItem.NodeItem -> item.node.toMap() @@ -25,7 +26,7 @@ fun Meta.toMap(): Map { /** * Convert map of maps to meta */ -fun Map.toMeta(): Meta = buildMeta { +fun Map.toMeta(descriptor: NodeDescriptor? = null): Meta = buildMeta { entries.forEach { (key, value) -> @Suppress("UNCHECKED_CAST") when (value) { diff --git a/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/fileData.kt b/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/fileData.kt index cd6aafea..a483c78b 100644 --- a/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/fileData.kt +++ b/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/fileData.kt @@ -63,7 +63,7 @@ fun IOPlugin.readData( dataFormat.run { Files.newByteChannel(path, StandardOpenOption.READ) .asInput() - .readThis() + .readObject() } } } diff --git a/settings.gradle.kts b/settings.gradle.kts index abf004a2..b486c03f 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -24,6 +24,7 @@ enableFeaturePreview("GRADLE_METADATA") include( ":dataforge-meta", ":dataforge-io", + ":dataforge-io:dataforge-io-yaml", ":dataforge-context", ":dataforge-data", ":dataforge-output", From 04e9601def4f639d62063adbbe8250e8ad532bbb Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Tue, 29 Oct 2019 18:20:05 +0300 Subject: [PATCH 34/48] Fixed array representation in DynamicMeta --- .../kotlin/hep/dataforge/meta/DynamicMeta.kt | 41 ++++++++++++++----- .../hep/dataforge/meta/DynamicMetaTest.kt | 29 +++++++++++++ 2 files changed, 59 insertions(+), 11 deletions(-) diff --git a/dataforge-meta/src/jsMain/kotlin/hep/dataforge/meta/DynamicMeta.kt b/dataforge-meta/src/jsMain/kotlin/hep/dataforge/meta/DynamicMeta.kt index 694e2441..ff23a571 100644 --- a/dataforge-meta/src/jsMain/kotlin/hep/dataforge/meta/DynamicMeta.kt +++ b/dataforge-meta/src/jsMain/kotlin/hep/dataforge/meta/DynamicMeta.kt @@ -3,18 +3,27 @@ package hep.dataforge.meta import hep.dataforge.names.NameToken import hep.dataforge.values.Null import hep.dataforge.values.Value +import hep.dataforge.values.isList //TODO add Meta wrapper for dynamic +fun Value.toDynamic(): dynamic { + return if (isList()) { + list.map { it.toDynamic() }.toTypedArray().asDynamic() + } else { + value.asDynamic() + } +} + /** * Represent or copy this [Meta] to dynamic object to be passed to JS libraries */ fun Meta.toDynamic(): dynamic { - if(this is DynamicMeta) return this.obj + if (this is DynamicMeta) return this.obj fun MetaItem<*>.toDynamic(): dynamic = when (this) { - is MetaItem.ValueItem -> this.value.value.asDynamic() + is MetaItem.ValueItem -> this.value.toDynamic() is MetaItem.NodeItem -> this.node.toDynamic() } @@ -35,15 +44,21 @@ class DynamicMeta(val obj: dynamic) : MetaBase() { private fun isArray(@Suppress("UNUSED_PARAMETER") obj: dynamic): Boolean = js("Array.isArray(obj)") as Boolean + private fun isPrimitive(obj: dynamic): Boolean = + (jsTypeOf(obj) != "object") + @Suppress("UNCHECKED_CAST") private fun asItem(obj: dynamic): MetaItem? { - if (obj == null) return MetaItem.ValueItem(Null) - return when (jsTypeOf(obj as? Any)) { - "boolean" -> MetaItem.ValueItem(Value.of(obj as Boolean)) - "number" -> MetaItem.ValueItem(Value.of(obj as Number)) - "string" -> MetaItem.ValueItem(Value.of(obj as String)) - "object" -> MetaItem.NodeItem(DynamicMeta(obj)) - else -> null + return when { + obj == null -> MetaItem.ValueItem(Null) + isArray(obj) && (obj as Array).all { isPrimitive(it) } -> MetaItem.ValueItem(Value.of(obj as Array)) + else -> when (jsTypeOf(obj)) { + "boolean" -> MetaItem.ValueItem(Value.of(obj as Boolean)) + "number" -> MetaItem.ValueItem(Value.of(obj as Number)) + "string" -> MetaItem.ValueItem(Value.of(obj as String)) + "object" -> MetaItem.NodeItem(DynamicMeta(obj)) + else -> null + } } } @@ -51,11 +66,15 @@ class DynamicMeta(val obj: dynamic) : MetaBase() { get() = keys().flatMap>> { key -> val value = obj[key] ?: return@flatMap emptyList() if (isArray(value)) { - return@flatMap (value as Array) - .mapIndexedNotNull() { index, it -> + val array = value as Array + return@flatMap if (array.all { isPrimitive(it) }) { + listOf(NameToken(key) to MetaItem.ValueItem(Value.of(array))) + } else { + array.mapIndexedNotNull { index, it -> val item = asItem(it) ?: return@mapIndexedNotNull null NameToken(key, index.toString()) to item } + } } else { val item = asItem(value) ?: return@flatMap emptyList() listOf(NameToken(key) to item) diff --git a/dataforge-meta/src/jsTest/kotlin/hep/dataforge/meta/DynamicMetaTest.kt b/dataforge-meta/src/jsTest/kotlin/hep/dataforge/meta/DynamicMetaTest.kt index ce46a740..772adf06 100644 --- a/dataforge-meta/src/jsTest/kotlin/hep/dataforge/meta/DynamicMetaTest.kt +++ b/dataforge-meta/src/jsTest/kotlin/hep/dataforge/meta/DynamicMetaTest.kt @@ -2,6 +2,7 @@ package hep.dataforge.meta import kotlin.test.Test import kotlin.test.assertEquals +import kotlin.test.assertTrue class DynamicMetaTest { @@ -19,6 +20,34 @@ class DynamicMetaTest { val meta = DynamicMeta(d) assertEquals(true, meta["ob.booleanNode"].boolean) assertEquals(2, meta["array[1]"].int) + assertEquals(4, meta.items.size) + } + + @Test + fun testMetaToDynamic(){ + val meta = buildMeta { + "a" to 22 + "array" to arrayOf(1, 2, 3) + "b" to "myString" + "ob" to { + "childNode" to 18 + "booleanNode" to true + } + } + + val dynamic = meta.toDynamic() + + assertEquals(2,dynamic.array[1]) + + assertEquals(22, dynamic.a) + + val keys = js("Object.keys(dynamic)") as Array + + assertTrue { keys.contains("ob") } + + assertEquals(18, dynamic.ob.childNode) + + assertEquals(meta, DynamicMeta(dynamic)) } } \ No newline at end of file From 264ed14cf468e39fe549914ffbbd0917f8fc0ec0 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Tue, 29 Oct 2019 18:45:24 +0300 Subject: [PATCH 35/48] Fixed array representation in DynamicMeta --- .../src/jsTest/kotlin/hep/dataforge/meta/DynamicMetaTest.kt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dataforge-meta/src/jsTest/kotlin/hep/dataforge/meta/DynamicMetaTest.kt b/dataforge-meta/src/jsTest/kotlin/hep/dataforge/meta/DynamicMetaTest.kt index 772adf06..1614addc 100644 --- a/dataforge-meta/src/jsTest/kotlin/hep/dataforge/meta/DynamicMetaTest.kt +++ b/dataforge-meta/src/jsTest/kotlin/hep/dataforge/meta/DynamicMetaTest.kt @@ -1,5 +1,6 @@ package hep.dataforge.meta +import hep.dataforge.values.int import kotlin.test.Test import kotlin.test.assertEquals import kotlin.test.assertTrue @@ -19,7 +20,7 @@ class DynamicMetaTest { val meta = DynamicMeta(d) assertEquals(true, meta["ob.booleanNode"].boolean) - assertEquals(2, meta["array[1]"].int) + assertEquals(2, meta["array"].value?.list?.get(1)?.int) assertEquals(4, meta.items.size) } From 10de05240c160e28a0927a1908837078ea8ec8cc Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Tue, 29 Oct 2019 20:11:57 +0300 Subject: [PATCH 36/48] Fix #12 --- .../io/yaml/FrontMatterEnvelopeFormat.kt | 6 ++-- .../hep/dataforge/io/yaml/YamlMetaFormat.kt | 2 ++ .../hep/dataforge/meta/configDelegates.kt | 8 ++--- .../kotlin/hep/dataforge/meta/mapMeta.kt | 2 ++ .../kotlin/hep/dataforge/values/Value.kt | 30 ++++++++++++------- .../hep/dataforge/values/exoticValues.kt | 2 +- .../hep/dataforge/meta/SpecificationTest.kt | 23 ++++++++++++++ 7 files changed, 53 insertions(+), 20 deletions(-) create mode 100644 dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/SpecificationTest.kt diff --git a/dataforge-io/dataforge-io-yaml/src/main/kotlin/hep/dataforge/io/yaml/FrontMatterEnvelopeFormat.kt b/dataforge-io/dataforge-io-yaml/src/main/kotlin/hep/dataforge/io/yaml/FrontMatterEnvelopeFormat.kt index d12e0a26..9306b336 100644 --- a/dataforge-io/dataforge-io-yaml/src/main/kotlin/hep/dataforge/io/yaml/FrontMatterEnvelopeFormat.kt +++ b/dataforge-io/dataforge-io-yaml/src/main/kotlin/hep/dataforge/io/yaml/FrontMatterEnvelopeFormat.kt @@ -2,13 +2,11 @@ package hep.dataforge.io.yaml import hep.dataforge.context.Context import hep.dataforge.io.* -import hep.dataforge.meta.EmptyMeta -import hep.dataforge.meta.Meta -import hep.dataforge.meta.get -import hep.dataforge.meta.string +import hep.dataforge.meta.* import kotlinx.io.core.* import kotlinx.serialization.toUtf8Bytes +@DFExperimental class FrontMatterEnvelopeFormat( val io: IOPlugin, val metaType: String = YamlMetaFormat.name.toString(), diff --git a/dataforge-io/dataforge-io-yaml/src/main/kotlin/hep/dataforge/io/yaml/YamlMetaFormat.kt b/dataforge-io/dataforge-io-yaml/src/main/kotlin/hep/dataforge/io/yaml/YamlMetaFormat.kt index 39fa68df..24ea44ec 100644 --- a/dataforge-io/dataforge-io-yaml/src/main/kotlin/hep/dataforge/io/yaml/YamlMetaFormat.kt +++ b/dataforge-io/dataforge-io-yaml/src/main/kotlin/hep/dataforge/io/yaml/YamlMetaFormat.kt @@ -4,6 +4,7 @@ import hep.dataforge.context.Context import hep.dataforge.descriptors.NodeDescriptor import hep.dataforge.io.MetaFormat import hep.dataforge.io.MetaFormatFactory +import hep.dataforge.meta.DFExperimental import hep.dataforge.meta.Meta import hep.dataforge.meta.toMap import hep.dataforge.meta.toMeta @@ -29,6 +30,7 @@ private class InputAsStream(val input: Input) : InputStream() { private fun Input.asStream() = InputAsStream(this) +@DFExperimental class YamlMetaFormat(val meta: Meta) : MetaFormat { private val yaml = Yaml() diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/configDelegates.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/configDelegates.kt index 1dabd05b..34fa0e71 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/configDelegates.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/configDelegates.kt @@ -117,11 +117,11 @@ fun Configurable.spec(builder: (Config) -> T, key: Name? = null) * Extra delegates for special cases */ -fun Configurable.stringList(key: Name? = null): ReadWriteDelegateWrapper> = - value(emptyList(), key) { it?.list?.map { value -> value.string } ?: emptyList() } +fun Configurable.stringList(vararg strings: String, key: Name? = null): ReadWriteDelegateWrapper> = + value(strings.asList(), key) { it?.list?.map { value -> value.string } ?: emptyList() } -fun Configurable.numberList(key: Name? = null): ReadWriteDelegateWrapper> = - value(emptyList(), key) { it?.list?.map { value -> value.number } ?: emptyList() } +fun Configurable.numberList(vararg numbers: Number, key: Name? = null): ReadWriteDelegateWrapper> = + value(numbers.asList(), key) { it?.list?.map { value -> value.number } ?: emptyList() } /** * A special delegate for double arrays diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/mapMeta.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/mapMeta.kt index d146ea1f..b0d8162c 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/mapMeta.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/mapMeta.kt @@ -14,6 +14,7 @@ import hep.dataforge.values.Value /** * Convert meta to map of maps */ +@DFExperimental fun Meta.toMap(descriptor: NodeDescriptor? = null): Map { return items.entries.associate { (token, item) -> token.toString() to when (item) { @@ -26,6 +27,7 @@ fun Meta.toMap(descriptor: NodeDescriptor? = null): Map { /** * Convert map of maps to meta */ +@DFExperimental fun Map.toMeta(descriptor: NodeDescriptor? = null): Meta = buildMeta { entries.forEach { (key, value) -> @Suppress("UNCHECKED_CAST") diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/values/Value.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/values/Value.kt index 2fce145f..f044b018 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/values/Value.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/values/Value.kt @@ -41,7 +41,7 @@ interface Value { * get this value represented as List */ val list: List - get() = listOf(this) + get() = if(this == Null) emptyList() else listOf(this) override fun equals(other: Any?): Boolean @@ -60,7 +60,14 @@ interface Value { true -> True false -> False is Number -> value.asValue() - is Iterable<*> -> ListValue(value.map { of(it) }) + is Iterable<*> -> { + val list = value.map { of(it) } + if (list.isEmpty()) { + Null + } else { + ListValue(list) + } + } is DoubleArray -> value.asValue() is IntArray -> value.asValue() is FloatArray -> value.asValue() @@ -174,9 +181,7 @@ class EnumValue>(override val value: E) : Value { class ListValue(override val list: List) : Value { init { - if (list.isEmpty()) { - throw IllegalArgumentException("Can't create list value from empty list") - } + require(list.isNotEmpty()) { "Can't create list value from empty list" } } override val value: List get() = list @@ -208,17 +213,20 @@ fun Boolean.asValue(): Value = if (this) True else False fun String.asValue(): Value = StringValue(this) -fun Iterable.asValue(): Value = ListValue(this.toList()) +fun Iterable.asValue(): Value { + val list = toList() + return if (list.isEmpty()) Null else ListValue(this.toList()) +} -fun IntArray.asValue(): Value = ListValue(map { NumberValue(it) }) +fun IntArray.asValue(): Value = if (isEmpty()) Null else ListValue(map { NumberValue(it) }) -fun LongArray.asValue(): Value = ListValue(map { NumberValue(it) }) +fun LongArray.asValue(): Value = if (isEmpty()) Null else ListValue(map { NumberValue(it) }) -fun ShortArray.asValue(): Value = ListValue(map { NumberValue(it) }) +fun ShortArray.asValue(): Value = if (isEmpty()) Null else ListValue(map { NumberValue(it) }) -fun FloatArray.asValue(): Value = ListValue(map { NumberValue(it) }) +fun FloatArray.asValue(): Value = if (isEmpty()) Null else ListValue(map { NumberValue(it) }) -fun ByteArray.asValue(): Value = ListValue(map { NumberValue(it) }) +fun ByteArray.asValue(): Value = if (isEmpty()) Null else ListValue(map { NumberValue(it) }) /** diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/values/exoticValues.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/values/exoticValues.kt index fa69e8da..d0fb86e8 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/values/exoticValues.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/values/exoticValues.kt @@ -46,4 +46,4 @@ class DoubleArrayValue(override val value: DoubleArray) : Value { override fun toString(): String = list.joinToString (prefix = "[", postfix = "]") } -fun DoubleArray.asValue(): DoubleArrayValue = DoubleArrayValue(this) +fun DoubleArray.asValue(): Value = if(isEmpty()) Null else DoubleArrayValue(this) diff --git a/dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/SpecificationTest.kt b/dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/SpecificationTest.kt new file mode 100644 index 00000000..9098cf18 --- /dev/null +++ b/dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/SpecificationTest.kt @@ -0,0 +1,23 @@ +package hep.dataforge.meta + +import kotlin.test.Test +import kotlin.test.assertEquals + +class SpecificationTest { + class TestSpecific(override val config: Config) : Specific { + var list by numberList(1, 2, 3) + + companion object : Specification { + override fun wrap(config: Config): TestSpecific = TestSpecific(config) + } + } + + + @Test + fun testSpecific(){ + val testObject = TestSpecific.build { + list = emptyList() + } + assertEquals(emptyList(), testObject.list) + } +} \ No newline at end of file From dce9199b7888b967175b74a29f0f5e00b693109d Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Tue, 29 Oct 2019 20:21:38 +0300 Subject: [PATCH 37/48] #18 --- .../kotlin/hep/dataforge/data/Data.kt | 12 +++++----- .../kotlin/hep/dataforge/data/Goal.kt | 8 +++---- .../data/{PipeAction.kt => MapAction.kt} | 16 ++++++------- .../data/{JoinAction.kt => ReduceAction.kt} | 10 ++++---- .../kotlin/hep/dataforge/data/SplitAction.kt | 2 +- .../hep/dataforge/workspace/TaskBuilder.kt | 24 +++++++++---------- .../workspace/SimpleWorkspaceTest.kt | 16 ++++++------- 7 files changed, 44 insertions(+), 44 deletions(-) rename dataforge-data/src/commonMain/kotlin/hep/dataforge/data/{PipeAction.kt => MapAction.kt} (70%) rename dataforge-data/src/commonMain/kotlin/hep/dataforge/data/{JoinAction.kt => ReduceAction.kt} (87%) diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/Data.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/Data.kt index bd93ddfc..e15a5210 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/Data.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/Data.kt @@ -85,7 +85,7 @@ class StaticData( class NamedData(val name: String, data: Data) : Data by data -fun Data.pipe( +fun Data.map( outputType: KClass, coroutineContext: CoroutineContext = EmptyCoroutineContext, meta: Meta = this.meta, @@ -98,7 +98,7 @@ fun Data.pipe( /** * Create a data pipe */ -inline fun Data.pipe( +inline fun Data.map( coroutineContext: CoroutineContext = EmptyCoroutineContext, meta: Meta = this.meta, noinline block: suspend CoroutineScope.(T) -> R @@ -109,7 +109,7 @@ inline fun Data.pipe( /** * Create a joined data. */ -inline fun Collection>.join( +inline fun Collection>.reduce( coroutineContext: CoroutineContext = EmptyCoroutineContext, meta: Meta, noinline block: suspend CoroutineScope.(Collection) -> R @@ -119,10 +119,10 @@ inline fun Collection>.join( coroutineContext, this ) { - block(map { this.run { it.await(this) } }) + block(map { run { it.await(this) } }) } -fun Map>.join( +fun Map>.reduce( outputType: KClass, coroutineContext: CoroutineContext = EmptyCoroutineContext, meta: Meta, @@ -143,7 +143,7 @@ fun Map>.join( * @param T type of the input goal * @param R type of the result goal */ -inline fun Map>.join( +inline fun Map>.reduce( coroutineContext: CoroutineContext = EmptyCoroutineContext, meta: Meta, noinline block: suspend CoroutineScope.(Map) -> R diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/Goal.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/Goal.kt index 54bb743e..8275d31e 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/Goal.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/Goal.kt @@ -85,7 +85,7 @@ open class DynamicGoal( /** * Create a one-to-one goal based on existing goal */ -fun Goal.pipe( +fun Goal.map( coroutineContext: CoroutineContext = EmptyCoroutineContext, block: suspend CoroutineScope.(T) -> R ): Goal = DynamicGoal(coroutineContext, listOf(this)) { @@ -95,11 +95,11 @@ fun Goal.pipe( /** * Create a joining goal. */ -fun Collection>.join( +fun Collection>.reduce( coroutineContext: CoroutineContext = EmptyCoroutineContext, block: suspend CoroutineScope.(Collection) -> R ): Goal = DynamicGoal(coroutineContext, this) { - block(map { this.run { it.await(this) } }) + block(map { run { it.await(this) } }) } /** @@ -108,7 +108,7 @@ fun Collection>.join( * @param T type of the input goal * @param R type of the result goal */ -fun Map>.join( +fun Map>.reduce( coroutineContext: CoroutineContext = EmptyCoroutineContext, block: suspend CoroutineScope.(Map) -> R ): Goal = DynamicGoal(coroutineContext, this.values) { diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/PipeAction.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/MapAction.kt similarity index 70% rename from dataforge-data/src/commonMain/kotlin/hep/dataforge/data/PipeAction.kt rename to dataforge-data/src/commonMain/kotlin/hep/dataforge/data/MapAction.kt index 91a204d8..b184a2a8 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/PipeAction.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/MapAction.kt @@ -10,7 +10,7 @@ class ActionEnv(val name: Name, val meta: Meta) /** * Action environment */ -class PipeBuilder(var name: Name, var meta: MetaBuilder) { +class MapActionBuilder(var name: Name, var meta: MetaBuilder) { lateinit var result: suspend ActionEnv.(T) -> R /** @@ -22,10 +22,10 @@ class PipeBuilder(var name: Name, var meta: MetaBuilder) { } -class PipeAction( +class MapAction( val inputType: KClass, val outputType: KClass, - private val block: PipeBuilder.() -> Unit + private val block: MapActionBuilder.() -> Unit ) : Action { override fun invoke(node: DataNode, meta: Meta): DataNode { @@ -38,12 +38,12 @@ class PipeAction( // creating environment from old meta and name val env = ActionEnv(name, oldMeta) //applying transformation from builder - val builder = PipeBuilder(name, oldMeta).apply(block) + val builder = MapActionBuilder(name, oldMeta).apply(block) //getting new name val newName = builder.name //getting new meta val newMeta = builder.meta.seal() - val newData = data.pipe(outputType, meta = newMeta) { builder.result(env, it) } + val newData = data.map(outputType, meta = newMeta) { builder.result(env, it) } //setting the data node this[newName] = newData } @@ -51,10 +51,10 @@ class PipeAction( } } -inline fun DataNode.pipe( +inline fun DataNode.map( meta: Meta, - noinline action: PipeBuilder.() -> Unit -): DataNode = PipeAction(T::class, R::class, action).invoke(this, meta) + noinline action: MapActionBuilder.() -> Unit +): DataNode = MapAction(T::class, R::class, action).invoke(this, meta) diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/JoinAction.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/ReduceAction.kt similarity index 87% rename from dataforge-data/src/commonMain/kotlin/hep/dataforge/data/JoinAction.kt rename to dataforge-data/src/commonMain/kotlin/hep/dataforge/data/ReduceAction.kt index 7ad4055e..48738452 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/JoinAction.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/ReduceAction.kt @@ -21,7 +21,7 @@ class JoinGroup(var name: String, internal val node: DataNode< } -class JoinGroupBuilder(val actionMeta: Meta) { +class ReduceGroupBuilder(val actionMeta: Meta) { private val groupRules: MutableList<(DataNode) -> List>> = ArrayList(); /** @@ -73,16 +73,16 @@ class JoinGroupBuilder(val actionMeta: Meta) { /** * The same rules as for KPipe */ -class JoinAction( +class ReduceAction( val inputType: KClass, val outputType: KClass, - private val action: JoinGroupBuilder.() -> Unit + private val action: ReduceGroupBuilder.() -> Unit ) : Action { override fun invoke(node: DataNode, meta: Meta): DataNode { node.ensureType(inputType) return DataNode.invoke(outputType) { - JoinGroupBuilder(meta).apply(action).buildGroups(node).forEach { group -> + ReduceGroupBuilder(meta).apply(action).buildGroups(node).forEach { group -> val laminate = Laminate(group.meta, meta) @@ -92,7 +92,7 @@ class JoinAction( val env = ActionEnv(groupName.toName(), laminate.builder()) - val res: DynamicData = dataMap.join(outputType, meta = laminate) { group.result.invoke(env, it) } + val res: DynamicData = dataMap.reduce(outputType, meta = laminate) { group.result.invoke(env, it) } set(env.name, res) } diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/SplitAction.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/SplitAction.kt index 637baeeb..a4f82931 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/SplitAction.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/SplitAction.kt @@ -55,7 +55,7 @@ class SplitAction( rule(env) - val res = data.pipe(outputType, meta = env.meta) { env.result(it) } + val res = data.map(outputType, meta = env.meta) { env.result(it) } set(env.name, res) } } diff --git a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskBuilder.kt b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskBuilder.kt index 150d56a6..80d89e24 100644 --- a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskBuilder.kt +++ b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskBuilder.kt @@ -105,15 +105,15 @@ class TaskBuilder(val name: Name, val type: KClass) { } /** - * A customized pipe action with ability to change meta and name + * A customized map action with ability to change meta and name */ - inline fun customPipe( + inline fun mapAction( from: String = "", to: String = "", - crossinline block: PipeBuilder.(TaskEnv) -> Unit + crossinline block: MapActionBuilder.(TaskEnv) -> Unit ) { action(from, to) { - PipeAction( + MapAction( inputType = T::class, outputType = type ) { block(this@action) } @@ -121,15 +121,15 @@ class TaskBuilder(val name: Name, val type: KClass) { } /** - * A simple pipe action without changing meta or name + * A simple map action without changing meta or name */ - inline fun pipe( + inline fun map( from: String = "", to: String = "", crossinline block: suspend TaskEnv.(T) -> R ) { action(from, to) { - PipeAction( + MapAction( inputType = T::class, outputType = type ) { @@ -144,13 +144,13 @@ class TaskBuilder(val name: Name, val type: KClass) { /** * Join elements in gathered data by multiple groups */ - inline fun joinByGroup( + inline fun reduceByGroup( from: String = "", to: String = "", - crossinline block: JoinGroupBuilder.(TaskEnv) -> Unit //TODO needs KEEP-176 + crossinline block: ReduceGroupBuilder.(TaskEnv) -> Unit //TODO needs KEEP-176 ) { action(from, to) { - JoinAction( + ReduceAction( inputType = T::class, outputType = type ) { block(this@action) } @@ -160,13 +160,13 @@ class TaskBuilder(val name: Name, val type: KClass) { /** * Join all elemlents in gathered data matching input type */ - inline fun join( + inline fun reduce( from: String = "", to: String = "", crossinline block: suspend TaskEnv.(Map) -> R ) { action(from, to) { - JoinAction( + ReduceAction( inputType = T::class, outputType = type, action = { diff --git a/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/SimpleWorkspaceTest.kt b/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/SimpleWorkspaceTest.kt index 3059cb29..4d37f5d9 100644 --- a/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/SimpleWorkspaceTest.kt +++ b/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/SimpleWorkspaceTest.kt @@ -17,7 +17,7 @@ class SimpleWorkspaceTest { override val tag: PluginTag = PluginTag("test") val contextTask = task("test", Any::class) { - pipe { + map { context.logger.info { "Test: $it" } } } @@ -39,13 +39,13 @@ class SimpleWorkspaceTest { model { data("myData\\[12\\]") } - pipe{ + map{ it } } val square = task("square") { - pipe { data -> + map { data -> if (meta["testFlag"].boolean == true) { println("flag") } @@ -55,7 +55,7 @@ class SimpleWorkspaceTest { } val linear = task("linear") { - pipe { data -> + map { data -> context.logger.info { "Starting linear on $data" } data * 2 + 1 } @@ -86,14 +86,14 @@ class SimpleWorkspaceTest { model { dependsOn(square) } - join { data -> + reduce { data -> context.logger.info { "Starting sum" } data.values.sum() } } val average = task("average") { - joinByGroup { env -> + reduceByGroup { env -> group("even", filter = { name, _ -> name.toString().toInt() % 2 == 0 }) { result { data -> env.context.logger.info { "Starting even" } @@ -113,13 +113,13 @@ class SimpleWorkspaceTest { model { dependsOn(average) } - join { data -> + reduce { data -> data["even"]!! - data["odd"]!! } } val customPipeTask = task("custom") { - customPipe { + mapAction { meta = meta.builder().apply { "newValue" to 22 } From 245b5ebc52b9d73f72fbd9c7a8d65632f9f48467 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Tue, 29 Oct 2019 21:20:14 +0300 Subject: [PATCH 38/48] Fix #16 --- .../kotlin/hep/dataforge/context/Context.kt | 4 +- .../kotlin/hep/dataforge/context/Plugin.kt | 6 +- .../kotlin/hep/dataforge/context/PluginTag.kt | 6 +- .../kotlin/hep/dataforge/data/Data.kt | 4 +- .../kotlin/hep/dataforge/data/DataNode.kt | 14 +-- .../hep/dataforge/data/DataTreeBuilderTest.kt | 6 +- .../dataforge/io/yaml/YamlMetaFormatTest.kt | 20 ++-- .../io/functions/RemoteFunctionClient.kt | 2 +- .../hep/dataforge/io/EnvelopeFormatTest.kt | 2 +- .../kotlin/hep/dataforge/io/MetaFormatTest.kt | 20 ++-- .../hep/dataforge/io/MetaSerializerTest.kt | 20 ++-- .../kotlin/hep/dataforge/io/ioFormatsJVM.kt | 6 +- .../hep/dataforge/io/FileEnvelopeTest.kt | 4 +- .../dataforge/io/tcp/EnvelopeServerTest.kt | 2 +- .../dataforge/descriptors/ItemDescriptor.kt | 4 +- .../kotlin/hep/dataforge/meta/Laminate.kt | 2 +- .../kotlin/hep/dataforge/meta/MetaBuilder.kt | 99 ++++++++++++++++--- .../kotlin/hep/dataforge/meta/MutableMeta.kt | 2 +- .../kotlin/hep/dataforge/meta/metaMatcher.kt | 8 +- .../hep/dataforge/values/valueExtensions.kt | 2 +- .../hep/dataforge/meta/MetaBuilderTest.kt | 16 +-- .../hep/dataforge/meta/MetaExtensionTest.kt | 4 +- .../kotlin/hep/dataforge/meta/MetaTest.kt | 22 ++--- .../hep/dataforge/meta/MutableMetaTest.kt | 10 +- .../kotlin/hep/dataforge/meta/StyledTest.kt | 8 +- .../hep/dataforge/meta/DynamicMetaTest.kt | 12 +-- .../hep/dataforge/scripting/BuildersKtTest.kt | 8 +- .../hep/dataforge/workspace/Dependency.kt | 14 +-- .../hep/dataforge/workspace/TaskModel.kt | 6 +- .../dataforge/workspace/WorkspaceBuilder.kt | 2 +- .../workspace/SimpleWorkspaceTest.kt | 4 +- 31 files changed, 204 insertions(+), 135 deletions(-) diff --git a/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/Context.kt b/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/Context.kt index cc79ec44..ffc5b197 100644 --- a/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/Context.kt +++ b/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/Context.kt @@ -105,8 +105,8 @@ open class Context( override fun toMeta(): Meta = buildMeta { "parent" to parent?.name - "properties" to properties.seal() - "plugins" to plugins.map { it.toMeta() } + "properties" put properties.seal() + "plugins" put plugins.map { it.toMeta() } } } diff --git a/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/Plugin.kt b/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/Plugin.kt index d795d881..90d669e0 100644 --- a/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/Plugin.kt +++ b/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/Plugin.kt @@ -66,10 +66,10 @@ interface Plugin : Named, ContextAware, Provider, MetaRepr { fun detach() override fun toMeta(): Meta = buildMeta { - "context" to context.name + "context" put context.name.toString() "type" to this::class.simpleName - "tag" to tag - "meta" to meta + "tag" put tag + "meta" put meta } companion object { diff --git a/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/PluginTag.kt b/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/PluginTag.kt index 25164c72..390de7bc 100644 --- a/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/PluginTag.kt +++ b/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/PluginTag.kt @@ -37,9 +37,9 @@ data class PluginTag( override fun toString(): String = listOf(group, name, version).joinToString(separator = ":") override fun toMeta(): Meta = buildMeta { - "name" to name - "group" to group - "version" to version + "name" put name + "group" put group + "version" put version } companion object { diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/Data.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/Data.kt index e15a5210..718fb46f 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/Data.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/Data.kt @@ -20,9 +20,9 @@ interface Data : Goal, MetaRepr{ val meta: Meta override fun toMeta(): Meta = buildMeta { - "type" to (type.simpleName?:"undefined") + "type" put (type.simpleName?:"undefined") if(!meta.isEmpty()) { - "meta" to meta + "meta" put meta } } diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataNode.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataNode.kt index 53ba081d..12bb06ab 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataNode.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataNode.kt @@ -39,10 +39,10 @@ interface DataNode : MetaRepr { val items: Map> override fun toMeta(): Meta = buildMeta { - "type" to (type.simpleName ?: "undefined") - "items" to { + "type" put (type.simpleName ?: "undefined") + "items" put { this@DataNode.items.forEach { - it.key.toString() to it.value.toMeta() + it.key.toString() put it.value.toMeta() } } } @@ -196,19 +196,19 @@ class DataTreeBuilder(val type: KClass) { /** * Append data to node */ - infix fun String.to(data: Data) = set(toName(), data) + infix fun String.put(data: Data) = set(toName(), data) /** * Append node */ - infix fun String.to(node: DataNode) = set(toName(), node) + infix fun String.put(node: DataNode) = set(toName(), node) - infix fun String.to(item: DataItem) = set(toName(), item) + infix fun String.put(item: DataItem) = set(toName(), item) /** * Build and append node */ - infix fun String.to(block: DataTreeBuilder.() -> Unit) = set(toName(), DataTreeBuilder(type).apply(block)) + infix fun String.put(block: DataTreeBuilder.() -> Unit) = set(toName(), DataTreeBuilder(type).apply(block)) fun update(node: DataNode) { diff --git a/dataforge-data/src/commonTest/kotlin/hep/dataforge/data/DataTreeBuilderTest.kt b/dataforge-data/src/commonTest/kotlin/hep/dataforge/data/DataTreeBuilderTest.kt index c27974bd..32bf1760 100644 --- a/dataforge-data/src/commonTest/kotlin/hep/dataforge/data/DataTreeBuilderTest.kt +++ b/dataforge-data/src/commonTest/kotlin/hep/dataforge/data/DataTreeBuilderTest.kt @@ -8,9 +8,9 @@ internal class DataTreeBuilderTest{ @Test fun testDataUpdate(){ val updateData = DataNode{ - "update" to { - "a" to Data.static("a") - "b" to Data.static("b") + "update" put { + "a" put Data.static("a") + "b" put Data.static("b") } } diff --git a/dataforge-io/dataforge-io-yaml/src/test/kotlin/hep/dataforge/io/yaml/YamlMetaFormatTest.kt b/dataforge-io/dataforge-io-yaml/src/test/kotlin/hep/dataforge/io/yaml/YamlMetaFormatTest.kt index 0827ff1e..414162f7 100644 --- a/dataforge-io/dataforge-io-yaml/src/test/kotlin/hep/dataforge/io/yaml/YamlMetaFormatTest.kt +++ b/dataforge-io/dataforge-io-yaml/src/test/kotlin/hep/dataforge/io/yaml/YamlMetaFormatTest.kt @@ -14,18 +14,18 @@ class YamlMetaFormatTest{ @Test fun testYamlMetaFormat(){ val meta = buildMeta { - "a" to 22 - "node" to { - "b" to "DDD" - "c" to 11.1 - "d" to { - "d1" to { - "d11" to "aaa" - "d12" to "bbb" + "a" put 22 + "node" put { + "b" put "DDD" + "c" put 11.1 + "d" put { + "d1" put { + "d11" put "aaa" + "d12" put "bbb" } - "d2" to 2 + "d2" put 2 } - "array" to doubleArrayOf(1.0, 2.0, 3.0) + "array" put doubleArrayOf(1.0, 2.0, 3.0) } } val string = meta.toString(YamlMetaFormat) diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/functions/RemoteFunctionClient.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/functions/RemoteFunctionClient.kt index 23df23cc..7c294891 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/functions/RemoteFunctionClient.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/functions/RemoteFunctionClient.kt @@ -33,7 +33,7 @@ class RemoteFunctionClient(override val context: Context, val responder: Respond meta(meta) type = REQUEST_TYPE meta { - SIZE_KEY to values.size + SIZE_KEY put values.size } data { val inputFormat: IOFormat = getInputFormat(meta, valueType) diff --git a/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/EnvelopeFormatTest.kt b/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/EnvelopeFormatTest.kt index fca95bbb..29e60f2f 100644 --- a/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/EnvelopeFormatTest.kt +++ b/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/EnvelopeFormatTest.kt @@ -8,7 +8,7 @@ class EnvelopeFormatTest { val envelope = Envelope.invoke { type = "test.format" meta{ - "d" to 22.2 + "d" put 22.2 } data{ writeDouble(22.2) diff --git a/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/MetaFormatTest.kt b/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/MetaFormatTest.kt index 16d946e3..02180dbc 100644 --- a/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/MetaFormatTest.kt +++ b/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/MetaFormatTest.kt @@ -11,11 +11,11 @@ class MetaFormatTest { @Test fun testBinaryMetaFormat() { val meta = buildMeta { - "a" to 22 - "node" to { - "b" to "DDD" - "c" to 11.1 - "array" to doubleArrayOf(1.0, 2.0, 3.0) + "a" put 22 + "node" put { + "b" put "DDD" + "c" put 11.1 + "array" put doubleArrayOf(1.0, 2.0, 3.0) } } val bytes = meta.toBytes(BinaryMetaFormat) @@ -26,11 +26,11 @@ class MetaFormatTest { @Test fun testJsonMetaFormat() { val meta = buildMeta { - "a" to 22 - "node" to { - "b" to "DDD" - "c" to 11.1 - "array" to doubleArrayOf(1.0, 2.0, 3.0) + "a" put 22 + "node" put { + "b" put "DDD" + "c" put 11.1 + "array" put doubleArrayOf(1.0, 2.0, 3.0) } } val string = meta.toString(JsonMetaFormat) diff --git a/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/MetaSerializerTest.kt b/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/MetaSerializerTest.kt index fcf9efb4..ba0ed7c7 100644 --- a/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/MetaSerializerTest.kt +++ b/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/MetaSerializerTest.kt @@ -14,11 +14,11 @@ class MetaSerializerTest { @Test fun testMetaSerialization() { val meta = buildMeta { - "a" to 22 - "node" to { - "b" to "DDD" - "c" to 11.1 - "array" to doubleArrayOf(1.0, 2.0, 3.0) + "a" put 22 + "node" put { + "b" put "DDD" + "c" put 11.1 + "array" put doubleArrayOf(1.0, 2.0, 3.0) } } @@ -30,11 +30,11 @@ class MetaSerializerTest { @Test fun testCborSerialization() { val meta = buildMeta { - "a" to 22 - "node" to { - "b" to "DDD" - "c" to 11.1 - "array" to doubleArrayOf(1.0, 2.0, 3.0) + "a" put 22 + "node" put { + "b" put "DDD" + "c" put 11.1 + "array" put doubleArrayOf(1.0, 2.0, 3.0) } } diff --git a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/ioFormatsJVM.kt b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/ioFormatsJVM.kt index ca3ca17e..c926d07a 100644 --- a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/ioFormatsJVM.kt +++ b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/ioFormatsJVM.kt @@ -26,9 +26,9 @@ fun IOPlugin.resolveIOFormatName(type: KClass<*>): Name { } inline fun IOPlugin.generateFunctionMeta(functionName: String): Meta = buildMeta { - FUNCTION_NAME_KEY to functionName - INPUT_FORMAT_KEY to resolveIOFormatName(T::class) - OUTPUT_FORMAT_KEY to resolveIOFormatName(R::class) + FUNCTION_NAME_KEY put functionName + INPUT_FORMAT_KEY put resolveIOFormatName(T::class).toString() + OUTPUT_FORMAT_KEY put resolveIOFormatName(R::class).toString() } inline fun FunctionServer.function( diff --git a/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/FileEnvelopeTest.kt b/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/FileEnvelopeTest.kt index 280bd1bb..ba7f7cc5 100644 --- a/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/FileEnvelopeTest.kt +++ b/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/FileEnvelopeTest.kt @@ -9,8 +9,8 @@ import kotlin.test.assertTrue class FileEnvelopeTest { val envelope = Envelope { meta { - "a" to "AAA" - "b" to 22.2 + "a" put "AAA" + "b" put 22.2 } dataType = "hep.dataforge.test" dataID = "myData" // добавил только что diff --git a/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/tcp/EnvelopeServerTest.kt b/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/tcp/EnvelopeServerTest.kt index d41ce5be..37c35efc 100644 --- a/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/tcp/EnvelopeServerTest.kt +++ b/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/tcp/EnvelopeServerTest.kt @@ -48,7 +48,7 @@ class EnvelopeServerTest { val request = Envelope.invoke { type = "test.echo" meta { - "test.value" to 22 + "test.value" put 22 } data { writeDouble(22.7) diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/descriptors/ItemDescriptor.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/descriptors/ItemDescriptor.kt index 0a25ff4d..0658dfd8 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/descriptors/ItemDescriptor.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/descriptors/ItemDescriptor.kt @@ -72,7 +72,7 @@ class NodeDescriptor(config: Config) : ItemDescriptor(config) { * The list of value descriptors */ val values: Map - get() = config.getAll(VALUE_KEY.toName()).entries.associate { (name, node) -> + get() = config.getIndexed(VALUE_KEY.toName()).entries.associate { (name, node) -> name to ValueDescriptor.wrap(node.node ?: error("Value descriptor must be a node")) } @@ -93,7 +93,7 @@ class NodeDescriptor(config: Config) : ItemDescriptor(config) { * The map of children node descriptors */ val nodes: Map - get() = config.getAll(NODE_KEY.toName()).entries.associate { (name, node) -> + get() = config.getIndexed(NODE_KEY.toName()).entries.associate { (name, node) -> name to wrap(node.node ?: error("Node descriptor must be a node")) } diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Laminate.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Laminate.kt index 294140ae..b403544c 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Laminate.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Laminate.kt @@ -61,7 +61,7 @@ class Laminate(layers: List) : MetaBase() { } else -> map { when (it) { - is MetaItem.ValueItem -> MetaItem.NodeItem(buildMeta { Meta.VALUE_KEY to it.value }) + is MetaItem.ValueItem -> MetaItem.NodeItem(buildMeta { Meta.VALUE_KEY put it.value }) is MetaItem.NodeItem -> it } }.merge() diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MetaBuilder.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MetaBuilder.kt index 3792491d..671d1f13 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MetaBuilder.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MetaBuilder.kt @@ -2,7 +2,10 @@ package hep.dataforge.meta import hep.dataforge.names.Name import hep.dataforge.names.asName +import hep.dataforge.values.EnumValue import hep.dataforge.values.Value +import hep.dataforge.values.asValue +import kotlin.jvm.JvmName /** * DSL builder for meta. Is not intended to store mutable state @@ -12,41 +15,105 @@ class MetaBuilder : AbstractMutableMeta() { override fun wrapNode(meta: Meta): MetaBuilder = if (meta is MetaBuilder) meta else meta.builder() override fun empty(): MetaBuilder = MetaBuilder() - infix fun String.to(value: Any) { - if (value is Meta) { - this@MetaBuilder[this] = value - } - this@MetaBuilder[this] = Value.of(value) + infix fun String.put(value: Value){ + set(this,value) } - infix fun String.to(meta: Meta) { + infix fun String.put(string: String){ + set(this,string.asValue()) + } + + infix fun String.put(number: Number){ + set(this,number.asValue()) + } + + infix fun String.put(boolean: Boolean){ + set(this, boolean.asValue()) + } + + infix fun String.put(enum: Enum<*>){ + set(this, EnumValue(enum)) + } + + @JvmName("putValues") + infix fun String.put(iterable: Iterable){ + set(this, iterable.asValue()) + } + + @JvmName("putNumbers") + infix fun String.put(iterable: Iterable){ + set(this, iterable.map { it.asValue() }.asValue()) + } + + @JvmName("putStrings") + infix fun String.put(iterable: Iterable){ + set(this, iterable.map { it.asValue() }.asValue()) + } + + infix fun String.put(array: DoubleArray){ + set(this, array.asValue()) + } + + infix fun String.putValue(any: Any?){ + set(this, Value.of(any)) + } + + infix fun String.put(meta: Meta) { this@MetaBuilder[this] = meta } - infix fun String.to(value: Iterable) { + infix fun String.put(repr: MetaRepr){ + set(this,repr.toMeta()) + } + + @JvmName("putMetas") + infix fun String.put(value: Iterable) { this@MetaBuilder[this] = value.toList() } - infix fun String.to(metaBuilder: MetaBuilder.() -> Unit) { + infix fun String.put(metaBuilder: MetaBuilder.() -> Unit) { this@MetaBuilder[this] = MetaBuilder().apply(metaBuilder) } - infix fun Name.to(value: Any) { - if (value is Meta) { - this@MetaBuilder[this] = value - } - this@MetaBuilder[this] = Value.of(value) + infix fun Name.put(value: Value){ + set(this,value) } - infix fun Name.to(meta: Meta) { + infix fun Name.put(string: String){ + set(this,string.asValue()) + } + + infix fun Name.put(number: Number){ + set(this,number.asValue()) + } + + infix fun Name.put(boolean: Boolean){ + set(this, boolean.asValue()) + } + + infix fun Name.put(enum: Enum<*>){ + set(this, EnumValue(enum)) + } + + @JvmName("putValues") + infix fun Name.put(iterable: Iterable){ + set(this, iterable.asValue()) + } + + infix fun Name.put(meta: Meta) { this@MetaBuilder[this] = meta } - infix fun Name.to(value: Iterable) { + infix fun Name.put(repr: MetaRepr){ + set(this,repr.toMeta()) + } + + @JvmName("putMetas") + infix fun Name.put(value: Iterable) { this@MetaBuilder[this] = value.toList() } - infix fun Name.to(metaBuilder: MetaBuilder.() -> Unit) { + infix fun Name.put(metaBuilder: MetaBuilder.() -> Unit) { this@MetaBuilder[this] = MetaBuilder().apply(metaBuilder) } } diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MutableMeta.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MutableMeta.kt index 285edd89..4da08ce4 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MutableMeta.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MutableMeta.kt @@ -164,7 +164,7 @@ fun MutableMeta<*>.append(name: Name, value: Any?) { if (newIndex.isNotEmpty()) { set(name, value) } else { - val index = (getAll(name).keys.mapNotNull { it.toIntOrNull() }.max() ?: -1) + 1 + val index = (getIndexed(name).keys.mapNotNull { it.toIntOrNull() }.max() ?: -1) + 1 set(name.withIndex(index.toString()), value) } } diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/metaMatcher.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/metaMatcher.kt index 8e2ea991..6f16f537 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/metaMatcher.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/metaMatcher.kt @@ -7,7 +7,7 @@ import hep.dataforge.names.toName * Get all items matching given name. */ @DFExperimental -fun Meta.getAll(name: Name): Map> { +fun Meta.getIndexed(name: Name): Map> { val root = when (name.length) { 0 -> error("Can't use empty name for that") 1 -> this @@ -24,14 +24,14 @@ fun Meta.getAll(name: Name): Map> { } @DFExperimental -fun Meta.getAll(name: String): Map> = getAll(name.toName()) +fun Meta.getIndexed(name: String): Map> = this@getIndexed.getIndexed(name.toName()) /** * Get all items matching given name. */ @DFExperimental -fun > M.getAll(name: Name): Map> { +fun > M.getIndexed(name: Name): Map> { val root: MetaNode? = when (name.length) { 0 -> error("Can't use empty name for that") 1 -> this @@ -48,4 +48,4 @@ fun > M.getAll(name: Name): Map> { } @DFExperimental -fun > M.getAll(name: String): Map> = getAll(name.toName()) \ No newline at end of file +fun > M.getIndexed(name: String): Map> = getIndexed(name.toName()) \ No newline at end of file diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/values/valueExtensions.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/values/valueExtensions.kt index 41a5bda3..a63d5ec1 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/values/valueExtensions.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/values/valueExtensions.kt @@ -34,4 +34,4 @@ val Value.doubleArray: DoubleArray } -fun Value.toMeta() = buildMeta { Meta.VALUE_KEY to this } \ No newline at end of file +fun Value.toMeta() = buildMeta { Meta.VALUE_KEY put this } \ No newline at end of file diff --git a/dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/MetaBuilderTest.kt b/dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/MetaBuilderTest.kt index 849a34fc..3fe10ceb 100644 --- a/dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/MetaBuilderTest.kt +++ b/dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/MetaBuilderTest.kt @@ -9,13 +9,13 @@ class MetaBuilderTest { @Test fun testBuilder() { val meta = buildMeta { - "a" to 22 - "b" to listOf(1, 2, 3) + "a" put 22 + "b" put listOf(1, 2, 3) this["c"] = "myValue".asValue() - "node" to { - "e" to 12.2 - "childNode" to { - "f" to true + "node" put { + "e" put 12.2 + "childNode" put { + "f" put true } } } @@ -27,12 +27,12 @@ class MetaBuilderTest { fun testSNS(){ val meta = buildMeta { repeat(10){ - "b.a[$it]" to it + "b.a[$it]" put it } }.seal() assertEquals(10, meta.values().count()) - val nodes = meta.getAll("b.a") + val nodes = meta.getIndexed("b.a") assertEquals(3, nodes["3"]?.int) } diff --git a/dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/MetaExtensionTest.kt b/dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/MetaExtensionTest.kt index 30b6df18..f2fffd19 100644 --- a/dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/MetaExtensionTest.kt +++ b/dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/MetaExtensionTest.kt @@ -11,12 +11,12 @@ class MetaExtensionTest { @Test fun testEnum(){ - val meta = buildMeta{"enum" to TestEnum.test} + val meta = buildMeta{"enum" put TestEnum.test} meta["enum"].enum() } @Test fun testEnumByString(){ - val meta = buildMeta{"enum" to TestEnum.test.name} + val meta = buildMeta{"enum" put TestEnum.test.name} println(meta["enum"].enum()) } diff --git a/dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/MetaTest.kt b/dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/MetaTest.kt index 7528f8d2..fb424116 100644 --- a/dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/MetaTest.kt +++ b/dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/MetaTest.kt @@ -17,16 +17,16 @@ class MetaTest { @Test fun metaEqualityTest() { val meta1 = buildMeta { - "a" to 22 - "b" to { - "c" to "ddd" + "a" put 22 + "b" put { + "c" put "ddd" } } val meta2 = buildMeta { - "b" to { - "c" to "ddd" + "b" put { + "c" put "ddd" } - "a" to 22 + "a" put 22 }.seal() assertEquals(meta1, meta2) } @@ -34,13 +34,13 @@ class MetaTest { @Test fun metaToMap(){ val meta = buildMeta { - "a" to 22 - "b" to { - "c" to "ddd" + "a" put 22 + "b" put { + "c" put "ddd" } - "list" to (0..4).map { + "list" put (0..4).map { buildMeta { - "value" to it + "value" put it } } } diff --git a/dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/MutableMetaTest.kt b/dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/MutableMetaTest.kt index 5ab75fd4..9057782f 100644 --- a/dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/MutableMetaTest.kt +++ b/dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/MutableMetaTest.kt @@ -7,12 +7,12 @@ class MutableMetaTest{ @Test fun testRemove(){ val meta = buildMeta { - "aNode" to { - "innerNode" to { - "innerValue" to true + "aNode" put { + "innerNode" put { + "innerValue" put true } - "b" to 22 - "c" to "StringValue" + "b" put 22 + "c" put "StringValue" } }.toConfig() diff --git a/dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/StyledTest.kt b/dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/StyledTest.kt index 5a4699f9..a4cbe18e 100644 --- a/dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/StyledTest.kt +++ b/dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/StyledTest.kt @@ -9,8 +9,8 @@ class StyledTest{ fun testSNS(){ val meta = buildMeta { repeat(10){ - "b.a[$it]" to { - "d" to it + "b.a[$it]" put { + "d" put it } } }.seal().withStyle() @@ -18,9 +18,9 @@ class StyledTest{ val bNode = meta["b"].node - val aNodes = bNode?.getAll("a") + val aNodes = bNode?.getIndexed("a") - val allNodes = meta.getAll("b.a") + val allNodes = meta.getIndexed("b.a") assertEquals(3, aNodes?.get("3").node["d"].int) assertEquals(3, allNodes["3"].node["d"].int) diff --git a/dataforge-meta/src/jsTest/kotlin/hep/dataforge/meta/DynamicMetaTest.kt b/dataforge-meta/src/jsTest/kotlin/hep/dataforge/meta/DynamicMetaTest.kt index 1614addc..71eca4f5 100644 --- a/dataforge-meta/src/jsTest/kotlin/hep/dataforge/meta/DynamicMetaTest.kt +++ b/dataforge-meta/src/jsTest/kotlin/hep/dataforge/meta/DynamicMetaTest.kt @@ -27,12 +27,12 @@ class DynamicMetaTest { @Test fun testMetaToDynamic(){ val meta = buildMeta { - "a" to 22 - "array" to arrayOf(1, 2, 3) - "b" to "myString" - "ob" to { - "childNode" to 18 - "booleanNode" to true + "a" put 22 + "array" put listOf(1, 2, 3) + "b" put "myString" + "ob" put { + "childNode" put 18 + "booleanNode" put true } } diff --git a/dataforge-scripting/src/jvmTest/kotlin/hep/dataforge/scripting/BuildersKtTest.kt b/dataforge-scripting/src/jvmTest/kotlin/hep/dataforge/scripting/BuildersKtTest.kt index bcc357d4..5a9ba56d 100644 --- a/dataforge-scripting/src/jvmTest/kotlin/hep/dataforge/scripting/BuildersKtTest.kt +++ b/dataforge-scripting/src/jvmTest/kotlin/hep/dataforge/scripting/BuildersKtTest.kt @@ -3,7 +3,9 @@ package hep.dataforge.scripting import hep.dataforge.context.Global import hep.dataforge.meta.get import hep.dataforge.meta.int -import hep.dataforge.workspace.* +import hep.dataforge.workspace.SimpleWorkspaceBuilder +import hep.dataforge.workspace.context +import hep.dataforge.workspace.target import org.junit.Test import kotlin.test.assertEquals @@ -17,7 +19,7 @@ class BuildersKtTest { context("test") target("testTarget"){ - "a" to 12 + "a" put 12 } } } @@ -30,7 +32,7 @@ class BuildersKtTest { context("test") target("testTarget"){ - "a" to 12 + "a" put 12 } """.trimIndent() val workspace = Builders.buildWorkspace(script) diff --git a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/Dependency.kt b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/Dependency.kt index 33b7c8d2..ddb53d5d 100644 --- a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/Dependency.kt +++ b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/Dependency.kt @@ -26,8 +26,8 @@ class DataDependency(val filter: DataFilter, val placement: Name = EmptyName) : } override fun toMeta(): Meta = buildMeta { - "data" to filter.config - "to" to placement + "data" put filter.config + "to" put placement.toString() } } @@ -39,8 +39,8 @@ class AllDataDependency(val placement: Name = EmptyName) : Dependency() { } override fun toMeta() = buildMeta { - "data" to "@all" - "to" to placement + "data" put "@all" + "to" put placement.toString() } } @@ -67,9 +67,9 @@ abstract class TaskDependency( } override fun toMeta(): Meta = buildMeta { - "task" to name - "meta" to meta - "to" to placement + "task" put name.toString() + "meta" put meta + "to" put placement.toString() } } diff --git a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskModel.kt b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskModel.kt index f71f01a0..b4ccb7ae 100644 --- a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskModel.kt +++ b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskModel.kt @@ -31,9 +31,9 @@ data class TaskModel( //TODO add pre-run check of task result type? override fun toMeta(): Meta = buildMeta { - "name" to name - "meta" to meta - "dependsOn" to { + "name" put name.toString() + "meta" put meta + "dependsOn" put { val dataDependencies = dependencies.filterIsInstance() val taskDependencies = dependencies.filterIsInstance>() setIndexed("data".toName(), dataDependencies.map { it.toMeta() }) diff --git a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/WorkspaceBuilder.kt b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/WorkspaceBuilder.kt index 75a660cd..2f717f78 100644 --- a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/WorkspaceBuilder.kt +++ b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/WorkspaceBuilder.kt @@ -62,7 +62,7 @@ fun WorkspaceBuilder.target(name: String, block: MetaBuilder.() -> Unit) { fun WorkspaceBuilder.target(name: String, base: String, block: MetaBuilder.() -> Unit) { val parentTarget = targets[base] ?: error("Base target with name $base not found") targets[name] = parentTarget.builder() - .apply { "@baseTarget" to base } + .apply { "@baseTarget" put base } .apply(block) .seal() } diff --git a/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/SimpleWorkspaceTest.kt b/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/SimpleWorkspaceTest.kt index 4d37f5d9..3a40e783 100644 --- a/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/SimpleWorkspaceTest.kt +++ b/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/SimpleWorkspaceTest.kt @@ -121,7 +121,7 @@ class SimpleWorkspaceTest { val customPipeTask = task("custom") { mapAction { meta = meta.builder().apply { - "newValue" to 22 + "newValue" put 22 } name += "new" result { @@ -142,7 +142,7 @@ class SimpleWorkspaceTest { @Test fun testMetaPropagation() { - val node = workspace.run("sum") { "testFlag" to true } + val node = workspace.run("sum") { "testFlag" put true } val res = node.first()?.get() } From 84c5f6d925e7c0c78693d558fe4eb86eb2dacc6c Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Tue, 29 Oct 2019 21:24:45 +0300 Subject: [PATCH 39/48] Version to 0.1.4 --- build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle.kts b/build.gradle.kts index eb287fa1..04cce594 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -4,7 +4,7 @@ plugins { id("scientifik.publish") version "0.2.1" apply false } -val dataforgeVersion by extra("0.1.4-dev-8") +val dataforgeVersion by extra("0.1.4") val bintrayRepo by extra("dataforge") val githubProject by extra("dataforge-core") From ade4eec96ea19f07ef833602412716e43a6f1cdc Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Wed, 30 Oct 2019 17:13:31 +0300 Subject: [PATCH 40/48] Make putted values optional in MetaBuilder --- .../kotlin/hep/dataforge/meta/MetaBuilder.kt | 60 +++++++++---------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MetaBuilder.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MetaBuilder.kt index 671d1f13..c5ad3831 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MetaBuilder.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MetaBuilder.kt @@ -15,55 +15,55 @@ class MetaBuilder : AbstractMutableMeta() { override fun wrapNode(meta: Meta): MetaBuilder = if (meta is MetaBuilder) meta else meta.builder() override fun empty(): MetaBuilder = MetaBuilder() - infix fun String.put(value: Value){ - set(this,value) + infix fun String.put(value: Value?) { + set(this, value) } - infix fun String.put(string: String){ - set(this,string.asValue()) + infix fun String.put(string: String?) { + set(this, string?.asValue()) } - infix fun String.put(number: Number){ - set(this,number.asValue()) + infix fun String.put(number: Number?) { + set(this, number?.asValue()) } - infix fun String.put(boolean: Boolean){ - set(this, boolean.asValue()) + infix fun String.put(boolean: Boolean?) { + set(this, boolean?.asValue()) } - infix fun String.put(enum: Enum<*>){ + infix fun String.put(enum: Enum<*>) { set(this, EnumValue(enum)) } @JvmName("putValues") - infix fun String.put(iterable: Iterable){ + infix fun String.put(iterable: Iterable) { set(this, iterable.asValue()) } @JvmName("putNumbers") - infix fun String.put(iterable: Iterable){ + infix fun String.put(iterable: Iterable) { set(this, iterable.map { it.asValue() }.asValue()) } @JvmName("putStrings") - infix fun String.put(iterable: Iterable){ + infix fun String.put(iterable: Iterable) { set(this, iterable.map { it.asValue() }.asValue()) } - infix fun String.put(array: DoubleArray){ + infix fun String.put(array: DoubleArray) { set(this, array.asValue()) } - infix fun String.putValue(any: Any?){ + infix fun String.putValue(any: Any?) { set(this, Value.of(any)) } - infix fun String.put(meta: Meta) { + infix fun String.put(meta: Meta?) { this@MetaBuilder[this] = meta } - infix fun String.put(repr: MetaRepr){ - set(this,repr.toMeta()) + infix fun String.put(repr: MetaRepr?) { + set(this, repr?.toMeta()) } @JvmName("putMetas") @@ -75,37 +75,37 @@ class MetaBuilder : AbstractMutableMeta() { this@MetaBuilder[this] = MetaBuilder().apply(metaBuilder) } - infix fun Name.put(value: Value){ - set(this,value) + infix fun Name.put(value: Value?) { + set(this, value) } - infix fun Name.put(string: String){ - set(this,string.asValue()) + infix fun Name.put(string: String?) { + set(this, string?.asValue()) } - infix fun Name.put(number: Number){ - set(this,number.asValue()) + infix fun Name.put(number: Number?) { + set(this, number?.asValue()) } - infix fun Name.put(boolean: Boolean){ - set(this, boolean.asValue()) + infix fun Name.put(boolean: Boolean?) { + set(this, boolean?.asValue()) } - infix fun Name.put(enum: Enum<*>){ + infix fun Name.put(enum: Enum<*>) { set(this, EnumValue(enum)) } @JvmName("putValues") - infix fun Name.put(iterable: Iterable){ + infix fun Name.put(iterable: Iterable) { set(this, iterable.asValue()) } - infix fun Name.put(meta: Meta) { + infix fun Name.put(meta: Meta?) { this@MetaBuilder[this] = meta } - infix fun Name.put(repr: MetaRepr){ - set(this,repr.toMeta()) + infix fun Name.put(repr: MetaRepr?) { + set(this, repr?.toMeta()) } @JvmName("putMetas") From 33c8e7088e3e0b22e56e1ac7cbca5ab1b4ac401d Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Wed, 30 Oct 2019 19:43:26 +0300 Subject: [PATCH 41/48] Fixed leaked byte packets in io --- .../kotlin/hep/dataforge/io/Binary.kt | 11 ++-- .../kotlin/hep/dataforge/io/MetaFormat.kt | 2 +- .../hep/dataforge/io/TaggedEnvelopeFormat.kt | 11 ++-- .../hep/dataforge/io/TaglessEnvelopeFormat.kt | 10 +++- .../kotlin/hep/dataforge/io/FileBinary.kt | 4 +- .../kotlin/hep/dataforge/io/FileEnvelope.kt | 2 +- .../kotlin/hep/dataforge/io/FileBinaryTest.kt | 56 +++++++++++++++++++ 7 files changed, 79 insertions(+), 17 deletions(-) create mode 100644 dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/FileBinaryTest.kt diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Binary.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Binary.kt index 252de7bc..ca05de4d 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Binary.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Binary.kt @@ -1,9 +1,6 @@ package hep.dataforge.io -import kotlinx.io.core.ByteReadPacket -import kotlinx.io.core.Input -import kotlinx.io.core.buildPacket -import kotlinx.io.core.readBytes +import kotlinx.io.core.* import kotlin.math.min /** @@ -45,7 +42,7 @@ fun Binary.toBytes(): ByteArray = read { @ExperimentalUnsignedTypes fun RandomAccessBinary.readPacket(from: UInt, size: UInt): ByteReadPacket = read(from, size) { - ByteReadPacket(this.readBytes()) + buildPacket { copyTo(this) } } @ExperimentalUnsignedTypes @@ -65,7 +62,9 @@ inline class ArrayBinary(val array: ByteArray) : RandomAccessBinary { override fun read(from: UInt, size: UInt, block: Input.() -> R): R { val theSize = min(size, array.size.toUInt() - from) - return ByteReadPacket(array, from.toInt(), theSize.toInt()).block() + return buildPacket { + writeFully(array, from.toInt(), theSize.toInt()) + }.block() } } diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/MetaFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/MetaFormat.kt index 313d04ae..ca9a53a2 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/MetaFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/MetaFormat.kt @@ -52,7 +52,7 @@ fun Meta.toBytes(format: MetaFormat = JsonMetaFormat.default): ByteReadPacket = } fun MetaFormat.parse(str: String): Meta { - return ByteReadPacket(str.toByteArray()).readObject() + return buildPacket { writeText(str) }.readObject() } fun MetaFormatFactory.parse(str: String): Meta = invoke().parse(str) diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt index b69a5f19..c51cb36d 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt @@ -7,6 +7,7 @@ import hep.dataforge.meta.string import hep.dataforge.names.Name import hep.dataforge.names.plus import hep.dataforge.names.toName +import kotlinx.io.charsets.Charsets import kotlinx.io.core.* @ExperimentalUnsignedTypes @@ -30,10 +31,12 @@ class TaggedEnvelopeFormat( override fun Output.writeObject(obj: Envelope) { val metaBytes = metaFormat.writeBytes(obj.meta) - val tag = Tag(metaFormatKey, metaBytes.size.toUInt(), obj.data?.size ?: 0.toULong()) + val tag = Tag(metaFormatKey, metaBytes.size.toUInt() + 2u, obj.data?.size ?: 0.toULong()) writePacket(tag.toBytes()) writeFully(metaBytes) + writeText("\r\n") obj.data?.read { copyTo(this@writeObject) } + flush() } /** @@ -92,14 +95,14 @@ class TaggedEnvelopeFormat( } private fun Input.readTag(): Tag { - val start = readTextExactBytes(2) + val start = readTextExactBytes(2, charset = Charsets.ISO_8859_1) if (start != START_SEQUENCE) error("The input is not an envelope") - val version = readTextExactBytes(4) + val version = readTextExactBytes(4, charset = Charsets.ISO_8859_1) if (version != VERSION) error("Wrong version of DataForge: expected $VERSION but found $version") val metaFormatKey = readShort() val metaLength = readUInt() val dataLength = readULong() - val end = readTextExactBytes(4) + val end = readTextExactBytes(4, charset = Charsets.ISO_8859_1) if (end != END_SEQUENCE) error("The input is not an envelope") return Tag(metaFormatKey, metaLength, dataLength) } diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaglessEnvelopeFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaglessEnvelopeFormat.kt index 83fbe46b..8e7c9e2b 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaglessEnvelopeFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaglessEnvelopeFormat.kt @@ -72,7 +72,9 @@ class TaglessEnvelopeFormat( val metaFormat = properties[META_TYPE_PROPERTY]?.let { io.metaFormat(it) } ?: JsonMetaFormat.default val metaSize = properties.get(META_LENGTH_PROPERTY)?.toInt() meta = if (metaSize != null) { - val metaPacket = ByteReadPacket(readBytes(metaSize)) + val metaPacket = buildPacket { + writeFully(readBytes(metaSize)) + } metaFormat.run { metaPacket.readObject() } } else { metaFormat.run { @@ -126,11 +128,13 @@ class TaglessEnvelopeFormat( val metaSize = properties.get(META_LENGTH_PROPERTY)?.toInt() meta = if (metaSize != null) { - val metaPacket = ByteReadPacket(readBytes(metaSize)) + val metaPacket = buildPacket { + writeFully(readBytes(metaSize)) + } offset += metaSize.toUInt() metaFormat.run { metaPacket.readObject() } } else { - error("Can't partially read an envelope with undefined meta size") + error("Can't partially read an envelope with undefined meta size") } } diff --git a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileBinary.kt b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileBinary.kt index 2c357a47..b935a103 100644 --- a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileBinary.kt +++ b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileBinary.kt @@ -1,7 +1,7 @@ package hep.dataforge.io -import kotlinx.io.core.ByteReadPacket import kotlinx.io.core.Input +import kotlinx.io.core.buildPacket import java.nio.channels.FileChannel import java.nio.file.Files import java.nio.file.Path @@ -17,7 +17,7 @@ class FileBinary(val path: Path, private val offset: UInt = 0u, size: ULong? = n FileChannel.open(path, StandardOpenOption.READ).use { val theSize: UInt = min(size, Files.size(path).toUInt() - offset) val buffer = it.map(FileChannel.MapMode.READ_ONLY, (from + offset).toLong(), theSize.toLong()) - return ByteReadPacket(buffer).block() + return buildPacket { writeFully(buffer) }.block() } } } diff --git a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileEnvelope.kt b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileEnvelope.kt index dcc8685d..3187cd54 100644 --- a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileEnvelope.kt +++ b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileEnvelope.kt @@ -15,7 +15,7 @@ class FileEnvelope internal constructor(val path: Path, val format: EnvelopeForm init { val input = Files.newByteChannel(path, StandardOpenOption.READ).asInput() - partialEnvelope = format.run { input.readPartial() } + partialEnvelope = format.run { input.use { it.readPartial()} } } override val meta: Meta get() = partialEnvelope.meta diff --git a/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/FileBinaryTest.kt b/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/FileBinaryTest.kt new file mode 100644 index 00000000..94403dcd --- /dev/null +++ b/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/FileBinaryTest.kt @@ -0,0 +1,56 @@ +package hep.dataforge.io + +import hep.dataforge.context.Global +import java.nio.file.Files +import kotlin.test.Test +import kotlin.test.assertEquals + +class FileBinaryTest { + val envelope = Envelope { + meta { + "a" put "AAA" + "b" put 22.2 + } + dataType = "hep.dataforge.test" + dataID = "myData" // добавил только что + data { + writeDouble(16.7) + } + } + + @Test + fun testSize() { + val binary = envelope.data + assertEquals(binary?.size?.toInt(), binary?.toBytes()?.size) + } + + @Test + fun testFileData(){ + val dataFile = Files.createTempFile("dataforge_test_bin", ".bin") + dataFile.toFile().writeText("This is my binary") + val envelopeFromFile = Envelope { + meta { + "a" put "AAA" + "b" put 22.2 + } + dataType = "hep.dataforge.satellite" + dataID = "cellDepositTest" // добавил только что + data = dataFile.asBinary() + } + val binary = envelopeFromFile.data!! + println(binary.toBytes().size) + assertEquals(binary.size.toInt(), binary.toBytes().size) + + } + + @Test + fun testFileDataSizeRewriting() { + println(System.getProperty("user.dir")) + val tmpPath = Files.createTempFile("dataforge_test", ".df") + Global.io.writeEnvelopeFile(tmpPath, envelope) + + val binary = Global.io.readEnvelopeFile(tmpPath).data!! + assertEquals(binary.size.toInt(), binary.toBytes().size) + } + +} \ No newline at end of file From 0ef0a430773c3a3279f05484daf4196491cd21a8 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Wed, 30 Oct 2019 19:57:03 +0300 Subject: [PATCH 42/48] Added check for file size in FileBinary --- .../src/jvmMain/kotlin/hep/dataforge/io/FileBinary.kt | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileBinary.kt b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileBinary.kt index b935a103..aa90a638 100644 --- a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileBinary.kt +++ b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileBinary.kt @@ -13,6 +13,12 @@ class FileBinary(val path: Path, private val offset: UInt = 0u, size: ULong? = n override val size: ULong = size ?: (Files.size(path).toULong() - offset).toULong() + init { + if( size != null && Files.size(path) < offset.toLong() + size.toLong()){ + error("Can't read binary from file. File is to short.") + } + } + override fun read(from: UInt, size: UInt, block: Input.() -> R): R { FileChannel.open(path, StandardOpenOption.READ).use { val theSize: UInt = min(size, Files.size(path).toUInt() - offset) From 6e98bc7408e7acc14df9b67736c6cb4306e2b5c4 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Thu, 31 Oct 2019 10:21:52 +0300 Subject: [PATCH 43/48] Added descriptors to custom serializers --- .../io/serialization/MetaSerializer.kt | 13 ++-- .../io/serialization/serializationUtils.kt | 59 ++++++++++++++++--- .../hep/dataforge/io/MetaSerializerTest.kt | 5 ++ 3 files changed, 64 insertions(+), 13 deletions(-) diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/serialization/MetaSerializer.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/serialization/MetaSerializer.kt index 2f6a00de..271f95e2 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/serialization/MetaSerializer.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/serialization/MetaSerializer.kt @@ -14,12 +14,12 @@ import kotlinx.serialization.json.JsonOutput @Serializer(Value::class) object ValueSerializer : KSerializer { private val valueTypeSerializer = EnumSerializer(ValueType::class) - private val listSerializer by lazy { ArrayListSerializer(ValueSerializer)} + private val listSerializer by lazy { ArrayListSerializer(ValueSerializer) } override val descriptor: SerialDescriptor = descriptor("Value") { - element("isList") - element("valueType") - element("value") + boolean("isList") + enum("valueType") + element("value", PolymorphicClassDescriptor) } private fun Decoder.decodeValue(): Value { @@ -68,10 +68,11 @@ object ValueSerializer : KSerializer { @Serializer(MetaItem::class) object MetaItemSerializer : KSerializer> { override val descriptor: SerialDescriptor = descriptor("MetaItem") { - element("isNode") - element("value") + boolean("isNode") + element("value", PolymorphicClassDescriptor) } + override fun deserialize(decoder: Decoder): MetaItem<*> { val isNode = decoder.decodeBoolean() return if (isNode) { diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/serialization/serializationUtils.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/serialization/serializationUtils.kt index f83b2197..4d33fad9 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/serialization/serializationUtils.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/serialization/serializationUtils.kt @@ -4,19 +4,64 @@ import kotlinx.serialization.CompositeDecoder import kotlinx.serialization.Decoder import kotlinx.serialization.KSerializer import kotlinx.serialization.SerialDescriptor -import kotlinx.serialization.internal.SerialClassDescImpl +import kotlinx.serialization.internal.* +/** + * A convenience builder for serial descriptors + */ inline class SerialDescriptorBuilder(private val impl: SerialClassDescImpl) { - fun element(name: String, isOptional: Boolean = false) = impl.addElement(name, isOptional) + fun element( + name: String, + descriptor: SerialDescriptor, + isOptional: Boolean = false, + vararg annotations: Annotation + ) { + impl.addElement(name, isOptional) + impl.pushDescriptor(descriptor) + annotations.forEach { + impl.pushAnnotation(it) + } + } - fun annotation(a: Annotation) = impl.pushAnnotation(a) + fun element( + name: String, + isOptional: Boolean = false, + vararg annotations: Annotation, + block: SerialDescriptorBuilder.() -> Unit + ) { + impl.addElement(name, isOptional) + impl.pushDescriptor(SerialDescriptorBuilder(SerialClassDescImpl(name)).apply(block).build()) + annotations.forEach { + impl.pushAnnotation(it) + } + } + + fun boolean(name: String, isOptional: Boolean = false, vararg annotations: Annotation) = + element(name, BooleanDescriptor, isOptional, *annotations) + + fun string(name: String, isOptional: Boolean = false, vararg annotations: Annotation) = + element(name, StringDescriptor, isOptional, *annotations) + + fun int(name: String, isOptional: Boolean = false, vararg annotations: Annotation) = + element(name, IntDescriptor, isOptional, *annotations) + + fun double(name: String, isOptional: Boolean = false, vararg annotations: Annotation) = + element(name, DoubleDescriptor, isOptional, *annotations) + + fun float(name: String, isOptional: Boolean = false, vararg annotations: Annotation) = + element(name, FloatDescriptor, isOptional, *annotations) + + fun long(name: String, isOptional: Boolean = false, vararg annotations: Annotation) = + element(name, LongDescriptor, isOptional, *annotations) + + fun doubleArray(name: String, isOptional: Boolean = false, vararg annotations: Annotation) = + element(name, DoubleArraySerializer.descriptor, isOptional, *annotations) + + inline fun > enum(name: String, isOptional: Boolean = false, vararg annotations: Annotation) = + element(name, EnumSerializer(E::class).descriptor, isOptional, *annotations) fun classAnnotation(a: Annotation) = impl.pushClassAnnotation(a) - fun descriptor(name: String, block: SerialDescriptorBuilder.() -> Unit) = impl.pushDescriptor( - SerialDescriptorBuilder(SerialClassDescImpl(name)).apply(block).build() - ) - fun build(): SerialDescriptor = impl } diff --git a/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/MetaSerializerTest.kt b/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/MetaSerializerTest.kt index ba0ed7c7..f33ad894 100644 --- a/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/MetaSerializerTest.kt +++ b/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/MetaSerializerTest.kt @@ -51,4 +51,9 @@ class MetaSerializerTest { val restored = Json.plain.parse(NameSerializer, string) assertEquals(restored, name) } + + @Test + fun testMetaItemDescriptor(){ + val descriptor = MetaItemSerializer.descriptor.getElementDescriptor(0) + } } \ No newline at end of file From 592f911db7a8793d4e7e1f38c35071287e4edec6 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Fri, 1 Nov 2019 14:41:16 +0300 Subject: [PATCH 44/48] Changed map and reduce action API to include both meta from data and meta from task/action --- .../kotlin/hep/dataforge/data/MapAction.kt | 27 +++++++++++++------ .../kotlin/hep/dataforge/data/ReduceAction.kt | 15 ++++++----- 2 files changed, 28 insertions(+), 14 deletions(-) diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/MapAction.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/MapAction.kt index b184a2a8..ea24ca77 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/MapAction.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/MapAction.kt @@ -1,16 +1,19 @@ package hep.dataforge.data -import hep.dataforge.meta.* +import hep.dataforge.meta.Meta +import hep.dataforge.meta.MetaBuilder +import hep.dataforge.meta.builder +import hep.dataforge.meta.seal import hep.dataforge.names.Name import kotlin.reflect.KClass -class ActionEnv(val name: Name, val meta: Meta) +data class ActionEnv(val name: Name, val actionMeta: Meta, val meta: Meta) /** * Action environment */ -class MapActionBuilder(var name: Name, var meta: MetaBuilder) { +class MapActionBuilder(var name: Name, var meta: MetaBuilder, val actionMeta: Meta) { lateinit var result: suspend ActionEnv.(T) -> R /** @@ -33,16 +36,24 @@ class MapAction( return DataNode.invoke(outputType) { node.dataSequence().forEach { (name, data) -> - //merging data meta with action meta (data meta is primary) - val oldMeta = meta.builder().apply { update(data.meta) } - // creating environment from old meta and name - val env = ActionEnv(name, oldMeta) + /* + * Creating a new environment for action using **old** name, old meta and task meta + */ + val env = ActionEnv(name, meta, data.meta) + //applying transformation from builder - val builder = MapActionBuilder(name, oldMeta).apply(block) + val builder = MapActionBuilder( + name, + data.meta.builder(), // using data meta + meta + ).apply(block) + //getting new name val newName = builder.name + //getting new meta val newMeta = builder.meta.seal() + val newData = data.map(outputType, meta = newMeta) { builder.result(env, it) } //setting the data node this[newName] = newData diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/ReduceAction.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/ReduceAction.kt index 48738452..1ce6d3c2 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/ReduceAction.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/ReduceAction.kt @@ -1,9 +1,7 @@ package hep.dataforge.data -import hep.dataforge.meta.Laminate import hep.dataforge.meta.Meta import hep.dataforge.meta.MetaBuilder -import hep.dataforge.meta.builder import hep.dataforge.names.Name import hep.dataforge.names.toName import kotlin.reflect.KClass @@ -84,15 +82,20 @@ class ReduceAction( return DataNode.invoke(outputType) { ReduceGroupBuilder(meta).apply(action).buildGroups(node).forEach { group -> - val laminate = Laminate(group.meta, meta) + //val laminate = Laminate(group.meta, meta) val dataMap = group.node.dataSequence().associate { it } - val groupName: String = group.name; + val groupName: String = group.name - val env = ActionEnv(groupName.toName(), laminate.builder()) + val groupMeta = group.meta - val res: DynamicData = dataMap.reduce(outputType, meta = laminate) { group.result.invoke(env, it) } + val env = ActionEnv(groupName.toName(), meta, groupMeta) + + val res: DynamicData = dataMap.reduce( + outputType, + meta = groupMeta + ) { group.result.invoke(env, it) } set(env.name, res) } From 257601d945a55ef225fce2aafdc9d5a28aece1c1 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Fri, 1 Nov 2019 14:53:58 +0300 Subject: [PATCH 45/48] Changed parameter order in ActionEnv --- .../commonMain/kotlin/hep/dataforge/data/MapAction.kt | 11 +++++++++-- .../kotlin/hep/dataforge/data/ReduceAction.kt | 2 +- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/MapAction.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/MapAction.kt index ea24ca77..8c543927 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/MapAction.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/MapAction.kt @@ -7,7 +7,14 @@ import hep.dataforge.meta.seal import hep.dataforge.names.Name import kotlin.reflect.KClass -data class ActionEnv(val name: Name, val actionMeta: Meta, val meta: Meta) +/** + * Action environment includes data name, data meta and action configuration meta + */ +data class ActionEnv( + val name: Name, + val meta: Meta, + val actionMeta: Meta +) /** @@ -39,7 +46,7 @@ class MapAction( /* * Creating a new environment for action using **old** name, old meta and task meta */ - val env = ActionEnv(name, meta, data.meta) + val env = ActionEnv(name, data.meta, meta) //applying transformation from builder val builder = MapActionBuilder( diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/ReduceAction.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/ReduceAction.kt index 1ce6d3c2..9bb49151 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/ReduceAction.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/ReduceAction.kt @@ -90,7 +90,7 @@ class ReduceAction( val groupMeta = group.meta - val env = ActionEnv(groupName.toName(), meta, groupMeta) + val env = ActionEnv(groupName.toName(), groupMeta, meta) val res: DynamicData = dataMap.reduce( outputType, From 6f341f705adc134c9484252671388da7696ee5dc Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Fri, 1 Nov 2019 19:47:03 +0300 Subject: [PATCH 46/48] Added DF02 format to TaggedEnvelopeFormat. --- .../hep/dataforge/io/TaggedEnvelopeFormat.kt | 50 +++++++++++++------ 1 file changed, 34 insertions(+), 16 deletions(-) diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt index c51cb36d..97a23fb9 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt @@ -13,7 +13,8 @@ import kotlinx.io.core.* @ExperimentalUnsignedTypes class TaggedEnvelopeFormat( val io: IOPlugin, - private val metaFormatKey: Short + private val metaFormatKey: Short, + val version: VERSION = TaggedEnvelopeFormat.VERSION.DF02 ) : EnvelopeFormat { private val metaFormat = io.metaFormat(metaFormatKey) @@ -22,10 +23,17 @@ class TaggedEnvelopeFormat( private fun Tag.toBytes(): ByteReadPacket = buildPacket(24) { writeText(START_SEQUENCE) - writeText(VERSION) + writeText(version.name) writeShort(metaFormatKey) writeUInt(metaSize) - writeULong(dataSize) + when (version) { + TaggedEnvelopeFormat.VERSION.DF02 -> { + writeUInt(dataSize.toUInt()) + } + TaggedEnvelopeFormat.VERSION.DF03 -> { + writeULong(dataSize) + } + } writeText(END_SEQUENCE) } @@ -46,7 +54,7 @@ class TaggedEnvelopeFormat( * @param formats a collection of meta formats to resolve */ override fun Input.readObject(): Envelope { - val tag = readTag() + val tag = readTag(version) val metaFormat = io.metaFormat(tag.metaFormatKey) ?: error("Meta format with key ${tag.metaFormatKey} not found") @@ -59,7 +67,7 @@ class TaggedEnvelopeFormat( } override fun Input.readPartial(): PartialEnvelope { - val tag = readTag() + val tag = readTag(version) val metaFormat = io.metaFormat(tag.metaFormatKey) ?: error("Meta format with key ${tag.metaFormatKey} not found") @@ -67,7 +75,7 @@ class TaggedEnvelopeFormat( val metaPacket = ByteReadPacket(readBytes(tag.metaSize.toInt())) val meta = metaFormat.run { metaPacket.readObject() } - return PartialEnvelope(meta, TAG_SIZE + tag.metaSize, tag.dataSize) + return PartialEnvelope(meta, version.tagSize + tag.metaSize, tag.dataSize) } private data class Tag( @@ -76,13 +84,16 @@ class TaggedEnvelopeFormat( val dataSize: ULong ) + enum class VERSION(val tagSize: UInt) { + DF02(20u), + DF03(24u) + } + companion object : EnvelopeFormatFactory { - const val VERSION = "DF03" private const val START_SEQUENCE = "#~" private const val END_SEQUENCE = "~#\r\n" - private const val TAG_SIZE = 24u - override val name: Name = super.name + VERSION + override val name: Name = super.name + "tagged" override fun invoke(meta: Meta, context: Context): EnvelopeFormat { val io = context.io @@ -94,14 +105,17 @@ class TaggedEnvelopeFormat( return TaggedEnvelopeFormat(io, metaFormatFactory.key) } - private fun Input.readTag(): Tag { + private fun Input.readTag(version: VERSION): Tag { val start = readTextExactBytes(2, charset = Charsets.ISO_8859_1) if (start != START_SEQUENCE) error("The input is not an envelope") - val version = readTextExactBytes(4, charset = Charsets.ISO_8859_1) - if (version != VERSION) error("Wrong version of DataForge: expected $VERSION but found $version") + val versionString = readTextExactBytes(4, charset = Charsets.ISO_8859_1) + if (version.name != versionString) error("Wrong version of DataForge: expected $version but found $versionString") val metaFormatKey = readShort() val metaLength = readUInt() - val dataLength = readULong() + val dataLength: ULong = when (version) { + VERSION.DF02 -> readUInt().toULong() + VERSION.DF03 -> readULong() + } val end = readTextExactBytes(4, charset = Charsets.ISO_8859_1) if (end != END_SEQUENCE) error("The input is not an envelope") return Tag(metaFormatKey, metaLength, dataLength) @@ -109,14 +123,18 @@ class TaggedEnvelopeFormat( override fun peekFormat(io: IOPlugin, input: Input): EnvelopeFormat? { return try { - val tag = input.readTag() - TaggedEnvelopeFormat(io, tag.metaFormatKey) + val header = input.readTextExactBytes(6) + when(header.substring(2..5)){ + VERSION.DF02.name-> TaggedEnvelopeFormat(io, JsonMetaFormat.key,VERSION.DF02) + VERSION.DF03.name-> TaggedEnvelopeFormat(io, JsonMetaFormat.key,VERSION.DF03) + else -> null + } } catch (ex: Exception) { null } } - val default by lazy { invoke()} + val default by lazy { invoke() } } } \ No newline at end of file From 33b1de286596dbb482ecdd951de3a73061a1e005 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Fri, 1 Nov 2019 20:04:11 +0300 Subject: [PATCH 47/48] Optimized EnvelopeFormat generation API --- .../io/yaml/FrontMatterEnvelopeFormat.kt | 18 +++++++-------- .../kotlin/hep/dataforge/io/EnvelopeFormat.kt | 7 +++++- .../hep/dataforge/io/TaggedEnvelopeFormat.kt | 22 +++++++++---------- .../hep/dataforge/io/TaglessEnvelopeFormat.kt | 20 +++++++---------- 4 files changed, 33 insertions(+), 34 deletions(-) diff --git a/dataforge-io/dataforge-io-yaml/src/main/kotlin/hep/dataforge/io/yaml/FrontMatterEnvelopeFormat.kt b/dataforge-io/dataforge-io-yaml/src/main/kotlin/hep/dataforge/io/yaml/FrontMatterEnvelopeFormat.kt index 9306b336..db701625 100644 --- a/dataforge-io/dataforge-io-yaml/src/main/kotlin/hep/dataforge/io/yaml/FrontMatterEnvelopeFormat.kt +++ b/dataforge-io/dataforge-io-yaml/src/main/kotlin/hep/dataforge/io/yaml/FrontMatterEnvelopeFormat.kt @@ -2,20 +2,18 @@ package hep.dataforge.io.yaml import hep.dataforge.context.Context import hep.dataforge.io.* -import hep.dataforge.meta.* +import hep.dataforge.meta.DFExperimental +import hep.dataforge.meta.EmptyMeta +import hep.dataforge.meta.Meta import kotlinx.io.core.* import kotlinx.serialization.toUtf8Bytes @DFExperimental class FrontMatterEnvelopeFormat( val io: IOPlugin, - val metaType: String = YamlMetaFormat.name.toString(), meta: Meta = EmptyMeta ) : EnvelopeFormat { - val metaFormat = io.metaFormat(metaType, meta) - ?: error("Meta format with type $metaType could not be resolved in $io") - override fun Input.readPartial(): PartialEnvelope { var line: String = "" var offset = 0u @@ -60,11 +58,12 @@ class FrontMatterEnvelopeFormat( return SimpleEnvelope(meta, data) } - override fun Output.writeObject(obj: Envelope) { + override fun Output.writeEnvelope(envelope: Envelope, metaFormatFactory: MetaFormatFactory, formatMeta: Meta) { + val metaFormat = metaFormatFactory(formatMeta, io.context) writeText("$SEPARATOR\r\n") - metaFormat.run { writeObject(obj.meta) } + metaFormat.run { writeObject(envelope.meta) } writeText("$SEPARATOR\r\n") - obj.data?.read { copyTo(this@writeObject) } + envelope.data?.read { copyTo(this@writeEnvelope) } } companion object : EnvelopeFormatFactory { @@ -73,8 +72,7 @@ class FrontMatterEnvelopeFormat( private val metaTypeRegex = "---(\\w*)\\s*".toRegex() override fun invoke(meta: Meta, context: Context): EnvelopeFormat { - val metaFormatName: String = meta["name"].string ?: YamlMetaFormat.name.toString() - return FrontMatterEnvelopeFormat(context.io, metaFormatName, meta) + return FrontMatterEnvelopeFormat(context.io, meta) } override fun peekFormat(io: IOPlugin, input: Input): EnvelopeFormat? { diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/EnvelopeFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/EnvelopeFormat.kt index 4c5c5839..c52b9e1d 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/EnvelopeFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/EnvelopeFormat.kt @@ -2,6 +2,7 @@ package hep.dataforge.io import hep.dataforge.context.Context import hep.dataforge.io.EnvelopeFormatFactory.Companion.ENVELOPE_FORMAT_TYPE +import hep.dataforge.meta.EmptyMeta import hep.dataforge.meta.Meta import hep.dataforge.names.Name import hep.dataforge.names.asName @@ -18,11 +19,15 @@ data class PartialEnvelope(val meta: Meta, val dataOffset: UInt, val dataSize: U interface EnvelopeFormat : IOFormat { + val defaultMetaFormat: MetaFormatFactory get() = JsonMetaFormat + fun Input.readPartial(): PartialEnvelope + fun Output.writeEnvelope(envelope: Envelope, metaFormatFactory: MetaFormatFactory, formatMeta: Meta = EmptyMeta) + override fun Input.readObject(): Envelope - override fun Output.writeObject(obj: Envelope) + override fun Output.writeObject(obj: Envelope): Unit = writeEnvelope(obj, defaultMetaFormat) } @Type(ENVELOPE_FORMAT_TYPE) diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt index 97a23fb9..cce3eade 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt @@ -13,12 +13,11 @@ import kotlinx.io.core.* @ExperimentalUnsignedTypes class TaggedEnvelopeFormat( val io: IOPlugin, - private val metaFormatKey: Short, val version: VERSION = TaggedEnvelopeFormat.VERSION.DF02 ) : EnvelopeFormat { - private val metaFormat = io.metaFormat(metaFormatKey) - ?: error("Meta format with key $metaFormatKey could not be resolved in $io") +// private val metaFormat = io.metaFormat(metaFormatKey) +// ?: error("Meta format with key $metaFormatKey could not be resolved in $io") private fun Tag.toBytes(): ByteReadPacket = buildPacket(24) { @@ -37,13 +36,14 @@ class TaggedEnvelopeFormat( writeText(END_SEQUENCE) } - override fun Output.writeObject(obj: Envelope) { - val metaBytes = metaFormat.writeBytes(obj.meta) - val tag = Tag(metaFormatKey, metaBytes.size.toUInt() + 2u, obj.data?.size ?: 0.toULong()) + override fun Output.writeEnvelope(envelope: Envelope, metaFormatFactory: MetaFormatFactory, formatMeta: Meta) { + val metaFormat = metaFormatFactory.invoke(formatMeta, io.context) + val metaBytes = metaFormat.writeBytes(envelope.meta) + val tag = Tag(metaFormatFactory.key, metaBytes.size.toUInt() + 2u, envelope.data?.size ?: 0.toULong()) writePacket(tag.toBytes()) writeFully(metaBytes) writeText("\r\n") - obj.data?.read { copyTo(this@writeObject) } + envelope.data?.read { copyTo(this@writeEnvelope) } flush() } @@ -102,7 +102,7 @@ class TaggedEnvelopeFormat( val metaFormatFactory = io.metaFormatFactories.find { it.name == metaFormatName } ?: error("Meta format could not be resolved") - return TaggedEnvelopeFormat(io, metaFormatFactory.key) + return TaggedEnvelopeFormat(io) } private fun Input.readTag(version: VERSION): Tag { @@ -124,9 +124,9 @@ class TaggedEnvelopeFormat( override fun peekFormat(io: IOPlugin, input: Input): EnvelopeFormat? { return try { val header = input.readTextExactBytes(6) - when(header.substring(2..5)){ - VERSION.DF02.name-> TaggedEnvelopeFormat(io, JsonMetaFormat.key,VERSION.DF02) - VERSION.DF03.name-> TaggedEnvelopeFormat(io, JsonMetaFormat.key,VERSION.DF03) + when (header.substring(2..5)) { + VERSION.DF02.name -> TaggedEnvelopeFormat(io, VERSION.DF02) + VERSION.DF03.name -> TaggedEnvelopeFormat(io, VERSION.DF03) else -> null } } catch (ex: Exception) { diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaglessEnvelopeFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaglessEnvelopeFormat.kt index 8e7c9e2b..14d871db 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaglessEnvelopeFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaglessEnvelopeFormat.kt @@ -8,33 +8,30 @@ import kotlinx.serialization.toUtf8Bytes class TaglessEnvelopeFormat( val io: IOPlugin, - val metaType: String = JsonMetaFormat.name.toString(), meta: Meta = EmptyMeta ) : EnvelopeFormat { private val metaStart = meta[META_START_PROPERTY].string ?: DEFAULT_META_START private val dataStart = meta[DATA_START_PROPERTY].string ?: DEFAULT_DATA_START - val metaFormat = io.metaFormat(metaType, meta) - ?: error("Meta format with type $metaType could not be resolved in $io") - private fun Output.writeProperty(key: String, value: Any) { writeText("#? $key: $value;\r\n") } - override fun Output.writeObject(obj: Envelope) { + override fun Output.writeEnvelope(envelope: Envelope, metaFormatFactory: MetaFormatFactory, formatMeta: Meta) { + val metaFormat = metaFormatFactory(formatMeta, io.context) //printing header writeText(TAGLESS_ENVELOPE_HEADER + "\r\n") //printing all properties - writeProperty(META_TYPE_PROPERTY, metaType) + writeProperty(META_TYPE_PROPERTY, metaFormatFactory.type) //TODO add optional metaFormat properties - writeProperty(DATA_LENGTH_PROPERTY, obj.data?.size ?: 0) + writeProperty(DATA_LENGTH_PROPERTY, envelope.data?.size ?: 0) //Printing meta - if (!obj.meta.isEmpty()) { - val metaBytes = metaFormat.writeBytes(obj.meta) + if (!envelope.meta.isEmpty()) { + val metaBytes = metaFormat.writeBytes(envelope.meta) writeProperty(META_LENGTH_PROPERTY, metaBytes.size) writeText(metaStart + "\r\n") writeFully(metaBytes) @@ -42,7 +39,7 @@ class TaglessEnvelopeFormat( } //Printing data - obj.data?.let { data -> + envelope.data?.let { data -> writeText(dataStart + "\r\n") writeFully(data.toBytes()) } @@ -170,8 +167,7 @@ class TaglessEnvelopeFormat( override val name = TAGLESS_ENVELOPE_TYPE.asName() override fun invoke(meta: Meta, context: Context): EnvelopeFormat { - val metaFormatName: String = meta["name"].string ?: JsonMetaFormat.name.toString() - return TaglessEnvelopeFormat(context.io, metaFormatName, meta) + return TaglessEnvelopeFormat(context.io, meta) } val default by lazy { invoke() } From aefee64581ec56badc4e4c81ec10ea50f36b6188 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Fri, 1 Nov 2019 21:15:55 +0300 Subject: [PATCH 48/48] optional descriptor for descriptor builder --- .../io/serialization/MetaSerializer.kt | 25 +++++++++++-------- .../io/serialization/serializationUtils.kt | 11 +++++--- .../hep/dataforge/io/MetaSerializerTest.kt | 2 ++ 3 files changed, 24 insertions(+), 14 deletions(-) diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/serialization/MetaSerializer.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/serialization/MetaSerializer.kt index 271f95e2..b22fed4a 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/serialization/MetaSerializer.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/serialization/MetaSerializer.kt @@ -1,6 +1,7 @@ -package hep.dataforge.io +package hep.dataforge.io.serialization -import hep.dataforge.io.serialization.descriptor +import hep.dataforge.io.toJson +import hep.dataforge.io.toMeta import hep.dataforge.meta.* import hep.dataforge.names.NameToken import hep.dataforge.values.* @@ -16,10 +17,10 @@ object ValueSerializer : KSerializer { private val valueTypeSerializer = EnumSerializer(ValueType::class) private val listSerializer by lazy { ArrayListSerializer(ValueSerializer) } - override val descriptor: SerialDescriptor = descriptor("Value") { + override val descriptor: SerialDescriptor = descriptor("hep.dataforge.values.Value") { boolean("isList") enum("valueType") - element("value", PolymorphicClassDescriptor) + element("value", null) } private fun Decoder.decodeValue(): Value { @@ -69,7 +70,7 @@ object ValueSerializer : KSerializer { object MetaItemSerializer : KSerializer> { override val descriptor: SerialDescriptor = descriptor("MetaItem") { boolean("isNode") - element("value", PolymorphicClassDescriptor) + element("value", null) } @@ -93,17 +94,21 @@ object MetaItemSerializer : KSerializer> { private class DeserializedMeta(override val items: Map>) : MetaBase() - /** * Serialized for meta */ @Serializer(Meta::class) object MetaSerializer : KSerializer { - private val mapSerializer = - HashMapSerializer(StringSerializer, MetaItemSerializer) + private val mapSerializer = HashMapSerializer( + StringSerializer, + MetaItemSerializer + ) - override val descriptor: SerialDescriptor = - NamedMapClassDescriptor("Meta", StringSerializer.descriptor, MetaItemSerializer.descriptor) + override val descriptor: SerialDescriptor = NamedMapClassDescriptor( + "hep.dataforge.meta.Meta", + StringSerializer.descriptor, + MetaItemSerializer.descriptor + ) override fun deserialize(decoder: Decoder): Meta { return if (decoder is JsonInput) { diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/serialization/serializationUtils.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/serialization/serializationUtils.kt index 4d33fad9..09d17054 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/serialization/serializationUtils.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/serialization/serializationUtils.kt @@ -12,12 +12,12 @@ import kotlinx.serialization.internal.* inline class SerialDescriptorBuilder(private val impl: SerialClassDescImpl) { fun element( name: String, - descriptor: SerialDescriptor, + descriptor: SerialDescriptor?, isOptional: Boolean = false, vararg annotations: Annotation ) { impl.addElement(name, isOptional) - impl.pushDescriptor(descriptor) + descriptor?.let { impl.pushDescriptor(descriptor) } annotations.forEach { impl.pushAnnotation(it) } @@ -57,7 +57,7 @@ inline class SerialDescriptorBuilder(private val impl: SerialClassDescImpl) { fun doubleArray(name: String, isOptional: Boolean = false, vararg annotations: Annotation) = element(name, DoubleArraySerializer.descriptor, isOptional, *annotations) - inline fun > enum(name: String, isOptional: Boolean = false, vararg annotations: Annotation) = + inline fun > enum(name: String, isOptional: Boolean = false, vararg annotations: Annotation) = element(name, EnumSerializer(E::class).descriptor, isOptional, *annotations) fun classAnnotation(a: Annotation) = impl.pushClassAnnotation(a) @@ -65,7 +65,10 @@ inline class SerialDescriptorBuilder(private val impl: SerialClassDescImpl) { fun build(): SerialDescriptor = impl } -inline fun KSerializer<*>.descriptor(name: String, block: SerialDescriptorBuilder.() -> Unit): SerialDescriptor = +inline fun KSerializer.descriptor( + name: String, + block: SerialDescriptorBuilder.() -> Unit +): SerialDescriptor = SerialDescriptorBuilder(SerialClassDescImpl(name)).apply(block).build() fun Decoder.decodeStructure( diff --git a/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/MetaSerializerTest.kt b/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/MetaSerializerTest.kt index f33ad894..7a8447c0 100644 --- a/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/MetaSerializerTest.kt +++ b/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/MetaSerializerTest.kt @@ -1,5 +1,7 @@ package hep.dataforge.io +import hep.dataforge.io.serialization.MetaItemSerializer +import hep.dataforge.io.serialization.MetaSerializer import hep.dataforge.io.serialization.NameSerializer import hep.dataforge.meta.buildMeta import hep.dataforge.names.toName