From f63a2b6e93e7a744d3e14b0787842c7f9ae9bf13 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Tue, 5 Nov 2019 19:45:53 +0300 Subject: [PATCH 1/2] Some serialization fixes --- build.gradle.kts | 17 +++++++----- .../hep/dataforge/context/ContextBuilder.kt | 2 ++ .../kotlin/hep/dataforge/data/MapAction.kt | 6 ++--- .../io/serialization/serializationUtils.kt | 26 ++++++++++++++----- .../hep/dataforge/output/html/HtmlOutput.kt | 2 +- .../hep/dataforge/workspace/TaskBuilder.kt | 3 ++- .../hep/dataforge/workspace/TaskModel.kt | 3 --- 7 files changed, 36 insertions(+), 23 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index 04cce594..cd3e5d61 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -1,10 +1,12 @@ +import scientifik.ScientifikExtension + plugins { - id("scientifik.mpp") version "0.2.1" apply false - id("scientifik.jvm") version "0.2.1" apply false - id("scientifik.publish") version "0.2.1" apply false + id("scientifik.mpp") version "0.2.2" apply false + id("scientifik.jvm") version "0.2.2" apply false + id("scientifik.publish") version "0.2.2" apply false } -val dataforgeVersion by extra("0.1.4") +val dataforgeVersion by extra("0.1.5-dev-1") val bintrayRepo by extra("dataforge") val githubProject by extra("dataforge-core") @@ -15,7 +17,8 @@ allprojects { } subprojects { - if (name.startsWith("dataforge")) { - apply(plugin = "scientifik.publish") - } + apply(plugin = "scientifik.publish") + afterEvaluate { + extensions.findByType()?.apply { withDokka() } + } } \ No newline at end of file diff --git a/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/ContextBuilder.kt b/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/ContextBuilder.kt index 58a03554..0aed9b7f 100644 --- a/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/ContextBuilder.kt +++ b/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/ContextBuilder.kt @@ -1,5 +1,6 @@ package hep.dataforge.context +import hep.dataforge.meta.DFBuilder import hep.dataforge.meta.MetaBuilder import hep.dataforge.meta.buildMeta import hep.dataforge.names.toName @@ -7,6 +8,7 @@ import hep.dataforge.names.toName /** * A convenience builder for context */ +@DFBuilder class ContextBuilder(var name: String = "@anonymous", val parent: Context = Global) { private val plugins = ArrayList() private var meta = MetaBuilder() diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/MapAction.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/MapAction.kt index 8c543927..89e887db 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/MapAction.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/MapAction.kt @@ -1,9 +1,6 @@ package hep.dataforge.data -import hep.dataforge.meta.Meta -import hep.dataforge.meta.MetaBuilder -import hep.dataforge.meta.builder -import hep.dataforge.meta.seal +import hep.dataforge.meta.* import hep.dataforge.names.Name import kotlin.reflect.KClass @@ -20,6 +17,7 @@ data class ActionEnv( /** * Action environment */ +@DFBuilder class MapActionBuilder(var name: Name, var meta: MetaBuilder, val actionMeta: Meta) { lateinit var result: suspend ActionEnv.(T) -> R diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/serialization/serializationUtils.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/serialization/serializationUtils.kt index 09d17054..b32abb14 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/serialization/serializationUtils.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/serialization/serializationUtils.kt @@ -1,9 +1,7 @@ package hep.dataforge.io.serialization -import kotlinx.serialization.CompositeDecoder -import kotlinx.serialization.Decoder -import kotlinx.serialization.KSerializer -import kotlinx.serialization.SerialDescriptor +import hep.dataforge.meta.DFExperimental +import kotlinx.serialization.* import kotlinx.serialization.internal.* /** @@ -71,10 +69,24 @@ inline fun KSerializer.descriptor( ): SerialDescriptor = SerialDescriptorBuilder(SerialClassDescImpl(name)).apply(block).build() -fun Decoder.decodeStructure( +@DFExperimental +inline fun Decoder.decodeStructure( desc: SerialDescriptor, vararg typeParams: KSerializer<*> = emptyArray(), - block: CompositeDecoder.() -> Unit + crossinline block: CompositeDecoder.() -> R +): R { + val decoder = beginStructure(desc, *typeParams) + val res = decoder.block() + decoder.endStructure(desc) + return res +} + +inline fun Encoder.encodeStructure( + desc: SerialDescriptor, + vararg typeParams: KSerializer<*> = emptyArray(), + block: CompositeEncoder.() -> Unit ) { - beginStructure(desc, *typeParams).apply(block).endStructure(desc) + val encoder = beginStructure(desc, *typeParams) + encoder.block() + encoder.endStructure(desc) } \ No newline at end of file diff --git a/dataforge-output-html/src/commonMain/kotlin/hep/dataforge/output/html/HtmlOutput.kt b/dataforge-output-html/src/commonMain/kotlin/hep/dataforge/output/html/HtmlOutput.kt index b54b7eb7..3ff23403 100644 --- a/dataforge-output-html/src/commonMain/kotlin/hep/dataforge/output/html/HtmlOutput.kt +++ b/dataforge-output-html/src/commonMain/kotlin/hep/dataforge/output/html/HtmlOutput.kt @@ -47,7 +47,7 @@ class HtmlOutput(override val context: Context, private val consumer: T } /** - * A text or binary renderer based on [kotlinx.io.core.Output] + * A text or binary renderer based on [Output] */ @Type(HTML_CONVERTER_TYPE) interface HtmlBuilder { diff --git a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskBuilder.kt b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskBuilder.kt index 80d89e24..3ff86325 100644 --- a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskBuilder.kt +++ b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskBuilder.kt @@ -3,6 +3,7 @@ package hep.dataforge.workspace import hep.dataforge.context.Context import hep.dataforge.data.* import hep.dataforge.descriptors.NodeDescriptor +import hep.dataforge.meta.DFBuilder import hep.dataforge.meta.Meta import hep.dataforge.meta.get import hep.dataforge.meta.string @@ -13,7 +14,7 @@ import hep.dataforge.names.toName import kotlin.jvm.JvmName import kotlin.reflect.KClass -@TaskBuildScope +@DFBuilder class TaskBuilder(val name: Name, val type: KClass) { private var modelTransform: TaskModelBuilder.(Meta) -> Unit = { allData() } // private val additionalDependencies = HashSet() diff --git a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskModel.kt b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskModel.kt index b4ccb7ae..71a5c006 100644 --- a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskModel.kt +++ b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskModel.kt @@ -58,9 +58,6 @@ fun TaskModel.buildInput(workspace: Workspace): DataTree { }.build() } -@DslMarker -annotation class TaskBuildScope - interface TaskDependencyContainer { val defaultMeta: Meta fun add(dependency: Dependency) From 5265c0e5abe2dd26a64fc90f91ff862eaa30b669 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Tue, 5 Nov 2019 21:36:17 +0300 Subject: [PATCH 2/2] Implements Envelope parts An envelope consisting of other envelopes --- .../kotlin/hep/dataforge/io/Envelope.kt | 37 ++---------- .../hep/dataforge/io/EnvelopeBuilder.kt | 36 +++++++++++ .../kotlin/hep/dataforge/io/EnvelopeParts.kt | 60 +++++++++++++++++++ .../kotlin/hep/dataforge/io/IOPlugin.kt | 3 + .../hep/dataforge/io/EnvelopePartsTest.kt | 32 ++++++++++ 5 files changed, 135 insertions(+), 33 deletions(-) create mode 100644 dataforge-io/src/commonMain/kotlin/hep/dataforge/io/EnvelopeBuilder.kt create mode 100644 dataforge-io/src/commonMain/kotlin/hep/dataforge/io/EnvelopeParts.kt create mode 100644 dataforge-io/src/commonTest/kotlin/hep/dataforge/io/EnvelopePartsTest.kt diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Envelope.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Envelope.kt index 80e07b56..61aeb4d2 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Envelope.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Envelope.kt @@ -1,11 +1,11 @@ package hep.dataforge.io -import hep.dataforge.meta.* +import hep.dataforge.meta.Laminate +import hep.dataforge.meta.Meta +import hep.dataforge.meta.get +import hep.dataforge.meta.string import hep.dataforge.names.asName import hep.dataforge.names.plus -import kotlinx.io.core.Output -import kotlinx.io.core.buildPacket -import kotlinx.io.core.readBytes interface Envelope { val meta: Meta @@ -83,32 +83,3 @@ fun Envelope.withMetaLayers(vararg layers: Meta): Envelope { } } -class EnvelopeBuilder { - private val metaBuilder = MetaBuilder() - var data: Binary? = null - - fun meta(block: MetaBuilder.() -> Unit) { - metaBuilder.apply(block) - } - - fun meta(meta: Meta) { - metaBuilder.update(meta) - } - - var type by metaBuilder.string(key = Envelope.ENVELOPE_TYPE_KEY) - var dataType by metaBuilder.string(key = Envelope.ENVELOPE_DATA_TYPE_KEY) - var dataID by metaBuilder.string(key = Envelope.ENVELOPE_DATA_ID_KEY) - var description by metaBuilder.string(key = Envelope.ENVELOPE_DESCRIPTION_KEY) - - /** - * Construct a binary and transform it into byte-array based buffer - */ - fun data(block: Output.() -> Unit) { - val bytes = buildPacket { - block() - } - data = ArrayBinary(bytes.readBytes()) - } - - internal fun build() = SimpleEnvelope(metaBuilder.seal(), data) -} \ No newline at end of file diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/EnvelopeBuilder.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/EnvelopeBuilder.kt new file mode 100644 index 00000000..b8d0b660 --- /dev/null +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/EnvelopeBuilder.kt @@ -0,0 +1,36 @@ +package hep.dataforge.io + +import hep.dataforge.meta.* +import kotlinx.io.core.Output +import kotlinx.io.core.buildPacket +import kotlinx.io.core.readBytes + +class EnvelopeBuilder { + private val metaBuilder = MetaBuilder() + var data: Binary? = null + + fun meta(block: MetaBuilder.() -> Unit) { + metaBuilder.apply(block) + } + + fun meta(meta: Meta) { + metaBuilder.update(meta) + } + + var type by metaBuilder.string(key = Envelope.ENVELOPE_TYPE_KEY) + var dataType by metaBuilder.string(key = Envelope.ENVELOPE_DATA_TYPE_KEY) + var dataID by metaBuilder.string(key = Envelope.ENVELOPE_DATA_ID_KEY) + var description by metaBuilder.string(key = Envelope.ENVELOPE_DESCRIPTION_KEY) + + /** + * Construct a binary and transform it into byte-array based buffer + */ + fun data(block: Output.() -> Unit) { + val bytes = buildPacket { + block() + } + data = ArrayBinary(bytes.readBytes()) + } + + internal fun build() = SimpleEnvelope(metaBuilder.seal(), data) +} \ No newline at end of file diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/EnvelopeParts.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/EnvelopeParts.kt new file mode 100644 index 00000000..ca4efea4 --- /dev/null +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/EnvelopeParts.kt @@ -0,0 +1,60 @@ +package hep.dataforge.io + +import hep.dataforge.context.Global +import hep.dataforge.io.EnvelopeParts.FORMAT_META_KEY +import hep.dataforge.io.EnvelopeParts.FORMAT_NAME_KEY +import hep.dataforge.io.EnvelopeParts.PARTS_DATA_TYPE +import hep.dataforge.io.EnvelopeParts.SIZE_KEY +import hep.dataforge.meta.* +import hep.dataforge.names.plus +import hep.dataforge.names.toName + +object EnvelopeParts { + val SIZE_KEY = Envelope.ENVELOPE_NODE_KEY + "parts" + "size" + val FORMAT_NAME_KEY = Envelope.ENVELOPE_NODE_KEY + "parts" + "format" + val FORMAT_META_KEY = Envelope.ENVELOPE_NODE_KEY + "parts" + "meta" + + const val PARTS_DATA_TYPE = "envelope.parts" +} + +fun EnvelopeBuilder.parts(formatFactory: EnvelopeFormatFactory, envelopes: Collection) { + dataType = PARTS_DATA_TYPE + meta { + SIZE_KEY put envelopes.size + FORMAT_NAME_KEY put formatFactory.name.toString() + } + val format = formatFactory() + data { + format.run { + envelopes.forEach { + writeObject(it) + } + } + } +} + +fun EnvelopeBuilder.parts(formatFactory: EnvelopeFormatFactory, builder: suspend SequenceScope.() -> Unit) = + parts(formatFactory, sequence(builder).toList()) + +fun Envelope.parts(io: IOPlugin = Global.plugins.fetch(IOPlugin)): Sequence { + return when (dataType) { + PARTS_DATA_TYPE -> { + val size = meta[SIZE_KEY].int ?: error("Unsized parts not supported yet") + val formatName = meta[FORMAT_NAME_KEY].string?.toName() + ?: error("Inferring parts format is not supported at the moment") + val formatMeta = meta[FORMAT_META_KEY].node ?: EmptyMeta + val format = io.envelopeFormat(formatName, formatMeta) + ?: error("Format $formatName is not resolved by $io") + return format.run { + data?.read { + sequence { + repeat(size) { + yield(readObject()) + } + } + } ?: emptySequence() + } + } + else -> emptySequence() + } +} diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOPlugin.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOPlugin.kt index 7e61924f..ae88e4f4 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOPlugin.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOPlugin.kt @@ -26,6 +26,9 @@ class IOPlugin(meta: Meta) : AbstractPlugin(meta) { context.content(ENVELOPE_FORMAT_TYPE).values } + fun envelopeFormat(name: Name, meta: Meta = EmptyMeta) = + envelopeFormatFactories.find { it.name == name }?.invoke(meta, context) + override fun provideTop(target: String): Map { return when (target) { META_FORMAT_TYPE -> defaultMetaFormats.toMap() diff --git a/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/EnvelopePartsTest.kt b/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/EnvelopePartsTest.kt new file mode 100644 index 00000000..5122680a --- /dev/null +++ b/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/EnvelopePartsTest.kt @@ -0,0 +1,32 @@ +package hep.dataforge.io + +import hep.dataforge.meta.get +import hep.dataforge.meta.int +import kotlinx.io.core.writeText +import kotlin.test.Test +import kotlin.test.assertEquals + +class EnvelopePartsTest { + val envelopes = (0..5).map { + Envelope { + meta { + "value" put it + } + data { + writeText("Hello World $it") + } + } + } + val partsEnvelope = Envelope { + parts(TaggedEnvelopeFormat, envelopes) + } + + @Test + fun testParts() { + val bytes = TaggedEnvelopeFormat.default.writeBytes(partsEnvelope) + val reconstructed = TaggedEnvelopeFormat.default.readBytes(bytes) + val parts = reconstructed.parts().toList() + assertEquals(2, parts[2].meta["value"].int) + } + +} \ No newline at end of file