diff --git a/.github/workflows/dev_testing.yml b/.github/workflows/dev_testing.yml new file mode 100644 index 00000000..51d365cc --- /dev/null +++ b/.github/workflows/dev_testing.yml @@ -0,0 +1,18 @@ +name: CI + +on: + push: + branches: + - dev + +jobs: + build: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v1 + - name: Run a multi-line script + run: | + ./gradlew build + ./gradlew allTests diff --git a/README.md b/README.md index 79fa4cf1..b712501c 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ - +[![DOI](https://zenodo.org/badge/148831678.svg)](https://zenodo.org/badge/latestdoi/148831678) # Questions and Answers # diff --git a/build.gradle.kts b/build.gradle.kts index 430fcda4..04cce594 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -1,12 +1,10 @@ -import scientifik.ScientifikExtension - plugins { - id("scientifik.mpp") version "0.2.2" apply false - id("scientifik.jvm") version "0.2.2" apply false - id("scientifik.publish") version "0.2.2" apply false + id("scientifik.mpp") version "0.2.1" apply false + id("scientifik.jvm") version "0.2.1" apply false + id("scientifik.publish") version "0.2.1" apply false } -val dataforgeVersion by extra("0.1.5-dev-3") +val dataforgeVersion by extra("0.1.4") val bintrayRepo by extra("dataforge") val githubProject by extra("dataforge-core") @@ -17,8 +15,7 @@ allprojects { } subprojects { - apply(plugin = "scientifik.publish") - afterEvaluate { - extensions.findByType()?.apply { withDokka() } - } + if (name.startsWith("dataforge")) { + apply(plugin = "scientifik.publish") + } } \ No newline at end of file diff --git a/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/ContextBuilder.kt b/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/ContextBuilder.kt index 0aed9b7f..58a03554 100644 --- a/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/ContextBuilder.kt +++ b/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/ContextBuilder.kt @@ -1,6 +1,5 @@ package hep.dataforge.context -import hep.dataforge.meta.DFBuilder import hep.dataforge.meta.MetaBuilder import hep.dataforge.meta.buildMeta import hep.dataforge.names.toName @@ -8,7 +7,6 @@ import hep.dataforge.names.toName /** * A convenience builder for context */ -@DFBuilder class ContextBuilder(var name: String = "@anonymous", val parent: Context = Global) { private val plugins = ArrayList() private var meta = MetaBuilder() diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataNode.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataNode.kt index 65c07676..12bb06ab 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataNode.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataNode.kt @@ -13,22 +13,16 @@ import kotlin.reflect.KClass sealed class DataItem : MetaRepr { abstract val type: KClass - abstract val meta: Meta - class Node(val value: DataNode) : DataItem() { override val type: KClass get() = value.type override fun toMeta(): Meta = value.toMeta() - - override val meta: Meta get() = value.meta } class Leaf(val value: Data) : DataItem() { override val type: KClass get() = value.type override fun toMeta(): Meta = value.toMeta() - - override val meta: Meta get() = value.meta } } @@ -44,8 +38,6 @@ interface DataNode : MetaRepr { val items: Map> - val meta: Meta - override fun toMeta(): Meta = buildMeta { "type" put (type.simpleName ?: "undefined") "items" put { @@ -72,13 +64,21 @@ val DataItem?.node: DataNode? get() = (this as? DataItem.Node val DataItem?.data: Data? get() = (this as? DataItem.Leaf)?.value /** - * Start computation for all goals in data node and return a job for the whole node + * Start computation for all goals in data node */ -fun DataNode<*>.launchAll(scope: CoroutineScope): Job = scope.launch { - items.values.forEach { - when (it) { - is DataItem.Node<*> -> it.value.launchAll(scope) - is DataItem.Leaf<*> -> it.value.start(scope) +fun DataNode<*>.startAll(scope: CoroutineScope): Unit = items.values.forEach { + when (it) { + is DataItem.Node<*> -> it.value.startAll(scope) + is DataItem.Leaf<*> -> it.value.start(scope) + } +} + +fun DataNode<*>.joinAll(scope: CoroutineScope): Job = scope.launch { + startAll(scope) + items.forEach { + when (val value = it.value) { + is DataItem.Node -> value.value.joinAll(this).join() + is DataItem.Leaf -> value.value.await(scope) } } } @@ -125,9 +125,12 @@ operator fun DataNode.iterator(): Iterator>> class DataTree internal constructor( override val type: KClass, - override val items: Map>, - override val meta: Meta -) : DataNode + override val items: Map> +) : DataNode { + override fun toString(): String { + return super.toString() + } +} private sealed class DataTreeBuilderItem { class Node(val tree: DataTreeBuilder) : DataTreeBuilderItem() @@ -141,8 +144,6 @@ private sealed class DataTreeBuilderItem { class DataTreeBuilder(val type: KClass) { private val map = HashMap>() - private var meta = MetaBuilder() - operator fun set(token: NameToken, node: DataTreeBuilder) { if (map.containsKey(token)) error("Tree entry with name $token is not empty") map[token] = DataTreeBuilderItem.Node(node) @@ -210,19 +211,13 @@ class DataTreeBuilder(val type: KClass) { infix fun String.put(block: DataTreeBuilder.() -> Unit) = set(toName(), DataTreeBuilder(type).apply(block)) - /** - * Update data with given node data and meta with node meta. - */ fun update(node: DataNode) { node.dataSequence().forEach { //TODO check if the place is occupied this[it.first] = it.second } - meta.update(node.meta) } - fun meta(block: MetaBuilder.() -> Unit) = meta.apply(block) - fun build(): DataTree { val resMap = map.mapValues { (_, value) -> when (value) { @@ -230,7 +225,7 @@ class DataTreeBuilder(val type: KClass) { is DataTreeBuilderItem.Node -> DataItem.Node(value.tree.build()) } } - return DataTree(type, resMap, meta.seal()) + return DataTree(type, resMap) } } diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/MapAction.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/MapAction.kt index 89e887db..8c543927 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/MapAction.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/MapAction.kt @@ -1,6 +1,9 @@ package hep.dataforge.data -import hep.dataforge.meta.* +import hep.dataforge.meta.Meta +import hep.dataforge.meta.MetaBuilder +import hep.dataforge.meta.builder +import hep.dataforge.meta.seal import hep.dataforge.names.Name import kotlin.reflect.KClass @@ -17,7 +20,6 @@ data class ActionEnv( /** * Action environment */ -@DFBuilder class MapActionBuilder(var name: Name, var meta: MetaBuilder, val actionMeta: Meta) { lateinit var result: suspend ActionEnv.(T) -> R diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/dataCast.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/dataCast.kt index 0b9a4910..556b77fc 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/dataCast.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/dataCast.kt @@ -52,7 +52,6 @@ inline fun Data<*>.cast(): Data = cast(R::class) @Suppress("UNCHECKED_CAST") fun DataNode<*>.cast(type: KClass): DataNode { return object : DataNode { - override val meta: Meta get() = this@cast.meta override val type: KClass = type override val items: Map> get() = this@cast.items as Map> } diff --git a/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/TypeFilteredDataNode.kt b/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/TypeFilteredDataNode.kt index 331f3b0e..d24de964 100644 --- a/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/TypeFilteredDataNode.kt +++ b/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/TypeFilteredDataNode.kt @@ -1,6 +1,5 @@ package hep.dataforge.data -import hep.dataforge.meta.Meta import hep.dataforge.names.NameToken import kotlin.reflect.KClass @@ -9,7 +8,6 @@ import kotlin.reflect.KClass * A zero-copy data node wrapper that returns only children with appropriate type. */ class TypeFilteredDataNode(val origin: DataNode<*>, override val type: KClass) : DataNode { - override val meta: Meta get() = origin.meta override val items: Map> by lazy { origin.items.mapNotNull { (key, item) -> when (item) { diff --git a/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/dataJVM.kt b/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/dataJVM.kt index 29d048ed..5b5507b2 100644 --- a/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/dataJVM.kt +++ b/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/dataJVM.kt @@ -30,10 +30,12 @@ fun Data<*>.filterIsInstance(type: KClass): Data? = * but could contain empty nodes */ fun DataNode<*>.filterIsInstance(type: KClass): DataNode { - return when { - canCast(type) -> cast(type) - this is TypeFilteredDataNode -> origin.filterIsInstance(type) - else -> TypeFilteredDataNode(this, type) + return if (canCast(type)) { + cast(type) + } else if (this is TypeFilteredDataNode) { + origin.filterIsInstance(type) + } else { + TypeFilteredDataNode(this, type) } } diff --git a/dataforge-io/dataforge-io-yaml/src/main/kotlin/hep/dataforge/io/yaml/FrontMatterEnvelopeFormat.kt b/dataforge-io/dataforge-io-yaml/src/main/kotlin/hep/dataforge/io/yaml/FrontMatterEnvelopeFormat.kt index 023635e2..db701625 100644 --- a/dataforge-io/dataforge-io-yaml/src/main/kotlin/hep/dataforge/io/yaml/FrontMatterEnvelopeFormat.kt +++ b/dataforge-io/dataforge-io-yaml/src/main/kotlin/hep/dataforge/io/yaml/FrontMatterEnvelopeFormat.kt @@ -24,7 +24,7 @@ class FrontMatterEnvelopeFormat( val readMetaFormat = metaTypeRegex.matchEntire(line)?.groupValues?.first() - ?.let { io.metaFormat(it) } ?: YamlMetaFormat + ?.let { io.metaFormat(it) } ?: YamlMetaFormat.default val metaBlock = buildPacket { do { @@ -45,7 +45,7 @@ class FrontMatterEnvelopeFormat( val readMetaFormat = metaTypeRegex.matchEntire(line)?.groupValues?.first() - ?.let { io.metaFormat(it) } ?: YamlMetaFormat + ?.let { io.metaFormat(it) } ?: YamlMetaFormat.default val metaBlock = buildPacket { do { @@ -72,7 +72,7 @@ class FrontMatterEnvelopeFormat( private val metaTypeRegex = "---(\\w*)\\s*".toRegex() override fun invoke(meta: Meta, context: Context): EnvelopeFormat { - return FrontMatterEnvelopeFormat(context.io, meta) + return FrontMatterEnvelopeFormat(context.io, meta) } override fun peekFormat(io: IOPlugin, input: Input): EnvelopeFormat? { @@ -84,16 +84,5 @@ class FrontMatterEnvelopeFormat( } } - private val default by lazy { invoke() } - - override fun Input.readPartial(): PartialEnvelope = - default.run { readPartial() } - - override fun Output.writeEnvelope(envelope: Envelope, metaFormatFactory: MetaFormatFactory, formatMeta: Meta) = - default.run { writeEnvelope(envelope, metaFormatFactory, formatMeta) } - - override fun Input.readObject(): Envelope = - default.run { readObject() } - } } \ No newline at end of file diff --git a/dataforge-io/dataforge-io-yaml/src/main/kotlin/hep/dataforge/io/yaml/YamlMetaFormat.kt b/dataforge-io/dataforge-io-yaml/src/main/kotlin/hep/dataforge/io/yaml/YamlMetaFormat.kt index 7130518d..24ea44ec 100644 --- a/dataforge-io/dataforge-io-yaml/src/main/kotlin/hep/dataforge/io/yaml/YamlMetaFormat.kt +++ b/dataforge-io/dataforge-io-yaml/src/main/kotlin/hep/dataforge/io/yaml/YamlMetaFormat.kt @@ -45,18 +45,12 @@ class YamlMetaFormat(val meta: Meta) : MetaFormat { } companion object : MetaFormatFactory { + val default = YamlMetaFormat() + override fun invoke(meta: Meta, context: Context): MetaFormat = YamlMetaFormat(meta) override val name: Name = super.name + "yaml" override val key: Short = 0x594d //YM - - private val default = YamlMetaFormat() - - override fun Output.writeMeta(meta: Meta, descriptor: NodeDescriptor?) = - default.run { writeMeta(meta, descriptor) } - - override fun Input.readMeta(descriptor: NodeDescriptor?): Meta = - default.run { readMeta(descriptor) } } } \ No newline at end of file diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Binary.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Binary.kt index bd1f2249..ca05de4d 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Binary.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Binary.kt @@ -8,7 +8,7 @@ import kotlin.math.min */ interface Binary { /** - * The size of binary in bytes. [ULong.MAX_VALUE] if size is not defined and input should be read until its end is reached + * The size of binary in bytes */ val size: ULong @@ -18,10 +18,6 @@ interface Binary { * Some implementation may forbid this to be called twice. In this case second call will throw an exception. */ fun read(block: Input.() -> R): R - - companion object { - val EMPTY = EmptyBinary - } } /** @@ -52,11 +48,12 @@ fun RandomAccessBinary.readPacket(from: UInt, size: UInt): ByteReadPacket = read @ExperimentalUnsignedTypes object EmptyBinary : RandomAccessBinary { - override val size: ULong = 0u + override val size: ULong = 0.toULong() override fun read(from: UInt, size: UInt, block: Input.() -> R): R { error("The binary is empty") } + } @ExperimentalUnsignedTypes @@ -82,9 +79,9 @@ fun Binary.readWith(format: IOFormat): T = format.run { } } -//fun IOFormat.writeBinary(obj: T): Binary { -// val packet = buildPacket { -// writeObject(obj) -// } -// return ArrayBinary(packet.readBytes()) -//} \ No newline at end of file +fun IOFormat.writeBinary(obj: T): Binary { + val packet = buildPacket { + writeObject(obj) + } + return ArrayBinary(packet.readBytes()) +} \ No newline at end of file diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Envelope.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Envelope.kt index 7cb918df..80e07b56 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Envelope.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Envelope.kt @@ -1,11 +1,11 @@ package hep.dataforge.io -import hep.dataforge.meta.Laminate -import hep.dataforge.meta.Meta -import hep.dataforge.meta.get -import hep.dataforge.meta.string +import hep.dataforge.meta.* import hep.dataforge.names.asName import hep.dataforge.names.plus +import kotlinx.io.core.Output +import kotlinx.io.core.buildPacket +import kotlinx.io.core.readBytes interface Envelope { val meta: Meta @@ -21,7 +21,6 @@ interface Envelope { val ENVELOPE_DATA_TYPE_KEY = ENVELOPE_NODE_KEY + "dataType" val ENVELOPE_DATA_ID_KEY = ENVELOPE_NODE_KEY + "dataID" val ENVELOPE_DESCRIPTION_KEY = ENVELOPE_NODE_KEY + "description" - val ENVELOPE_NAME_KEY = ENVELOPE_NODE_KEY + "name" //const val ENVELOPE_TIME_KEY = "@envelope.time" /** @@ -84,3 +83,32 @@ fun Envelope.withMetaLayers(vararg layers: Meta): Envelope { } } +class EnvelopeBuilder { + private val metaBuilder = MetaBuilder() + var data: Binary? = null + + fun meta(block: MetaBuilder.() -> Unit) { + metaBuilder.apply(block) + } + + fun meta(meta: Meta) { + metaBuilder.update(meta) + } + + var type by metaBuilder.string(key = Envelope.ENVELOPE_TYPE_KEY) + var dataType by metaBuilder.string(key = Envelope.ENVELOPE_DATA_TYPE_KEY) + var dataID by metaBuilder.string(key = Envelope.ENVELOPE_DATA_ID_KEY) + var description by metaBuilder.string(key = Envelope.ENVELOPE_DESCRIPTION_KEY) + + /** + * Construct a binary and transform it into byte-array based buffer + */ + fun data(block: Output.() -> Unit) { + val bytes = buildPacket { + block() + } + data = ArrayBinary(bytes.readBytes()) + } + + internal fun build() = SimpleEnvelope(metaBuilder.seal(), data) +} \ No newline at end of file diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/EnvelopeBuilder.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/EnvelopeBuilder.kt deleted file mode 100644 index 354b4586..00000000 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/EnvelopeBuilder.kt +++ /dev/null @@ -1,37 +0,0 @@ -package hep.dataforge.io - -import hep.dataforge.meta.* -import kotlinx.io.core.Output -import kotlinx.io.core.buildPacket -import kotlinx.io.core.readBytes - -class EnvelopeBuilder { - private val metaBuilder = MetaBuilder() - var data: Binary? = null - - fun meta(block: MetaBuilder.() -> Unit) { - metaBuilder.apply(block) - } - - fun meta(meta: Meta) { - metaBuilder.update(meta) - } - - var type by metaBuilder.string(key = Envelope.ENVELOPE_TYPE_KEY) - var dataType by metaBuilder.string(key = Envelope.ENVELOPE_DATA_TYPE_KEY) - var dataID by metaBuilder.string(key = Envelope.ENVELOPE_DATA_ID_KEY) - var description by metaBuilder.string(key = Envelope.ENVELOPE_DESCRIPTION_KEY) - var name by metaBuilder.string(key = Envelope.ENVELOPE_NAME_KEY) - - /** - * Construct a binary and transform it into byte-array based buffer - */ - fun data(block: Output.() -> Unit) { - val bytes = buildPacket { - block() - } - data = ArrayBinary(bytes.readBytes()) - } - - internal fun build() = SimpleEnvelope(metaBuilder.seal(), data) -} \ No newline at end of file diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/EnvelopeFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/EnvelopeFormat.kt index 4f747ea2..c52b9e1d 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/EnvelopeFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/EnvelopeFormat.kt @@ -31,7 +31,7 @@ interface EnvelopeFormat : IOFormat { } @Type(ENVELOPE_FORMAT_TYPE) -interface EnvelopeFormatFactory : IOFormatFactory, EnvelopeFormat { +interface EnvelopeFormatFactory : IOFormatFactory { override val name: Name get() = "envelope".asName() override val type: KClass get() = Envelope::class diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/EnvelopeParts.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/EnvelopeParts.kt deleted file mode 100644 index d1b86195..00000000 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/EnvelopeParts.kt +++ /dev/null @@ -1,98 +0,0 @@ -package hep.dataforge.io - -import hep.dataforge.context.Global -import hep.dataforge.io.EnvelopeParts.FORMAT_META_KEY -import hep.dataforge.io.EnvelopeParts.FORMAT_NAME_KEY -import hep.dataforge.io.EnvelopeParts.INDEX_KEY -import hep.dataforge.io.EnvelopeParts.MULTIPART_DATA_TYPE -import hep.dataforge.io.EnvelopeParts.SIZE_KEY -import hep.dataforge.meta.* -import hep.dataforge.names.asName -import hep.dataforge.names.plus -import hep.dataforge.names.toName - -object EnvelopeParts { - val MULTIPART_KEY = "multipart".asName() - val SIZE_KEY = Envelope.ENVELOPE_NODE_KEY + MULTIPART_KEY + "size" - val INDEX_KEY = Envelope.ENVELOPE_NODE_KEY + MULTIPART_KEY + "index" - val FORMAT_NAME_KEY = Envelope.ENVELOPE_NODE_KEY + MULTIPART_KEY + "format" - val FORMAT_META_KEY = Envelope.ENVELOPE_NODE_KEY + MULTIPART_KEY + "meta" - - const val MULTIPART_DATA_TYPE = "envelope.multipart" -} - -/** - * Append multiple serialized envelopes to the data block. Previous data is erased if it was present - */ -fun EnvelopeBuilder.multipart(format: EnvelopeFormatFactory, envelopes: Collection) { - dataType = MULTIPART_DATA_TYPE - meta { - SIZE_KEY put envelopes.size - FORMAT_NAME_KEY put format.name.toString() - } - data { - format.run { - envelopes.forEach { - writeObject(it) - } - } - } -} - -/** - * Create a multipart partition in the envelope adding additional name-index mapping in meta - */ -@DFExperimental -fun EnvelopeBuilder.multipart(format: EnvelopeFormatFactory, envelopes: Map) { - dataType = MULTIPART_DATA_TYPE - meta { - SIZE_KEY put envelopes.size - FORMAT_NAME_KEY put format.name.toString() - } - data { - format.run { - var counter = 0 - envelopes.forEach {(key, envelope)-> - writeObject(envelope) - meta{ - append(INDEX_KEY, buildMeta { - "key" put key - "index" put counter - }) - } - counter++ - } - } - } -} - -fun EnvelopeBuilder.multipart( - formatFactory: EnvelopeFormatFactory, - builder: suspend SequenceScope.() -> Unit -) = multipart(formatFactory, sequence(builder).toList()) - -/** - * If given envelope supports multipart data, return a sequence of those parts (could be empty). Otherwise return null. - */ -fun Envelope.parts(io: IOPlugin = Global.plugins.fetch(IOPlugin)): Sequence? { - return when (dataType) { - MULTIPART_DATA_TYPE -> { - val size = meta[SIZE_KEY].int ?: error("Unsized parts not supported yet") - val formatName = meta[FORMAT_NAME_KEY].string?.toName() - ?: error("Inferring parts format is not supported at the moment") - val formatMeta = meta[FORMAT_META_KEY].node ?: EmptyMeta - val format = io.envelopeFormat(formatName, formatMeta) - ?: error("Format $formatName is not resolved by $io") - return format.run { - data?.read { - sequence { - repeat(size) { - yield(readObject()) - } - } - } ?: emptySequence() - } - } - else -> null - } -} diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOFormat.kt index e5497365..093ffbc8 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOFormat.kt @@ -16,6 +16,7 @@ import kotlinx.serialization.ImplicitReflectionSerializer import kotlinx.serialization.KSerializer import kotlinx.serialization.cbor.Cbor import kotlinx.serialization.serializer +import kotlin.math.min import kotlin.reflect.KClass /** @@ -79,11 +80,33 @@ inline fun buildPacketWithoutPool(headerSizeHint: Int = 0, block: BytePacketBuil } fun IOFormat.writePacket(obj: T): ByteReadPacket = buildPacket { writeObject(obj) } - -@Deprecated("Not to be used outside tests due to double buffer write") fun IOFormat.writeBytes(obj: T): ByteArray = buildPacket { writeObject(obj) }.readBytes() -@Deprecated("Not to be used outside tests due to double buffer write") -fun IOFormat.readBytes(array: ByteArray): T = buildPacket { writeFully(array) }.readObject() +fun IOFormat.readBytes(array: ByteArray): T { + //= ByteReadPacket(array).readThis() + val byteArrayInput: Input = object : AbstractInput( + IoBuffer.Pool.borrow(), + remaining = array.size.toLong(), + pool = IoBuffer.Pool + ) { + var written = 0 + override fun closeSource() { + // do nothing + } + + override fun fill(): IoBuffer? { + if (array.size - written <= 0) return null + + return IoBuffer.Pool.fill { + reserveEndGap(IoBuffer.ReservedSize) + val toWrite = min(capacity, array.size - written) + writeFully(array, written, toWrite) + written += toWrite + } + } + + } + return byteArrayInput.readObject() +} object DoubleIOFormat : IOFormat, IOFormatFactory { override fun invoke(meta: Meta, context: Context): IOFormat = this diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOPlugin.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOPlugin.kt index a463a053..7e61924f 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOPlugin.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOPlugin.kt @@ -26,9 +26,6 @@ class IOPlugin(meta: Meta) : AbstractPlugin(meta) { context.content(ENVELOPE_FORMAT_TYPE).values } - fun envelopeFormat(name: Name, meta: Meta = EmptyMeta) = - envelopeFormatFactories.find { it.name == name }?.invoke(meta, context) - override fun provideTop(target: String): Map { return when (target) { META_FORMAT_TYPE -> defaultMetaFormats.toMap() @@ -52,7 +49,7 @@ class IOPlugin(meta: Meta) : AbstractPlugin(meta) { companion object : PluginFactory { val defaultMetaFormats: List = listOf(JsonMetaFormat, BinaryMetaFormat) - val defaultEnvelopeFormats = listOf(TaggedEnvelopeFormat, TaglessEnvelopeFormat) + val defaultEnvelopeFormats = listOf(TaggedEnvelopeFormat) override val tag: PluginTag = PluginTag("io", group = PluginTag.DATAFORGE_GROUP) diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/JsonMetaFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/JsonMetaFormat.kt index 5c10505d..a95cdec4 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/JsonMetaFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/JsonMetaFormat.kt @@ -38,18 +38,12 @@ class JsonMetaFormat(private val json: Json = Json.indented) : MetaFormat { } companion object : MetaFormatFactory { + val default = JsonMetaFormat() + override fun invoke(meta: Meta, context: Context): MetaFormat = default override val name: Name = super.name + "json" override val key: Short = 0x4a53//"JS" - - private val default = JsonMetaFormat() - - override fun Output.writeMeta(meta: Meta, descriptor: NodeDescriptor?) = - default.run { writeMeta(meta,descriptor) } - - override fun Input.readMeta(descriptor: NodeDescriptor?): Meta = - default.run { readMeta(descriptor) } } } @@ -96,7 +90,7 @@ fun Meta.toJson(descriptor: NodeDescriptor? = null): JsonObject { fun JsonElement.toMeta(descriptor: NodeDescriptor? = null): Meta { return when (val item = toMetaItem(descriptor)) { is MetaItem.NodeItem<*> -> item.node - is MetaItem.ValueItem -> item.value.toMeta() + is MetaItem.ValueItem ->item.value.toMeta() } } diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/MetaFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/MetaFormat.kt index 9d1af81a..ca9a53a2 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/MetaFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/MetaFormat.kt @@ -27,7 +27,7 @@ interface MetaFormat : IOFormat { } @Type(META_FORMAT_TYPE) -interface MetaFormatFactory : IOFormatFactory, MetaFormat { +interface MetaFormatFactory : IOFormatFactory { override val name: Name get() = "meta".asName() override val type: KClass get() = Meta::class @@ -47,7 +47,7 @@ fun Meta.toString(format: MetaFormat): String = buildPacket { fun Meta.toString(formatFactory: MetaFormatFactory): String = toString(formatFactory()) -fun Meta.toBytes(format: MetaFormat = JsonMetaFormat): ByteReadPacket = buildPacket { +fun Meta.toBytes(format: MetaFormat = JsonMetaFormat.default): ByteReadPacket = buildPacket { format.run { writeObject(this@toBytes) } } diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt index a461d257..cce3eade 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt @@ -38,15 +38,10 @@ class TaggedEnvelopeFormat( override fun Output.writeEnvelope(envelope: Envelope, metaFormatFactory: MetaFormatFactory, formatMeta: Meta) { val metaFormat = metaFormatFactory.invoke(formatMeta, io.context) - val metaBytes = metaFormat.writePacket(envelope.meta) - val actualSize: ULong = if (envelope.data == null) { - 0u - } else { - envelope.data?.size ?: ULong.MAX_VALUE - } - val tag = Tag(metaFormatFactory.key, metaBytes.remaining.toUInt() + 2u, actualSize) + val metaBytes = metaFormat.writeBytes(envelope.meta) + val tag = Tag(metaFormatFactory.key, metaBytes.size.toUInt() + 2u, envelope.data?.size ?: 0.toULong()) writePacket(tag.toBytes()) - writePacket(metaBytes) + writeFully(metaBytes) writeText("\r\n") envelope.data?.read { copyTo(this@writeEnvelope) } flush() @@ -64,10 +59,7 @@ class TaggedEnvelopeFormat( val metaFormat = io.metaFormat(tag.metaFormatKey) ?: error("Meta format with key ${tag.metaFormatKey} not found") - val metaBytes = readBytes(tag.metaSize.toInt()) - val metaPacket = buildPacket { - writeFully(metaBytes) - } + val metaPacket = ByteReadPacket(readBytes(tag.metaSize.toInt())) val dataBytes = readBytes(tag.dataSize.toInt()) val meta = metaFormat.run { metaPacket.readObject() } @@ -142,16 +134,7 @@ class TaggedEnvelopeFormat( } } - private val default by lazy { invoke() } - - override fun Input.readPartial(): PartialEnvelope = - default.run { readPartial() } - - override fun Output.writeEnvelope(envelope: Envelope, metaFormatFactory: MetaFormatFactory, formatMeta: Meta) = - default.run { writeEnvelope(envelope, metaFormatFactory, formatMeta) } - - override fun Input.readObject(): Envelope = - default.run { readObject() } + val default by lazy { invoke() } } } \ No newline at end of file diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaglessEnvelopeFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaglessEnvelopeFormat.kt index 1cc62a2b..14d871db 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaglessEnvelopeFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaglessEnvelopeFormat.kt @@ -27,20 +27,14 @@ class TaglessEnvelopeFormat( //printing all properties writeProperty(META_TYPE_PROPERTY, metaFormatFactory.type) //TODO add optional metaFormat properties - val actualSize: ULong = if (envelope.data == null) { - 0u - } else { - envelope.data?.size ?: ULong.MAX_VALUE - } - - writeProperty(DATA_LENGTH_PROPERTY, actualSize) + writeProperty(DATA_LENGTH_PROPERTY, envelope.data?.size ?: 0) //Printing meta if (!envelope.meta.isEmpty()) { - val metaBytes = metaFormat.writePacket(envelope.meta) - writeProperty(META_LENGTH_PROPERTY, metaBytes.remaining) + val metaBytes = metaFormat.writeBytes(envelope.meta) + writeProperty(META_LENGTH_PROPERTY, metaBytes.size) writeText(metaStart + "\r\n") - writePacket(metaBytes) + writeFully(metaBytes) writeText("\r\n") } @@ -49,7 +43,6 @@ class TaglessEnvelopeFormat( writeText(dataStart + "\r\n") writeFully(data.toBytes()) } - flush() } override fun Input.readObject(): Envelope { @@ -73,7 +66,7 @@ class TaglessEnvelopeFormat( var meta: Meta = EmptyMeta if (line.startsWith(metaStart)) { - val metaFormat = properties[META_TYPE_PROPERTY]?.let { io.metaFormat(it) } ?: JsonMetaFormat + val metaFormat = properties[META_TYPE_PROPERTY]?.let { io.metaFormat(it) } ?: JsonMetaFormat.default val metaSize = properties.get(META_LENGTH_PROPERTY)?.toInt() meta = if (metaSize != null) { val metaPacket = buildPacket { @@ -128,7 +121,7 @@ class TaglessEnvelopeFormat( var meta: Meta = EmptyMeta if (line.startsWith(metaStart)) { - val metaFormat = properties[META_TYPE_PROPERTY]?.let { io.metaFormat(it) } ?: JsonMetaFormat + val metaFormat = properties[META_TYPE_PROPERTY]?.let { io.metaFormat(it) } ?: JsonMetaFormat.default val metaSize = properties.get(META_LENGTH_PROPERTY)?.toInt() meta = if (metaSize != null) { @@ -177,22 +170,13 @@ class TaglessEnvelopeFormat( return TaglessEnvelopeFormat(context.io, meta) } - private val default by lazy { invoke() } - - override fun Input.readPartial(): PartialEnvelope = - default.run { readPartial() } - - override fun Output.writeEnvelope(envelope: Envelope, metaFormatFactory: MetaFormatFactory, formatMeta: Meta) = - default.run { writeEnvelope(envelope, metaFormatFactory, formatMeta) } - - override fun Input.readObject(): Envelope = - default.run { readObject() } + val default by lazy { invoke() } override fun peekFormat(io: IOPlugin, input: Input): EnvelopeFormat? { return try { val buffer = ByteArray(TAGLESS_ENVELOPE_HEADER.length) input.readFully(buffer) - return if (String(buffer) == TAGLESS_ENVELOPE_HEADER) { + return if (buffer.toString() == TAGLESS_ENVELOPE_HEADER) { TaglessEnvelopeFormat(io) } else { null diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/serialization/serializationUtils.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/serialization/serializationUtils.kt index b32abb14..09d17054 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/serialization/serializationUtils.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/serialization/serializationUtils.kt @@ -1,7 +1,9 @@ package hep.dataforge.io.serialization -import hep.dataforge.meta.DFExperimental -import kotlinx.serialization.* +import kotlinx.serialization.CompositeDecoder +import kotlinx.serialization.Decoder +import kotlinx.serialization.KSerializer +import kotlinx.serialization.SerialDescriptor import kotlinx.serialization.internal.* /** @@ -69,24 +71,10 @@ inline fun KSerializer.descriptor( ): SerialDescriptor = SerialDescriptorBuilder(SerialClassDescImpl(name)).apply(block).build() -@DFExperimental -inline fun Decoder.decodeStructure( +fun Decoder.decodeStructure( desc: SerialDescriptor, vararg typeParams: KSerializer<*> = emptyArray(), - crossinline block: CompositeDecoder.() -> R -): R { - val decoder = beginStructure(desc, *typeParams) - val res = decoder.block() - decoder.endStructure(desc) - return res -} - -inline fun Encoder.encodeStructure( - desc: SerialDescriptor, - vararg typeParams: KSerializer<*> = emptyArray(), - block: CompositeEncoder.() -> Unit + block: CompositeDecoder.() -> Unit ) { - val encoder = beginStructure(desc, *typeParams) - encoder.block() - encoder.endStructure(desc) + beginStructure(desc, *typeParams).apply(block).endStructure(desc) } \ No newline at end of file diff --git a/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/EnvelopeFormatTest.kt b/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/EnvelopeFormatTest.kt index 37ee827d..29e60f2f 100644 --- a/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/EnvelopeFormatTest.kt +++ b/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/EnvelopeFormatTest.kt @@ -18,7 +18,7 @@ class EnvelopeFormatTest { @ExperimentalStdlibApi @Test fun testTaggedFormat(){ - TaggedEnvelopeFormat.run { + TaggedEnvelopeFormat.default.run { val bytes = writeBytes(envelope) println(bytes.decodeToString()) val res = readBytes(bytes) @@ -32,7 +32,7 @@ class EnvelopeFormatTest { @Test fun testTaglessFormat(){ - TaglessEnvelopeFormat.run { + TaglessEnvelopeFormat.default.run { val bytes = writeBytes(envelope) println(bytes.decodeToString()) val res = readBytes(bytes) diff --git a/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/EnvelopePartsTest.kt b/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/EnvelopePartsTest.kt deleted file mode 100644 index d123d632..00000000 --- a/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/EnvelopePartsTest.kt +++ /dev/null @@ -1,36 +0,0 @@ -package hep.dataforge.io - -import hep.dataforge.meta.get -import hep.dataforge.meta.int -import kotlinx.io.core.writeText -import kotlin.test.Test -import kotlin.test.assertEquals - -class EnvelopePartsTest { - val envelopes = (0..5).map { - Envelope { - meta { - "value" put it - } - data { - writeText("Hello World $it") - repeat(200){ - writeInt(it) - } - } - } - } - val partsEnvelope = Envelope { - multipart(TaggedEnvelopeFormat, envelopes) - } - - @Test - fun testParts() { - val bytes = TaggedEnvelopeFormat.writeBytes(partsEnvelope) - val reconstructed = TaggedEnvelopeFormat.readBytes(bytes) - val parts = reconstructed.parts()?.toList() ?: emptyList() - assertEquals(2, parts[2].meta["value"].int) - println(reconstructed.data!!.size) - } - -} \ No newline at end of file diff --git a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileEnvelope.kt b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileEnvelope.kt index 5f21e6ae..3187cd54 100644 --- a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileEnvelope.kt +++ b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileEnvelope.kt @@ -1,7 +1,9 @@ package hep.dataforge.io +import hep.dataforge.meta.EmptyMeta import hep.dataforge.meta.Meta import kotlinx.io.nio.asInput +import kotlinx.io.nio.asOutput import java.nio.file.Files import java.nio.file.Path import java.nio.file.StandardOpenOption @@ -13,7 +15,7 @@ class FileEnvelope internal constructor(val path: Path, val format: EnvelopeForm init { val input = Files.newByteChannel(path, StandardOpenOption.READ).asInput() - partialEnvelope = format.run { input.use { it.readPartial() } } + partialEnvelope = format.run { input.use { it.readPartial()} } } override val meta: Meta get() = partialEnvelope.meta @@ -21,3 +23,30 @@ class FileEnvelope internal constructor(val path: Path, val format: EnvelopeForm override val data: Binary? = FileBinary(path, partialEnvelope.dataOffset, partialEnvelope.dataSize) } +fun IOPlugin.readEnvelopeFile( + path: Path, + formatFactory: EnvelopeFormatFactory = TaggedEnvelopeFormat, + formatMeta: Meta = EmptyMeta +): FileEnvelope { + val format = formatFactory(formatMeta, context) + return FileEnvelope(path, format) +} + +fun IOPlugin.writeEnvelopeFile( + path: Path, + envelope: Envelope, + formatFactory: EnvelopeFormatFactory = TaggedEnvelopeFormat, + formatMeta: Meta = EmptyMeta +) { + val output = Files.newByteChannel( + path, + StandardOpenOption.WRITE, + StandardOpenOption.CREATE, + StandardOpenOption.TRUNCATE_EXISTING + ).asOutput() + + with(formatFactory(formatMeta, context)) { + output.writeObject(envelope) + } +} + diff --git a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/fileIO.kt b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/fileIO.kt deleted file mode 100644 index eff9e705..00000000 --- a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/fileIO.kt +++ /dev/null @@ -1,148 +0,0 @@ -package hep.dataforge.io - -import hep.dataforge.descriptors.NodeDescriptor -import hep.dataforge.meta.DFExperimental -import hep.dataforge.meta.EmptyMeta -import hep.dataforge.meta.Meta -import kotlinx.io.nio.asInput -import kotlinx.io.nio.asOutput -import java.nio.file.Files -import java.nio.file.Path -import java.nio.file.StandardOpenOption -import kotlin.reflect.full.isSuperclassOf -import kotlin.streams.asSequence - -inline fun IOPlugin.resolveIOFormat(): IOFormat? { - return ioFormats.values.find { it.type.isSuperclassOf(T::class) } as IOFormat? -} - -/** - * Read file containing meta using given [formatOverride] or file extension to infer meta type. - * If [path] is a directory search for file starting with `meta` in it - */ -fun IOPlugin.readMetaFile(path: Path, formatOverride: MetaFormat? = null, descriptor: NodeDescriptor? = null): Meta { - if (!Files.exists(path)) error("Meta file $path does not exist") - - val actualPath: Path = if (Files.isDirectory(path)) { - Files.list(path).asSequence().find { it.fileName.startsWith("meta") } - ?: error("The directory $path does not contain meta file") - } else { - path - } - val extension = actualPath.fileName.toString().substringAfterLast('.') - - val metaFormat = formatOverride ?: metaFormat(extension) ?: error("Can't resolve meta format $extension") - return metaFormat.run { - Files.newByteChannel(actualPath, StandardOpenOption.READ).asInput().use { it.readMeta(descriptor) } - } -} - -/** - * Write meta to file using [metaFormat]. If [path] is a directory, write a file with name equals name of [metaFormat]. - * Like "meta.json" - */ -fun IOPlugin.writeMetaFile( - path: Path, - metaFormat: MetaFormatFactory = JsonMetaFormat, - descriptor: NodeDescriptor? = null -) { - val actualPath = if (Files.isDirectory(path)) { - path.resolve(metaFormat.name.toString()) - } else { - path - } - metaFormat.run { - Files.newByteChannel(actualPath, StandardOpenOption.WRITE, StandardOpenOption.CREATE_NEW).asOutput().use { - it.writeMeta(meta, descriptor) - } - } -} - -/** - * Return inferred [EnvelopeFormat] if only one format could read given file. If no format accepts file, return null. If - * multiple formats accepts file, throw an error. - */ -fun IOPlugin.peekBinaryFormat(binary: Binary): EnvelopeFormat? { - val formats = envelopeFormatFactories.mapNotNull { factory -> - binary.read { - factory.peekFormat(this@peekBinaryFormat, this@read) - } - } - - return when (formats.size) { - 0 -> null - 1 -> formats.first() - else -> error("Envelope format binary recognition clash") - } -} - -/** - * Read and envelope from file if the file exists, return null if file does not exist. - * - * If file is directory, then expect two files inside: - * * **meta.** for meta - * * **data** for data - * - * If the file is envelope read it using [EnvelopeFormatFactory.peekFormat] functionality to infer format. - * - * If the file is not an envelope and [readNonEnvelopes] is true, return an Envelope without meta, using file as binary. - * - * Return null otherwise. - */ -@DFExperimental -fun IOPlugin.readEnvelopeFile( - path: Path, - readNonEnvelopes: Boolean = false, - formatPeeker: IOPlugin.(Binary) -> EnvelopeFormat? = IOPlugin::peekBinaryFormat -): Envelope? { - if (!Files.exists(path)) return null - - //read two-files directory - if (Files.isDirectory(path)) { - val metaFile = Files.list(path).asSequence() - .singleOrNull { it.fileName.toString().startsWith("meta") } - - val meta = if (metaFile == null) { - EmptyMeta - } else { - readMetaFile(metaFile) - } - - val dataFile = path.resolve("data") - - val data: Binary? = if (Files.exists(dataFile)) { - dataFile.asBinary() - } else { - null - } - - return SimpleEnvelope(meta, data) - } - - val binary = path.asBinary() - - return formatPeeker(binary)?.run { - binary.read { - readObject() - } - } ?: if (readNonEnvelopes) { // if no format accepts file, read it as binary - SimpleEnvelope(Meta.empty, binary) - } else null -} - -fun IOPlugin.writeEnvelopeFile( - path: Path, - envelope: Envelope, - format: EnvelopeFormat = TaggedEnvelopeFormat -) { - val output = Files.newByteChannel( - path, - StandardOpenOption.WRITE, - StandardOpenOption.CREATE, - StandardOpenOption.TRUNCATE_EXISTING - ).asOutput() - - with(format) { - output.writeObject(envelope) - } -} \ No newline at end of file diff --git a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/functionsJVM.kt b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/functionsJVM.kt deleted file mode 100644 index fae986d7..00000000 --- a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/functionsJVM.kt +++ /dev/null @@ -1,29 +0,0 @@ -package hep.dataforge.io - -import hep.dataforge.io.functions.FunctionServer -import hep.dataforge.io.functions.function -import hep.dataforge.meta.Meta -import hep.dataforge.meta.buildMeta -import hep.dataforge.names.Name -import kotlin.reflect.KClass -import kotlin.reflect.full.isSuperclassOf - - -fun IOPlugin.resolveIOFormatName(type: KClass<*>): Name { - return ioFormats.entries.find { it.value.type.isSuperclassOf(type) }?.key - ?: error("Can't resolve IOFormat for type $type") -} - -inline fun IOPlugin.generateFunctionMeta(functionName: String): Meta = buildMeta { - FunctionServer.FUNCTION_NAME_KEY put functionName - FunctionServer.INPUT_FORMAT_KEY put resolveIOFormatName(T::class).toString() - FunctionServer.OUTPUT_FORMAT_KEY put resolveIOFormatName(R::class).toString() -} - -inline fun FunctionServer.function( - functionName: String -): (suspend (T) -> R) { - val plugin = context.plugins.get() ?: error("IO plugin not loaded") - val meta = plugin.generateFunctionMeta(functionName) - return function(meta) -} \ No newline at end of file diff --git a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/ioFormatsJVM.kt b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/ioFormatsJVM.kt new file mode 100644 index 00000000..c926d07a --- /dev/null +++ b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/ioFormatsJVM.kt @@ -0,0 +1,51 @@ +package hep.dataforge.io + +import hep.dataforge.descriptors.NodeDescriptor +import hep.dataforge.io.functions.FunctionServer +import hep.dataforge.io.functions.FunctionServer.Companion.FUNCTION_NAME_KEY +import hep.dataforge.io.functions.FunctionServer.Companion.INPUT_FORMAT_KEY +import hep.dataforge.io.functions.FunctionServer.Companion.OUTPUT_FORMAT_KEY +import hep.dataforge.io.functions.function +import hep.dataforge.meta.Meta +import hep.dataforge.meta.buildMeta +import hep.dataforge.names.Name +import kotlinx.io.nio.asOutput +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.StandardOpenOption +import kotlin.reflect.KClass +import kotlin.reflect.full.isSuperclassOf + +inline fun IOPlugin.resolveIOFormat(): IOFormat? { + return ioFormats.values.find { it.type.isSuperclassOf(T::class) } as IOFormat? +} + +fun IOPlugin.resolveIOFormatName(type: KClass<*>): Name { + return ioFormats.entries.find { it.value.type.isSuperclassOf(type) }?.key + ?: error("Can't resolve IOFormat for type $type") +} + +inline fun IOPlugin.generateFunctionMeta(functionName: String): Meta = buildMeta { + FUNCTION_NAME_KEY put functionName + INPUT_FORMAT_KEY put resolveIOFormatName(T::class).toString() + OUTPUT_FORMAT_KEY put resolveIOFormatName(R::class).toString() +} + +inline fun FunctionServer.function( + functionName: String +): (suspend (T) -> R) { + val plugin = context.plugins.get() ?: error("IO plugin not loaded") + val meta = plugin.generateFunctionMeta(functionName) + return function(meta) +} + +/** + * Write meta to file in a given [format] + */ +fun Meta.write(path: Path, format: MetaFormat, descriptor: NodeDescriptor? = null) { + format.run { + Files.newByteChannel(path, StandardOpenOption.WRITE, StandardOpenOption.CREATE_NEW) + .asOutput() + .writeMeta(this@write, descriptor) + } +} \ No newline at end of file diff --git a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/tcp/InputStreamAsInput.kt b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/tcp/InputStreamAsInput.kt index eb743625..1c711be0 100644 --- a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/tcp/InputStreamAsInput.kt +++ b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/tcp/InputStreamAsInput.kt @@ -3,6 +3,7 @@ package hep.dataforge.io.tcp import kotlinx.io.core.AbstractInput import kotlinx.io.core.Input import kotlinx.io.core.IoBuffer +import kotlinx.io.core.IoBuffer.Companion.NoPool import kotlinx.io.core.writePacket import kotlinx.io.streams.readPacketAtMost import java.io.InputStream @@ -12,7 +13,7 @@ import java.io.InputStream */ internal class InputStreamAsInput( private val stream: InputStream -) : AbstractInput(pool = IoBuffer.Pool) { +) : AbstractInput(pool = NoPool) { override fun fill(): IoBuffer? { diff --git a/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/FileBinaryTest.kt b/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/FileBinaryTest.kt index d8c7c67a..94403dcd 100644 --- a/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/FileBinaryTest.kt +++ b/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/FileBinaryTest.kt @@ -39,7 +39,7 @@ class FileBinaryTest { } val binary = envelopeFromFile.data!! println(binary.toBytes().size) - assertEquals(binary.size?.toInt(), binary.toBytes().size) + assertEquals(binary.size.toInt(), binary.toBytes().size) } @@ -49,7 +49,7 @@ class FileBinaryTest { val tmpPath = Files.createTempFile("dataforge_test", ".df") Global.io.writeEnvelopeFile(tmpPath, envelope) - val binary = Global.io.readEnvelopeFile(tmpPath)?.data!! + val binary = Global.io.readEnvelopeFile(tmpPath).data!! assertEquals(binary.size.toInt(), binary.toBytes().size) } diff --git a/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/FileEnvelopeTest.kt b/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/FileEnvelopeTest.kt index b8bfafaa..ba7f7cc5 100644 --- a/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/FileEnvelopeTest.kt +++ b/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/FileEnvelopeTest.kt @@ -22,23 +22,10 @@ class FileEnvelopeTest { @Test fun testFileWriteRead() { - Global.io.run { - val tmpPath = Files.createTempFile("dataforge_test", ".df") - writeEnvelopeFile(tmpPath, envelope) - println(tmpPath.toUri()) - val restored: Envelope = readEnvelopeFile(tmpPath)!! - assertTrue { envelope.contentEquals(restored) } - } - } - - @Test - fun testFileWriteReadTagless() { - Global.io.run { - val tmpPath = Files.createTempFile("dataforge_test_tagless", ".df") - writeEnvelopeFile(tmpPath, envelope, format = TaglessEnvelopeFormat) - println(tmpPath.toUri()) - val restored: Envelope = readEnvelopeFile(tmpPath)!! - assertTrue { envelope.contentEquals(restored) } - } + val tmpPath = Files.createTempFile("dataforge_test", ".df") + Global.io.writeEnvelopeFile(tmpPath,envelope) + println(tmpPath.toUri()) + val restored: Envelope = Global.io.readEnvelopeFile(tmpPath) + assertTrue { envelope.contentEquals(restored) } } } \ No newline at end of file diff --git a/dataforge-output-html/src/commonMain/kotlin/hep/dataforge/output/html/HtmlOutput.kt b/dataforge-output-html/src/commonMain/kotlin/hep/dataforge/output/html/HtmlOutput.kt index 3ff23403..b54b7eb7 100644 --- a/dataforge-output-html/src/commonMain/kotlin/hep/dataforge/output/html/HtmlOutput.kt +++ b/dataforge-output-html/src/commonMain/kotlin/hep/dataforge/output/html/HtmlOutput.kt @@ -47,7 +47,7 @@ class HtmlOutput(override val context: Context, private val consumer: T } /** - * A text or binary renderer based on [Output] + * A text or binary renderer based on [kotlinx.io.core.Output] */ @Type(HTML_CONVERTER_TYPE) interface HtmlBuilder { diff --git a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskBuilder.kt b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskBuilder.kt index 3ff86325..80d89e24 100644 --- a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskBuilder.kt +++ b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskBuilder.kt @@ -3,7 +3,6 @@ package hep.dataforge.workspace import hep.dataforge.context.Context import hep.dataforge.data.* import hep.dataforge.descriptors.NodeDescriptor -import hep.dataforge.meta.DFBuilder import hep.dataforge.meta.Meta import hep.dataforge.meta.get import hep.dataforge.meta.string @@ -14,7 +13,7 @@ import hep.dataforge.names.toName import kotlin.jvm.JvmName import kotlin.reflect.KClass -@DFBuilder +@TaskBuildScope class TaskBuilder(val name: Name, val type: KClass) { private var modelTransform: TaskModelBuilder.(Meta) -> Unit = { allData() } // private val additionalDependencies = HashSet() diff --git a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskModel.kt b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskModel.kt index 71a5c006..b4ccb7ae 100644 --- a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskModel.kt +++ b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskModel.kt @@ -58,6 +58,9 @@ fun TaskModel.buildInput(workspace: Workspace): DataTree { }.build() } +@DslMarker +annotation class TaskBuildScope + interface TaskDependencyContainer { val defaultMeta: Meta fun add(dependency: Dependency) diff --git a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/WorkspaceBuilder.kt b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/WorkspaceBuilder.kt index b8f3ffa0..2f717f78 100644 --- a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/WorkspaceBuilder.kt +++ b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/WorkspaceBuilder.kt @@ -12,7 +12,7 @@ import hep.dataforge.names.toName import kotlin.jvm.JvmName import kotlin.reflect.KClass -@DFBuilder +@TaskBuildScope interface WorkspaceBuilder { val parentContext: Context var context: Context diff --git a/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/fileData.kt b/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/fileData.kt index b5ecb519..a483c78b 100644 --- a/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/fileData.kt +++ b/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/fileData.kt @@ -1,47 +1,93 @@ package hep.dataforge.workspace -import hep.dataforge.data.* -import hep.dataforge.io.Envelope -import hep.dataforge.io.IOFormat -import hep.dataforge.io.IOPlugin -import hep.dataforge.io.readEnvelopeFile +import hep.dataforge.data.Data +import hep.dataforge.data.DataNode +import hep.dataforge.data.DataTreeBuilder +import hep.dataforge.data.datum +import hep.dataforge.descriptors.NodeDescriptor +import hep.dataforge.io.* +import hep.dataforge.meta.EmptyMeta import hep.dataforge.meta.Meta -import hep.dataforge.meta.get -import hep.dataforge.meta.string +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext +import kotlinx.io.nio.asInput +import kotlinx.io.nio.asOutput import java.nio.file.Files import java.nio.file.Path +import java.nio.file.StandardOpenOption import kotlin.reflect.KClass +/** + * Read meta from file in a given [MetaFormat] + */ +fun MetaFormat.readMetaFile(path: Path, descriptor: NodeDescriptor? = null): Meta { + return Files.newByteChannel(path, StandardOpenOption.READ) + .asInput() + .readMeta(descriptor) +} + +/** + * Write meta to file using given [MetaFormat] + */ +fun MetaFormat.writeMetaFile(path: Path, meta: Meta, descriptor: NodeDescriptor? = null) { + return Files.newByteChannel(path, StandardOpenOption.WRITE, StandardOpenOption.CREATE_NEW) + .asOutput() + .writeMeta(meta, descriptor) +} /** * Read data with supported envelope format and binary format. If envelope format is null, then read binary directly from file. * The operation is blocking since it must read meta header. The reading of envelope body is lazy * @param type explicit type of data read * @param dataFormat binary format - * @param envelopeFormat the format of envelope. If null, file is read directly + * @param envelopeFormatFactory the format of envelope. If null, file is read directly * @param metaFile the relative file for optional meta override * @param metaFileFormat the meta format for override */ -fun IOPlugin.readDataFile( +fun IOPlugin.readData( path: Path, type: KClass, - formatResolver: (Meta) -> IOFormat + dataFormat: IOFormat, + envelopeFormatFactory: EnvelopeFormatFactory? = null, + metaFile: Path = path.resolveSibling("${path.fileName}.meta"), + metaFileFormat: MetaFormat = JsonMetaFormat.default ): Data { - val envelope = readEnvelopeFile(path, true) ?: error("Can't read data from $path") - val format = formatResolver(envelope.meta) - return envelope.toData(type, format) + val externalMeta = if (Files.exists(metaFile)) { + metaFileFormat.readMetaFile(metaFile) + } else { + null + } + return if (envelopeFormatFactory == null) { + Data(type, externalMeta ?: EmptyMeta) { + withContext(Dispatchers.IO) { + dataFormat.run { + Files.newByteChannel(path, StandardOpenOption.READ) + .asInput() + .readObject() + } + } + } + } else { + readEnvelopeFile(path, envelopeFormatFactory).let { + if (externalMeta == null) { + it + } else { + it.withMetaLayers(externalMeta) + } + }.toData(type, dataFormat) + } } //TODO wants multi-receiver fun DataTreeBuilder.file( plugin: IOPlugin, path: Path, - formatResolver: (Meta) -> IOFormat + dataFormat: IOFormat, + envelopeFormatFactory: EnvelopeFormatFactory? = null ) { plugin.run { - val data = readDataFile(path, type, formatResolver) - val name = data.meta[Envelope.ENVELOPE_NAME_KEY].string - ?: path.fileName.toString().replace(".df", "") + val data = readData(path, type, dataFormat, envelopeFormatFactory) + val name = path.fileName.toString().replace(".df", "") datum(name, data) } } @@ -49,35 +95,23 @@ fun DataTreeBuilder.file( /** * Read the directory as a data node */ -fun IOPlugin.readDataDirectory( +fun IOPlugin.readDataNode( path: Path, type: KClass, - formatResolver: (Meta) -> IOFormat + dataFormat: IOFormat, + envelopeFormatFactory: EnvelopeFormatFactory? = null ): DataNode { if (!Files.isDirectory(path)) error("Provided path $this is not a directory") return DataNode(type) { Files.list(path).forEach { path -> if (!path.fileName.toString().endsWith(".meta")) { - file(this@readDataDirectory, path, formatResolver) + file(this@readDataNode,path, dataFormat, envelopeFormatFactory) } } } } -fun DataTreeBuilder.directory( - plugin: IOPlugin, - path: Path, - formatResolver: (Meta) -> IOFormat -) { - plugin.run { - val data = readDataDirectory(path, type, formatResolver) - val name = data.meta[Envelope.ENVELOPE_NAME_KEY].string - ?: path.fileName.toString().replace(".df", "") - node(name, data) - } -} - - - - - +//suspend fun Path.writeData( +// data: Data, +// format: IOFormat, +// ) \ No newline at end of file