diff --git a/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/_data.kt b/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/_data.kt deleted file mode 100644 index 00c9e656..00000000 --- a/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/_data.kt +++ /dev/null @@ -1,8 +0,0 @@ -package hep.dataforge.data - -import kotlinx.coroutines.runBlocking - -/** - * Block the thread and get data content - */ -fun Data.get(): T = runBlocking { await() } \ No newline at end of file diff --git a/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/checkType.kt b/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/dataJVM.kt similarity index 70% rename from dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/checkType.kt rename to dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/dataJVM.kt index 0b4c602f..f87c4155 100644 --- a/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/checkType.kt +++ b/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/dataJVM.kt @@ -1,8 +1,14 @@ package hep.dataforge.data +import kotlinx.coroutines.runBlocking import kotlin.reflect.KClass import kotlin.reflect.full.isSuperclassOf +/** + * Block the thread and get data content + */ +fun Data.get(): T = runBlocking { await() } + /** * Check that node is compatible with given type meaning that each element could be cast to the type */ diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/BinaryMetaFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/BinaryMetaFormat.kt index aa4be898..daf08756 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/BinaryMetaFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/BinaryMetaFormat.kt @@ -80,7 +80,7 @@ object BinaryMetaFormat : MetaFormat { writeValue(item.value) } is MetaItem.NodeItem -> { - writeObject(item.node) + writeThis(item.node) } } } diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Envelope.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Envelope.kt index 08fe44e5..bfbe40bf 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Envelope.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Envelope.kt @@ -1,6 +1,8 @@ package hep.dataforge.io +import hep.dataforge.context.Named import hep.dataforge.io.EnvelopeFormat.Companion.ENVELOPE_FORMAT_TYPE +import hep.dataforge.io.IOPlugin.Companion.defaultMetaFormats import hep.dataforge.meta.Meta import hep.dataforge.meta.get import hep.dataforge.meta.string @@ -49,15 +51,23 @@ val Envelope.dataType: String? get() = meta[Envelope.ENVELOPE_DATA_TYPE_KEY].str */ val Envelope.description: String? get() = meta[Envelope.ENVELOPE_DESCRIPTION_KEY].string +/** + * A partially read envelope with meta, but without data + */ +@ExperimentalUnsignedTypes data class PartialEnvelope(val meta: Meta, val dataOffset: UInt, val dataSize: ULong?) @Type(ENVELOPE_FORMAT_TYPE) -interface EnvelopeFormat : IOFormat{ - fun readPartial(input: Input): PartialEnvelope +interface EnvelopeFormat : IOFormat, Named { + fun Input.readPartial(formats: Collection = defaultMetaFormats): PartialEnvelope - fun Output.writeEnvelope(envelope: Envelope, format: MetaFormat) + fun Input.readEnvelope(formats: Collection = defaultMetaFormats): Envelope - override fun Output.writeObject(obj: Envelope) = writeEnvelope(obj, JsonMetaFormat) + override fun Input.readThis(): Envelope = readEnvelope() + + fun Output.writeEnvelope(envelope: Envelope, format: MetaFormat = JsonMetaFormat) + + override fun Output.writeThis(obj: Envelope) = writeEnvelope(obj) companion object { const val ENVELOPE_FORMAT_TYPE = "envelopeFormat" diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOFormat.kt index 9055b249..9cc9a584 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOFormat.kt @@ -6,9 +6,9 @@ import kotlinx.io.core.* * And interface for serialization facilities */ interface IOFormat { - fun Output.writeObject(obj: T) - fun Input.readObject(): T + fun Output.writeThis(obj: T) + fun Input.readThis(): T } -fun IOFormat.writePacket(obj: T): ByteReadPacket = buildPacket { writeObject(obj) } -fun IOFormat.writeBytes(obj: T): ByteArray = buildPacket { writeObject(obj) }.readBytes() \ No newline at end of file +fun IOFormat.writePacket(obj: T): ByteReadPacket = buildPacket { writeThis(obj) } +fun IOFormat.writeBytes(obj: T): ByteArray = buildPacket { writeThis(obj) }.readBytes() \ No newline at end of file diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOPlugin.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOPlugin.kt index 6416e5c4..b65e4982 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOPlugin.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOPlugin.kt @@ -6,7 +6,6 @@ import hep.dataforge.context.PluginTag import hep.dataforge.context.content import hep.dataforge.meta.Meta import hep.dataforge.names.Name -import hep.dataforge.names.asName import kotlin.reflect.KClass class IOPlugin(meta: Meta) : AbstractPlugin(meta) { @@ -21,16 +20,15 @@ class IOPlugin(meta: Meta) : AbstractPlugin(meta) { override fun provideTop(target: String): Map { return when (target) { - MetaFormat.META_FORMAT_TYPE -> internalMetaFormats - EnvelopeFormat.ENVELOPE_FORMAT_TYPE -> mapOf( - TaggedEnvelopeFormat.VERSION.asName() to TaggedEnvelopeFormat(metaFormats) - ) + MetaFormat.META_FORMAT_TYPE -> defaultMetaFormats.toMap() + EnvelopeFormat.ENVELOPE_FORMAT_TYPE -> defaultEnvelopeFormats.toMap() else -> super.provideTop(target) } } companion object : PluginFactory { - private val internalMetaFormats = listOf(JsonMetaFormat, BinaryMetaFormat).toMap() + val defaultMetaFormats: List = listOf(JsonMetaFormat, BinaryMetaFormat) + val defaultEnvelopeFormats = listOf(TaggedEnvelopeFormat) override val tag: PluginTag = PluginTag("io", group = PluginTag.DATAFORGE_GROUP) override val type: KClass = IOPlugin::class diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/JsonMetaFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/JsonMetaFormat.kt index 5dabf499..ab31f78c 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/JsonMetaFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/JsonMetaFormat.kt @@ -56,15 +56,16 @@ fun Value.toJson(descriptor: ValueDescriptor? = null): JsonElement { //Use theese methods to customize JSON key mapping private fun NameToken.toJsonKey(descriptor: ItemDescriptor?) = toString() + private fun NodeDescriptor?.getDescriptor(key: String) = this?.items?.get(key) fun Meta.toJson(descriptor: NodeDescriptor? = null): JsonObject { //TODO search for same name siblings and arrange them into arrays - val map = this.items.entries.associate {(name,item)-> + val map = this.items.entries.associate { (name, item) -> val itemDescriptor = descriptor?.items?.get(name.body) val key = name.toJsonKey(itemDescriptor) - val value = when (item) { + val value = when (item) { is MetaItem.ValueItem -> { item.value.toJson(itemDescriptor as? ValueDescriptor) } diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/MetaFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/MetaFormat.kt index f658ca32..b0ecafa4 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/MetaFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/MetaFormat.kt @@ -15,11 +15,11 @@ interface MetaFormat : IOFormat, Named { override val name: String val key: Short - override fun Output.writeObject(obj: Meta) { + override fun Output.writeThis(obj: Meta) { writeMeta(obj, null) } - override fun Input.readObject(): Meta = readMeta(null) + override fun Input.readThis(): Meta = readMeta(null) fun Output.writeMeta(meta: Meta, descriptor: NodeDescriptor? = null) fun Input.readMeta(descriptor: NodeDescriptor? = null): Meta @@ -30,20 +30,20 @@ interface MetaFormat : IOFormat, Named { } fun Meta.toString(format: MetaFormat = JsonMetaFormat): String = buildPacket { - format.run { writeObject(this@toString) } + format.run { writeThis(this@toString) } }.readText() fun Meta.toBytes(format: MetaFormat = JsonMetaFormat): ByteReadPacket = buildPacket { - format.run { writeObject(this@toBytes) } + format.run { writeThis(this@toBytes) } } fun MetaFormat.parse(str: String): Meta { - return ByteReadPacket(str.toByteArray()).readObject() + return ByteReadPacket(str.toByteArray()).readThis() } fun MetaFormat.fromBytes(packet: ByteReadPacket): Meta { - return packet.readObject() + return packet.readThis() } diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt index dbbe9c58..291f539d 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt @@ -2,21 +2,75 @@ package hep.dataforge.io import kotlinx.io.core.* -class TaggedEnvelopeFormat(val metaFormats: Collection) : EnvelopeFormat { + +@ExperimentalUnsignedTypes +object TaggedEnvelopeFormat : EnvelopeFormat { + const val VERSION = "DF03" + private const val START_SEQUENCE = "#~" + private const val END_SEQUENCE = "~#\r\n" + private const val TAG_SIZE = 26u + + override val name: String get() = VERSION + + private fun Tag.toBytes(): ByteReadPacket = buildPacket(24) { + writeText(START_SEQUENCE) + writeText(VERSION) + writeShort(metaFormatKey) + writeUInt(metaSize) + writeULong(dataSize) + writeText(END_SEQUENCE) + } + + private fun Input.readTag(): Tag { + val start = readTextExactBytes(2) + if (start != START_SEQUENCE) error("The input is not an envelope") + val version = readTextExactBytes(4) + if (version != VERSION) error("Wrong version of DataForge: expected $VERSION but found $version") + val metaFormatKey = readShort() + val metaLength = readUInt() + val dataLength = readULong() + return Tag(metaFormatKey, metaLength, dataLength) + } override fun Output.writeEnvelope(envelope: Envelope, format: MetaFormat) { - write(this, envelope, format) + val metaBytes = format.writeBytes(envelope.meta) + val tag = Tag(format.key, metaBytes.size.toUInt(), envelope.data?.size ?: 0.toULong()) + writePacket(tag.toBytes()) + writeFully(metaBytes) + envelope.data?.read { copyTo(this@writeEnvelope) } } /** * Read an envelope from input into memory * * @param input an input to read from - * @param metaFormats a collection of meta formats to resolve + * @param formats a collection of meta formats to resolve */ - override fun Input.readObject(): Envelope = read(this, metaFormats) + override fun Input.readEnvelope(formats: Collection): Envelope { + val tag = readTag() - override fun readPartial(input: Input): PartialEnvelope = Companion.readPartial(input, metaFormats) + val metaFormat = formats.find { it.key == tag.metaFormatKey } + ?: error("Meta format with key ${tag.metaFormatKey} not found") + + val metaPacket = ByteReadPacket(readBytes(tag.metaSize.toInt())) + val meta = metaFormat.run { metaPacket.readThis() } + + val dataBytes = readBytes(tag.dataSize.toInt()) + + return SimpleEnvelope(meta, ArrayBinary(dataBytes)) + } + + override fun Input.readPartial(formats: Collection): PartialEnvelope { + val tag = readTag() + + val metaFormat = formats.find { it.key == tag.metaFormatKey } + ?: error("Meta format with key ${tag.metaFormatKey} not found") + + val metaPacket = ByteReadPacket(readBytes(tag.metaSize.toInt())) + val meta = metaFormat.run { metaPacket.readThis() } + + return PartialEnvelope(meta, TAG_SIZE + tag.metaSize, tag.dataSize) + } private data class Tag( val metaFormatKey: Short, @@ -24,64 +78,4 @@ class TaggedEnvelopeFormat(val metaFormats: Collection) : EnvelopeFo val dataSize: ULong ) - companion object { - const val VERSION = "DF03" - private const val START_SEQUENCE = "#~" - private const val END_SEQUENCE = "~#\r\n" - private const val TAG_SIZE = 26u - - private fun Tag.toBytes(): ByteReadPacket = buildPacket(24) { - writeText(START_SEQUENCE) - writeText(VERSION) - writeShort(metaFormatKey) - writeUInt(metaSize) - writeULong(dataSize) - writeText(END_SEQUENCE) - } - - private fun Input.readTag(): Tag { - val start = readTextExactBytes(2) - if (start != START_SEQUENCE) error("The input is not an envelope") - val version = readTextExactBytes(4) - if (version != VERSION) error("Wrong version of DataForge: expected $VERSION but found $version") - val metaFormatKey = readShort() - val metaLength = readUInt() - val dataLength = readULong() - return Tag(metaFormatKey, metaLength, dataLength) - } - - fun read(input: Input, metaFormats: Collection): Envelope { - val tag = input.readTag() - - val metaFormat = metaFormats.find { it.key == tag.metaFormatKey } - ?: error("Meta format with key ${tag.metaFormatKey} not found") - - val metaPacket = ByteReadPacket(input.readBytes(tag.metaSize.toInt())) - val meta = metaFormat.run { metaPacket.readObject() } - - val dataBytes = input.readBytes(tag.dataSize.toInt()) - - return SimpleEnvelope(meta, ArrayBinary(dataBytes)) - } - - fun readPartial(input: Input, metaFormats: Collection): PartialEnvelope { - val tag = input.readTag() - - val metaFormat = metaFormats.find { it.key == tag.metaFormatKey } - ?: error("Meta format with key ${tag.metaFormatKey} not found") - - val metaPacket = ByteReadPacket(input.readBytes(tag.metaSize.toInt())) - val meta = metaFormat.run { metaPacket.readObject() } - - return PartialEnvelope(meta, TAG_SIZE + tag.metaSize, tag.dataSize) - } - - fun write(out: Output, envelope: Envelope, metaFormat: MetaFormat) { - val metaBytes = metaFormat.writeBytes(envelope.meta) - val tag = Tag(metaFormat.key, metaBytes.size.toUInt(), envelope.data?.size ?: 0.toULong()) - out.writePacket(tag.toBytes()) - out.writeFully(metaBytes) - envelope.data?.read { copyTo(out) } - } - } } \ No newline at end of file diff --git a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileBinary.kt b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileBinary.kt index e68d002d..038281d4 100644 --- a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileBinary.kt +++ b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileBinary.kt @@ -7,6 +7,7 @@ import java.nio.file.Files import java.nio.file.Path import java.nio.file.StandardOpenOption +@ExperimentalUnsignedTypes class FileBinary(val path: Path, private val offset: UInt = 0u, size: ULong? = null) : RandomAccessBinary { override val size: ULong = size ?: (Files.size(path).toULong() - offset).toULong() diff --git a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileEnvelope.kt b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileEnvelope.kt index de926db8..fdc4acc0 100644 --- a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileEnvelope.kt +++ b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileEnvelope.kt @@ -2,6 +2,7 @@ package hep.dataforge.io import hep.dataforge.meta.Meta import kotlinx.io.nio.asInput +import kotlinx.io.nio.asOutput import java.nio.file.Files import java.nio.file.Path import java.nio.file.StandardOpenOption @@ -13,7 +14,7 @@ class FileEnvelope internal constructor(val path: Path, val format: EnvelopeForm init { val input = Files.newByteChannel(path, StandardOpenOption.READ).asInput() - partialEnvelope = format.readPartial(input) + partialEnvelope = format.run { input.readPartial() } } override val meta: Meta get() = partialEnvelope.meta @@ -21,4 +22,21 @@ class FileEnvelope internal constructor(val path: Path, val format: EnvelopeForm override val data: Binary? = FileBinary(path, partialEnvelope.dataOffset, partialEnvelope.dataSize) } -fun Path.readEnvelope(format: EnvelopeFormat) = FileEnvelope(this,format) \ No newline at end of file +fun Path.readEnvelope(format: EnvelopeFormat) = FileEnvelope(this, format) + +fun Path.writeEnvelope( + envelope: Envelope, + format: EnvelopeFormat = TaggedEnvelopeFormat, + metaFormat: MetaFormat = JsonMetaFormat +) { + val output = Files.newByteChannel( + this, + StandardOpenOption.WRITE, + StandardOpenOption.CREATE, + StandardOpenOption.TRUNCATE_EXISTING + ).asOutput() + + with(format) { + output.writeEnvelope(envelope, metaFormat) + } +} \ No newline at end of file diff --git a/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/fileData.kt b/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/fileData.kt index 25350006..1367998d 100644 --- a/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/fileData.kt +++ b/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/fileData.kt @@ -62,7 +62,7 @@ suspend fun Context.readData( format.run { Files.newByteChannel(path, StandardOpenOption.READ) .asInput() - .readObject() + .readThis() } } }