diff --git a/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/AbstractPlugin.kt b/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/AbstractPlugin.kt index 232c7cd4..e1041935 100644 --- a/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/AbstractPlugin.kt +++ b/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/AbstractPlugin.kt @@ -1,13 +1,12 @@ package hep.dataforge.context -import hep.dataforge.meta.EmptyMeta import hep.dataforge.meta.Meta import hep.dataforge.names.Name import kotlin.properties.ReadOnlyProperty import kotlin.reflect.KClass import kotlin.reflect.KProperty -abstract class AbstractPlugin(override val meta: Meta = EmptyMeta) : Plugin { +abstract class AbstractPlugin(override val meta: Meta = Meta.EMPTY) : Plugin { private var _context: Context? = null private val dependencies = ArrayList>() diff --git a/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/Factory.kt b/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/Factory.kt index b20672c2..b01334e6 100644 --- a/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/Factory.kt +++ b/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/Factory.kt @@ -1,8 +1,7 @@ package hep.dataforge.context -import hep.dataforge.meta.EmptyMeta import hep.dataforge.meta.Meta interface Factory { - operator fun invoke(meta: Meta = EmptyMeta, context: Context = Global): T + operator fun invoke(meta: Meta = Meta.EMPTY, context: Context = Global): T } \ No newline at end of file diff --git a/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/PluginRepository.kt b/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/PluginRepository.kt index 06c1132b..248b239c 100644 --- a/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/PluginRepository.kt +++ b/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/PluginRepository.kt @@ -1,6 +1,5 @@ package hep.dataforge.context -import hep.dataforge.meta.EmptyMeta import hep.dataforge.meta.Meta import kotlin.reflect.KClass @@ -23,7 +22,7 @@ expect object PluginRepository { /** * Fetch specific plugin and instantiate it with given meta */ -fun PluginRepository.fetch(tag: PluginTag, meta: Meta = EmptyMeta): Plugin = +fun PluginRepository.fetch(tag: PluginTag, meta: Meta = Meta.EMPTY): Plugin = list().find { it.tag.matches(tag) }?.invoke(meta = meta) ?: error("Plugin with tag $tag not found in the repository") diff --git a/dataforge-io/dataforge-io-yaml/src/main/kotlin/hep/dataforge/io/yaml/FrontMatterEnvelopeFormat.kt b/dataforge-io/dataforge-io-yaml/src/main/kotlin/hep/dataforge/io/yaml/FrontMatterEnvelopeFormat.kt index bac63c9d..8a837d74 100644 --- a/dataforge-io/dataforge-io-yaml/src/main/kotlin/hep/dataforge/io/yaml/FrontMatterEnvelopeFormat.kt +++ b/dataforge-io/dataforge-io-yaml/src/main/kotlin/hep/dataforge/io/yaml/FrontMatterEnvelopeFormat.kt @@ -3,7 +3,6 @@ package hep.dataforge.io.yaml import hep.dataforge.context.Context import hep.dataforge.io.* import hep.dataforge.meta.DFExperimental -import hep.dataforge.meta.EmptyMeta import hep.dataforge.meta.Meta import kotlinx.io.* import kotlinx.io.text.readUtf8Line @@ -13,7 +12,7 @@ import kotlinx.serialization.toUtf8Bytes @DFExperimental class FrontMatterEnvelopeFormat( val io: IOPlugin, - meta: Meta = EmptyMeta + val meta: Meta = Meta.EMPTY ) : EnvelopeFormat { override fun Input.readPartial(): PartialEnvelope { @@ -26,7 +25,7 @@ class FrontMatterEnvelopeFormat( val readMetaFormat = metaTypeRegex.matchEntire(line)?.groupValues?.first() - ?.let { io.metaFormat(it) } ?: YamlMetaFormat + ?.let { io.resolveMetaFormat(it) } ?: YamlMetaFormat //TODO replace by preview val meta = Binary { @@ -51,11 +50,11 @@ class FrontMatterEnvelopeFormat( val readMetaFormat = metaTypeRegex.matchEntire(line)?.groupValues?.first() - ?.let { io.metaFormat(it) } ?: YamlMetaFormat + ?.let { io.resolveMetaFormat(it) } ?: YamlMetaFormat val meta = Binary { do { - writeUtf8String(readUtf8Line() + "\r\n") + writeUtf8String(readUtf8Line() + "\r\n") } while (!line.startsWith(SEPARATOR)) }.read { readMetaFormat.run { @@ -78,6 +77,11 @@ class FrontMatterEnvelopeFormat( } } + override fun toMeta(): Meta = Meta { + IOPlugin.IO_FORMAT_NAME_KEY put name.toString() + IOPlugin.IO_FORMAT_META_KEY put meta + } + companion object : EnvelopeFormatFactory { const val SEPARATOR = "---" @@ -88,11 +92,13 @@ class FrontMatterEnvelopeFormat( } override fun peekFormat(io: IOPlugin, input: Input): EnvelopeFormat? { - val line = input.readUtf8Line() - return if (line.startsWith("---")) { - invoke() - } else { - null + return input.preview { + val line = readUtf8Line() + return@preview if (line.startsWith("---")) { + invoke() + } else { + null + } } } diff --git a/dataforge-io/dataforge-io-yaml/src/main/kotlin/hep/dataforge/io/yaml/YamlMetaFormat.kt b/dataforge-io/dataforge-io-yaml/src/main/kotlin/hep/dataforge/io/yaml/YamlMetaFormat.kt index 71be6352..1799d072 100644 --- a/dataforge-io/dataforge-io-yaml/src/main/kotlin/hep/dataforge/io/yaml/YamlMetaFormat.kt +++ b/dataforge-io/dataforge-io-yaml/src/main/kotlin/hep/dataforge/io/yaml/YamlMetaFormat.kt @@ -1,6 +1,7 @@ package hep.dataforge.io.yaml import hep.dataforge.context.Context +import hep.dataforge.io.IOPlugin import hep.dataforge.io.MetaFormat import hep.dataforge.io.MetaFormatFactory import hep.dataforge.meta.DFExperimental @@ -11,10 +12,8 @@ import hep.dataforge.meta.toMeta import kotlinx.io.Input import kotlinx.io.Output import kotlinx.io.asInputStream -import kotlinx.io.readUByte import kotlinx.io.text.writeUtf8String import org.yaml.snakeyaml.Yaml -import java.io.InputStream @DFExperimental class YamlMetaFormat(val meta: Meta) : MetaFormat { @@ -30,6 +29,11 @@ class YamlMetaFormat(val meta: Meta) : MetaFormat { return map.toMeta(descriptor) } + override fun toMeta(): Meta = Meta{ + IOPlugin.IO_FORMAT_NAME_KEY put FrontMatterEnvelopeFormat.name.toString() + IOPlugin.IO_FORMAT_META_KEY put meta + } + companion object : MetaFormatFactory { override fun invoke(meta: Meta, context: Context): MetaFormat = YamlMetaFormat(meta) diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/EnvelopeFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/EnvelopeFormat.kt index bf5b85f5..c634d696 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/EnvelopeFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/EnvelopeFormat.kt @@ -2,7 +2,6 @@ package hep.dataforge.io import hep.dataforge.context.Context import hep.dataforge.io.EnvelopeFormatFactory.Companion.ENVELOPE_FORMAT_TYPE -import hep.dataforge.meta.EmptyMeta import hep.dataforge.meta.Meta import hep.dataforge.names.Name import hep.dataforge.names.asName @@ -26,7 +25,7 @@ interface EnvelopeFormat : IOFormat { fun Output.writeEnvelope( envelope: Envelope, metaFormatFactory: MetaFormatFactory = defaultMetaFormat, - formatMeta: Meta = EmptyMeta + formatMeta: Meta = Meta.EMPTY ) override fun Input.readObject(): Envelope diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/EnvelopeParts.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/EnvelopeParts.kt index 4780a71a..19ccb458 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/EnvelopeParts.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/EnvelopeParts.kt @@ -1,123 +1,121 @@ package hep.dataforge.io -import hep.dataforge.context.Global -import hep.dataforge.io.EnvelopeParts.FORMAT_META_KEY -import hep.dataforge.io.EnvelopeParts.FORMAT_NAME_KEY -import hep.dataforge.io.EnvelopeParts.INDEX_KEY -import hep.dataforge.io.EnvelopeParts.MULTIPART_DATA_SEPARATOR -import hep.dataforge.io.EnvelopeParts.MULTIPART_DATA_TYPE -import hep.dataforge.io.EnvelopeParts.SIZE_KEY +import hep.dataforge.io.Envelope.Companion.ENVELOPE_NODE_KEY +import hep.dataforge.io.PartDescriptor.Companion.DEFAULT_MULTIPART_DATA_SEPARATOR +import hep.dataforge.io.PartDescriptor.Companion.MULTIPART_DATA_TYPE +import hep.dataforge.io.PartDescriptor.Companion.MULTIPART_KEY +import hep.dataforge.io.PartDescriptor.Companion.PARTS_KEY +import hep.dataforge.io.PartDescriptor.Companion.PART_FORMAT_KEY +import hep.dataforge.io.PartDescriptor.Companion.SEPARATOR_KEY import hep.dataforge.meta.* import hep.dataforge.names.asName import hep.dataforge.names.plus -import hep.dataforge.names.toName +import kotlinx.io.Binary +import kotlinx.io.writeBinary -object EnvelopeParts { - val MULTIPART_KEY = "multipart".asName() - val SIZE_KEY = Envelope.ENVELOPE_NODE_KEY + MULTIPART_KEY + "size" - val INDEX_KEY = Envelope.ENVELOPE_NODE_KEY + MULTIPART_KEY + "index" - val FORMAT_NAME_KEY = Envelope.ENVELOPE_NODE_KEY + MULTIPART_KEY + "format" - val FORMAT_META_KEY = Envelope.ENVELOPE_NODE_KEY + MULTIPART_KEY + "meta" +private class PartDescriptor : Scheme() { + var offset by int(0) + var size by int(0) + var meta by node() - const val MULTIPART_DATA_SEPARATOR = "\r\n#~PART~#\r\n" + companion object : SchemeSpec(::PartDescriptor) { + val MULTIPART_KEY = ENVELOPE_NODE_KEY + "multipart" + val PARTS_KEY = MULTIPART_KEY + "parts" + val SEPARATOR_KEY = MULTIPART_KEY + "separator" - const val MULTIPART_DATA_TYPE = "envelope.multipart" + const val DEFAULT_MULTIPART_DATA_SEPARATOR = "\r\n#~PART~#\r\n" + + val PART_FORMAT_KEY = "format".asName() + + const val MULTIPART_DATA_TYPE = "envelope.multipart" + } } -/** - * Append multiple serialized envelopes to the data block. Previous data is erased if it was present - */ -@DFExperimental +data class EnvelopePart(val binary: Binary, val description: Meta?) + +typealias EnvelopeParts = List + fun EnvelopeBuilder.multipart( - envelopes: Collection, - format: EnvelopeFormatFactory, - formatMeta: Meta = EmptyMeta + parts: EnvelopeParts, + separator: String = DEFAULT_MULTIPART_DATA_SEPARATOR ) { dataType = MULTIPART_DATA_TYPE - meta { - SIZE_KEY put envelopes.size - FORMAT_NAME_KEY put format.name.toString() - if (!formatMeta.isEmpty()) { - FORMAT_META_KEY put formatMeta + + var offsetCounter = 0 + val separatorSize = separator.length + val partDescriptors = parts.map { (binary, description) -> + offsetCounter += separatorSize + PartDescriptor { + offset = offsetCounter + size = binary.size + meta = description + }.also { + offsetCounter += binary.size } } + + meta { + if (separator != DEFAULT_MULTIPART_DATA_SEPARATOR) { + SEPARATOR_KEY put separator + } + setIndexed(PARTS_KEY, partDescriptors.map { it.toMeta() }) + } + data { - format(formatMeta).run { - envelopes.forEach { - writeRawString(MULTIPART_DATA_SEPARATOR) - writeEnvelope(it) - } + parts.forEach { + writeRawString(separator) + writeBinary(it.binary) } } } -/** - * Create a multipart partition in the envelope adding additional name-index mapping in meta - */ -@DFExperimental -fun EnvelopeBuilder.multipart( - envelopes: Map, - format: EnvelopeFormatFactory, - formatMeta: Meta = EmptyMeta +fun EnvelopeBuilder.envelopes( + envelopes: List, + format: EnvelopeFormat = TaggedEnvelopeFormat, + separator: String = DEFAULT_MULTIPART_DATA_SEPARATOR ) { - dataType = MULTIPART_DATA_TYPE - meta { - SIZE_KEY put envelopes.size - FORMAT_NAME_KEY put format.name.toString() - if (!formatMeta.isEmpty()) { - FORMAT_META_KEY put formatMeta - } + val parts = envelopes.map { + val binary = format.toBinary(it) + EnvelopePart(binary, null) } - data { - format.run { - var counter = 0 - envelopes.forEach { (key, envelope) -> - writeRawString(MULTIPART_DATA_SEPARATOR) - writeEnvelope(envelope) - meta { - append(INDEX_KEY, Meta { - "key" put key - "index" put counter - }) - } - counter++ - } + meta{ + set(MULTIPART_KEY + PART_FORMAT_KEY, format.toMeta()) + } + multipart(parts, separator) +} + +fun Envelope.parts(): EnvelopeParts { + if (data == null) return emptyList() + //TODO add zip folder reader + val parts = meta.getIndexed(PARTS_KEY).values.mapNotNull { it.node }.map { + PartDescriptor.wrap(it) + } + return if (parts.isEmpty()) { + listOf(EnvelopePart(data!!, meta[MULTIPART_KEY].node)) + } else { + parts.map { + val binary = data!!.view(it.offset, it.size) + val meta = Laminate(it.meta, meta[MULTIPART_KEY].node) + EnvelopePart(binary, meta) } } } -@DFExperimental -fun EnvelopeBuilder.multipart( - formatFactory: EnvelopeFormatFactory, - formatMeta: Meta = EmptyMeta, - builder: suspend SequenceScope.() -> Unit -) = multipart(sequence(builder).toList(), formatFactory, formatMeta) +fun EnvelopePart.envelope(format: EnvelopeFormat): Envelope = binary.readWith(format) + +val EnvelopePart.name: String? get() = description?.get("name").string /** - * If given envelope supports multipart data, return a sequence of those parts (could be empty). Otherwise return null. + * Represent envelope part by an envelope */ -@DFExperimental -fun Envelope.parts(io: IOPlugin = Global.plugins.fetch(IOPlugin)): Sequence? { - return when (dataType) { - MULTIPART_DATA_TYPE -> { - val size = meta[SIZE_KEY].int ?: error("Unsized parts not supported yet") - val formatName = meta[FORMAT_NAME_KEY].string?.toName() - ?: error("Inferring parts format is not supported at the moment") - val formatMeta = meta[FORMAT_META_KEY].node ?: EmptyMeta - val format = io.envelopeFormat(formatName, formatMeta) - ?: error("Format $formatName is not resolved by $io") - return format.run { - data?.read { - sequence { - repeat(size) { - val separator = readRawString(MULTIPART_DATA_SEPARATOR.length) - if(separator!= MULTIPART_DATA_SEPARATOR) error("Separator is expected, but $separator found") - yield(readObject()) - } - } - } ?: emptySequence() - } - } - else -> null +fun EnvelopePart.envelope(plugin: IOPlugin): Envelope { + val formatItem = description?.get(PART_FORMAT_KEY) + return if (formatItem != null) { + val format: EnvelopeFormat = plugin.resolveEnvelopeFormat(formatItem) + ?: error("Envelope format for $formatItem is not resolved") + binary.readWith(format) + } else { + error("Envelope description not found") + //SimpleEnvelope(description ?: Meta.EMPTY, binary) } -} +} \ No newline at end of file diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOFormat.kt index 1300afe7..474d882e 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOFormat.kt @@ -4,8 +4,10 @@ import hep.dataforge.context.Context import hep.dataforge.context.Factory import hep.dataforge.context.Named import hep.dataforge.io.IOFormatFactory.Companion.IO_FORMAT_TYPE +import hep.dataforge.io.IOPlugin.Companion.IO_FORMAT_NAME_KEY import hep.dataforge.meta.Meta import hep.dataforge.meta.MetaItem +import hep.dataforge.meta.MetaRepr import hep.dataforge.names.Name import hep.dataforge.names.asName import hep.dataforge.provider.Type @@ -18,12 +20,20 @@ import kotlin.reflect.KClass /** * And interface for reading and writing objects into with IO streams */ -interface IOFormat { +interface IOFormat : MetaRepr { fun Output.writeObject(obj: T) fun Input.readObject(): T } fun Input.readWith(format: IOFormat): T = format.run { readObject() } + +/** + * Read given binary as object using given format + */ +fun Binary.readWith(format: IOFormat): T = read { + readWith(format) +} + fun Output.writeWith(format: IOFormat, obj: T) = format.run { writeObject(obj) } class ListIOFormat(val format: IOFormat) : IOFormat> { @@ -42,6 +52,11 @@ class ListIOFormat(val format: IOFormat) : IOFormat> { List(size) { readObject() } } } + + override fun toMeta(): Meta = Meta { + IO_FORMAT_NAME_KEY put "list" + "contentFormat" put format.toMeta() + } } val IOFormat.list get() = ListIOFormat(this) @@ -57,12 +72,16 @@ fun ObjectPool.fill(block: Buffer.() -> Unit): Buffer { } @Type(IO_FORMAT_TYPE) -interface IOFormatFactory : Factory>, Named { +interface IOFormatFactory : Factory>, Named, MetaRepr { /** * Explicit type for dynamic type checks */ val type: KClass + override fun toMeta(): Meta = Meta { + IO_FORMAT_NAME_KEY put name.toString() + } + companion object { const val IO_FORMAT_TYPE = "io.format" } @@ -99,13 +118,4 @@ object ValueIOFormat : IOFormat, IOFormatFactory { return (BinaryMetaFormat.run { readMetaItem() } as? MetaItem.ValueItem)?.value ?: error("The item is not a value") } -} - -/** - * Read given binary as object using given format - */ -fun Binary.readWith(format: IOFormat): T = format.run { - read { - readObject() - } } \ No newline at end of file diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOPlugin.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOPlugin.kt index 6144a211..d31e4264 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOPlugin.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOPlugin.kt @@ -6,29 +6,51 @@ import hep.dataforge.io.IOFormatFactory.Companion.IO_FORMAT_TYPE import hep.dataforge.io.MetaFormatFactory.Companion.META_FORMAT_TYPE import hep.dataforge.meta.* import hep.dataforge.names.Name -import hep.dataforge.names.get +import hep.dataforge.names.asName +import hep.dataforge.names.toName import kotlin.reflect.KClass class IOPlugin(meta: Meta) : AbstractPlugin(meta) { override val tag: PluginTag get() = Companion.tag + val ioFormatFactories by lazy { + context.content>(IO_FORMAT_TYPE).values + } + + fun resolveIOFormat(item: MetaItem<*>, type: KClass): IOFormat? { + val key = item.string ?: item.node[IO_FORMAT_NAME_KEY]?.string ?: error("Format name not defined") + val name = key.toName() + return ioFormatFactories.find { it.name == name }?.let { + @Suppress("UNCHECKED_CAST") + if (it.type != type) error("Format type ${it.type} is not the same as requested type $type") + else it.invoke(item.node[IO_FORMAT_META_KEY].node ?: Meta.EMPTY, context) as IOFormat + } + } + + val metaFormatFactories by lazy { context.content(META_FORMAT_TYPE).values } - fun metaFormat(key: Short, meta: Meta = EmptyMeta): MetaFormat? = + fun resolveMetaFormat(key: Short, meta: Meta = Meta.EMPTY): MetaFormat? = metaFormatFactories.find { it.key == key }?.invoke(meta) - fun metaFormat(name: String, meta: Meta = EmptyMeta): MetaFormat? = + fun resolveMetaFormat(name: String, meta: Meta = Meta.EMPTY): MetaFormat? = metaFormatFactories.find { it.shortName == name }?.invoke(meta) val envelopeFormatFactories by lazy { context.content(ENVELOPE_FORMAT_TYPE).values } - fun envelopeFormat(name: Name, meta: Meta = EmptyMeta) = + fun resolveEnvelopeFormat(name: Name, meta: Meta = Meta.EMPTY): EnvelopeFormat? = envelopeFormatFactories.find { it.name == name }?.invoke(meta, context) + fun resolveEnvelopeFormat(item: MetaItem<*>): EnvelopeFormat? { + val name = item.string ?: item.node[IO_FORMAT_NAME_KEY]?.string ?: error("Envelope format name not defined") + val meta = item.node[IO_FORMAT_META_KEY].node ?: Meta.EMPTY + return resolveEnvelopeFormat(name.toName(), meta) + } + override fun provideTop(target: String): Map { return when (target) { META_FORMAT_TYPE -> defaultMetaFormats.toMap() @@ -37,20 +59,10 @@ class IOPlugin(meta: Meta) : AbstractPlugin(meta) { } } - val ioFormats: Map> by lazy { - context.content>(IO_FORMAT_TYPE) - } - - fun resolveIOFormat(item: MetaItem<*>, type: KClass): IOFormat? { - val key = item.string ?: item.node["name"]?.string ?: error("Format name not defined") - return ioFormats[key]?.let { - @Suppress("UNCHECKED_CAST") - if (it.type != type) error("Format type ${it.type} is not the same as requested type $type") - else it.invoke(item.node["meta"].node ?: EmptyMeta, context) as IOFormat - } - } - companion object : PluginFactory { + val IO_FORMAT_NAME_KEY = "name".asName() + val IO_FORMAT_META_KEY = "meta".asName() + val defaultMetaFormats: List = listOf(JsonMetaFormat, BinaryMetaFormat) val defaultEnvelopeFormats = listOf(TaggedEnvelopeFormat, TaglessEnvelopeFormat) diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/JsonMetaFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/JsonMetaFormat.kt index 39902338..ac32d314 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/JsonMetaFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/JsonMetaFormat.kt @@ -12,7 +12,6 @@ import hep.dataforge.meta.toMetaItem import kotlinx.io.Input import kotlinx.io.Output import kotlinx.io.readByteArray -import kotlinx.io.text.readUtf8String import kotlinx.io.text.writeUtf8String import kotlinx.serialization.UnstableDefault import kotlinx.serialization.json.Json @@ -26,6 +25,10 @@ class JsonMetaFormat(private val json: Json = DEFAULT_JSON) : MetaFormat { writeUtf8String(json.stringify(JsonObjectSerializer, jsonObject)) } + override fun toMeta(): Meta = Meta{ + IOPlugin.IO_FORMAT_NAME_KEY put name.toString() + } + override fun Input.readMeta(descriptor: NodeDescriptor?): Meta { val str = readByteArray().decodeToString() val jsonElement = json.parseJson(str) diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt index 69c33bbd..10bbdab9 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt @@ -2,6 +2,7 @@ package hep.dataforge.io import hep.dataforge.context.Context import hep.dataforge.meta.Meta +import hep.dataforge.meta.enum import hep.dataforge.meta.get import hep.dataforge.meta.string import hep.dataforge.names.Name @@ -9,7 +10,6 @@ import hep.dataforge.names.plus import hep.dataforge.names.toName import kotlinx.io.* -@ExperimentalIoApi class TaggedEnvelopeFormat( val io: IOPlugin, val version: VERSION = VERSION.DF02 @@ -58,7 +58,7 @@ class TaggedEnvelopeFormat( override fun Input.readObject(): Envelope { val tag = readTag(version) - val metaFormat = io.metaFormat(tag.metaFormatKey) + val metaFormat = io.resolveMetaFormat(tag.metaFormatKey) ?: error("Meta format with key ${tag.metaFormatKey} not found") val meta: Meta = limit(tag.metaSize.toInt()).run { @@ -67,7 +67,7 @@ class TaggedEnvelopeFormat( } } - val data = ByteArray(tag.dataSize.toInt()).also { readByteArray(it) }.asBinary() + val data = readBinary(tag.dataSize.toInt()) return SimpleEnvelope(meta, data) } @@ -75,7 +75,7 @@ class TaggedEnvelopeFormat( override fun Input.readPartial(): PartialEnvelope { val tag = readTag(version) - val metaFormat = io.metaFormat(tag.metaFormatKey) + val metaFormat = io.resolveMetaFormat(tag.metaFormatKey) ?: error("Meta format with key ${tag.metaFormatKey} not found") val meta: Meta = limit(tag.metaSize.toInt()).run { @@ -98,6 +98,13 @@ class TaggedEnvelopeFormat( DF03(24u) } + override fun toMeta(): Meta = Meta { + IOPlugin.IO_FORMAT_NAME_KEY put name.toString() + IOPlugin.IO_FORMAT_META_KEY put { + "version" put version + } + } + companion object : EnvelopeFormatFactory { private const val START_SEQUENCE = "#~" private const val END_SEQUENCE = "~#\r\n" @@ -111,7 +118,9 @@ class TaggedEnvelopeFormat( //Check if appropriate factory exists io.metaFormatFactories.find { it.name == metaFormatName } ?: error("Meta format could not be resolved") - return TaggedEnvelopeFormat(io) + val version: VERSION = meta["version"].enum() ?: VERSION.DF02 + + return TaggedEnvelopeFormat(io, version) } private fun Input.readTag(version: VERSION): Tag { @@ -132,11 +141,13 @@ class TaggedEnvelopeFormat( override fun peekFormat(io: IOPlugin, input: Input): EnvelopeFormat? { return try { - val header = input.readRawString(6) - when (header.substring(2..5)) { - VERSION.DF02.name -> TaggedEnvelopeFormat(io, VERSION.DF02) - VERSION.DF03.name -> TaggedEnvelopeFormat(io, VERSION.DF03) - else -> null + input.preview { + val header = readRawString(6) + return@preview when (header.substring(2..5)) { + VERSION.DF02.name -> TaggedEnvelopeFormat(io, VERSION.DF02) + VERSION.DF03.name -> TaggedEnvelopeFormat(io, VERSION.DF03) + else -> null + } } } catch (ex: Exception) { null diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaglessEnvelopeFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaglessEnvelopeFormat.kt index f29132c8..2d86f814 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaglessEnvelopeFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaglessEnvelopeFormat.kt @@ -1,16 +1,19 @@ package hep.dataforge.io import hep.dataforge.context.Context -import hep.dataforge.meta.* +import hep.dataforge.meta.Meta +import hep.dataforge.meta.get +import hep.dataforge.meta.isEmpty +import hep.dataforge.meta.string import hep.dataforge.names.asName import kotlinx.io.* import kotlinx.io.text.readUtf8Line import kotlinx.io.text.writeUtf8String +import kotlin.collections.set -@ExperimentalIoApi class TaglessEnvelopeFormat( val io: IOPlugin, - meta: Meta = EmptyMeta + val meta: Meta = Meta.EMPTY ) : EnvelopeFormat { private val metaStart = meta[META_START_PROPERTY].string ?: DEFAULT_META_START @@ -69,10 +72,10 @@ class TaglessEnvelopeFormat( line = readUtf8Line() } - var meta: Meta = EmptyMeta + var meta: Meta = Meta.EMPTY if (line.startsWith(metaStart)) { - val metaFormat = properties[META_TYPE_PROPERTY]?.let { io.metaFormat(it) } ?: JsonMetaFormat + val metaFormat = properties[META_TYPE_PROPERTY]?.let { io.resolveMetaFormat(it) } ?: JsonMetaFormat val metaSize = properties[META_LENGTH_PROPERTY]?.toInt() meta = if (metaSize != null) { limit(metaSize).run { @@ -95,9 +98,10 @@ class TaglessEnvelopeFormat( } while (!line.startsWith(dataStart)) val data: Binary? = if (properties.containsKey(DATA_LENGTH_PROPERTY)) { - val bytes = ByteArray(properties[DATA_LENGTH_PROPERTY]!!.toInt()) - readByteArray(bytes) - bytes.asBinary() + readBinary(properties[DATA_LENGTH_PROPERTY]!!.toInt()) +// val bytes = ByteArray(properties[DATA_LENGTH_PROPERTY]!!.toInt()) +// readByteArray(bytes) +// bytes.asBinary() } else { Binary { copyTo(this) @@ -132,10 +136,10 @@ class TaglessEnvelopeFormat( } } - var meta: Meta = EmptyMeta + var meta: Meta = Meta.EMPTY if (line.startsWith(metaStart)) { - val metaFormat = properties[META_TYPE_PROPERTY]?.let { io.metaFormat(it) } ?: JsonMetaFormat + val metaFormat = properties[META_TYPE_PROPERTY]?.let { io.resolveMetaFormat(it) } ?: JsonMetaFormat val metaSize = properties[META_LENGTH_PROPERTY]?.toInt() meta = if (metaSize != null) { offset += metaSize.toUInt() @@ -157,6 +161,11 @@ class TaglessEnvelopeFormat( return PartialEnvelope(meta, offset, dataSize) } + override fun toMeta(): Meta = Meta { + IOPlugin.IO_FORMAT_NAME_KEY put name.toString() + IOPlugin.IO_FORMAT_META_KEY put meta + } + companion object : EnvelopeFormatFactory { private val propertyPattern = "#\\?\\s*(?[\\w.]*)\\s*:\\s*(?[^;]*);?".toRegex() @@ -195,11 +204,13 @@ class TaglessEnvelopeFormat( override fun peekFormat(io: IOPlugin, input: Input): EnvelopeFormat? { return try { - val string = input.readRawString(TAGLESS_ENVELOPE_HEADER.length) - return if (string == TAGLESS_ENVELOPE_HEADER) { - TaglessEnvelopeFormat(io) - } else { - null + input.preview { + val string = readRawString(TAGLESS_ENVELOPE_HEADER.length) + return@preview if (string == TAGLESS_ENVELOPE_HEADER) { + TaglessEnvelopeFormat(io) + } else { + null + } } } catch (ex: Exception) { null diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/ioMisc.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/ioMisc.kt index eb5341d6..eddb7b40 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/ioMisc.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/ioMisc.kt @@ -1,6 +1,7 @@ package hep.dataforge.io import kotlinx.io.* +import kotlin.math.min fun Output.writeRawString(str: String) { str.forEach { writeByte(it.toByte()) } @@ -18,5 +19,24 @@ inline fun buildByteArray(expectedSize: Int = 16, block: Output.() -> Unit): Byt inline fun Binary(expectedSize: Int = 16, block: Output.() -> Unit): Binary = buildByteArray(expectedSize, block).asBinary() -@Deprecated("To be replaced by Binary.EMPTY",level = DeprecationLevel.WARNING) -val EmptyBinary = ByteArrayBinary(ByteArray(0)) \ No newline at end of file +@Deprecated("To be replaced by Binary.EMPTY", level = DeprecationLevel.WARNING) +val EmptyBinary = ByteArrayBinary(ByteArray(0)) + +/** + * View section of a [Binary] as an independent binary + */ +class BinaryView(private val source: Binary, private val start: Int, override val size: Int) : Binary { + + init { + require(start > 0) + require(start + size <= source.size) { "View boundary is outside source binary size" } + } + + override fun read(offset: Int, atMost: Int, block: Input.() -> R): R { + return source.read(start + offset, min(size, atMost), block) + } +} + +fun Binary.view(start: Int, size: Int) = BinaryView(this, start, size) + +operator fun Binary.get(range: IntRange) = view(range.first, range.last - range.first) \ No newline at end of file diff --git a/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/BinaryTest.kt b/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/BinaryTest.kt new file mode 100644 index 00000000..157b695f --- /dev/null +++ b/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/BinaryTest.kt @@ -0,0 +1,20 @@ +package hep.dataforge.io + +import kotlinx.io.asBinary +import kotlinx.io.readByte +import kotlinx.io.readInt +import kotlin.test.Test +import kotlin.test.assertEquals + +class BinaryTest { + @Test + fun testBinaryAccess(){ + val binary = ByteArray(128){it.toByte()}.asBinary() + + binary[3..12].read { + readInt() + val res = readByte() + assertEquals(7, res) + } + } +} \ No newline at end of file diff --git a/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/MultipartTest.kt b/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/MultipartTest.kt index 344edf59..1b239edd 100644 --- a/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/MultipartTest.kt +++ b/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/MultipartTest.kt @@ -1,17 +1,19 @@ package hep.dataforge.io +import hep.dataforge.context.Global import hep.dataforge.meta.DFExperimental import hep.dataforge.meta.get import hep.dataforge.meta.int import kotlinx.io.text.writeUtf8String - import kotlin.test.Test import kotlin.test.assertEquals import kotlin.test.assertTrue @DFExperimental class MultipartTest { - val envelopes = (0..5).map { + val io: IOPlugin = Global.io + + val envelopes = (0 until 5).map { Envelope { meta { "value" put it @@ -26,19 +28,21 @@ class MultipartTest { } val partsEnvelope = Envelope { - multipart(envelopes, TaggedEnvelopeFormat) + envelopes(envelopes, TaglessEnvelopeFormat) } @Test fun testParts() { - TaggedEnvelopeFormat.run { + TaglessEnvelopeFormat.run { val singleEnvelopeData = toBinary(envelopes[0]) val singleEnvelopeSize = singleEnvelopeData.size val bytes = toBinary(partsEnvelope) - assertTrue(5*singleEnvelopeSize < bytes.size) + assertTrue(envelopes.size * singleEnvelopeSize < bytes.size) val reconstructed = bytes.readWith(this) - val parts = reconstructed.parts()?.toList() ?: emptyList() - assertEquals(2, parts[2].meta["value"].int) + println(reconstructed.meta) + val parts = reconstructed.parts() + val envelope = parts[2].envelope(io) + assertEquals(2, envelope.meta["value"].int) println(reconstructed.data!!.size) } } diff --git a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/fileIO.kt b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/fileIO.kt index 94b9b433..044cf932 100644 --- a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/fileIO.kt +++ b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/fileIO.kt @@ -1,7 +1,6 @@ package hep.dataforge.io import hep.dataforge.meta.DFExperimental -import hep.dataforge.meta.EmptyMeta import hep.dataforge.meta.Meta import hep.dataforge.meta.descriptors.NodeDescriptor import hep.dataforge.meta.isEmpty @@ -60,7 +59,7 @@ fun Path.readEnvelope(format: EnvelopeFormat): Envelope { @Suppress("UNCHECKED_CAST") @DFExperimental inline fun IOPlugin.resolveIOFormat(): IOFormat? { - return ioFormats.values.find { it.type.isSuperclassOf(T::class) } as IOFormat? + return ioFormatFactories.find { it.type.isSuperclassOf(T::class) } as IOFormat? } /** @@ -78,7 +77,7 @@ fun IOPlugin.readMetaFile(path: Path, formatOverride: MetaFormat? = null, descri } val extension = actualPath.fileName.toString().substringAfterLast('.') - val metaFormat = formatOverride ?: metaFormat(extension) ?: error("Can't resolve meta format $extension") + val metaFormat = formatOverride ?: resolveMetaFormat(extension) ?: error("Can't resolve meta format $extension") return metaFormat.run { actualPath.read { readMeta(descriptor) @@ -157,7 +156,7 @@ fun IOPlugin.readEnvelopeFile( .singleOrNull { it.fileName.toString().startsWith(IOPlugin.META_FILE_NAME) } val meta = if (metaFile == null) { - EmptyMeta + Meta.EMPTY } else { readMetaFile(metaFile) } diff --git a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/functionsJVM.kt b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/functionsJVM.kt index ffb924ef..ccd57c55 100644 --- a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/functionsJVM.kt +++ b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/functionsJVM.kt @@ -9,7 +9,7 @@ import kotlin.reflect.full.isSuperclassOf fun IOPlugin.resolveIOFormatName(type: KClass<*>): Name { - return ioFormats.entries.find { it.value.type.isSuperclassOf(type) }?.key + return ioFormatFactories.find { it.type.isSuperclassOf(type) }?.name ?: error("Can't resolve IOFormat for type $type") } diff --git a/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/FileEnvelopeTest.kt b/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/FileEnvelopeTest.kt index edee906b..8db2bf62 100644 --- a/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/FileEnvelopeTest.kt +++ b/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/FileEnvelopeTest.kt @@ -1,12 +1,14 @@ package hep.dataforge.io import hep.dataforge.context.Global +import hep.dataforge.meta.DFExperimental import kotlinx.io.writeDouble import java.nio.file.Files import kotlin.test.Test import kotlin.test.assertTrue +@DFExperimental class FileEnvelopeTest { val envelope = Envelope { meta { diff --git a/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/tcp/EnvelopeServerTest.kt b/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/tcp/EnvelopeServerTest.kt index 5847f46d..3e545421 100644 --- a/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/tcp/EnvelopeServerTest.kt +++ b/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/tcp/EnvelopeServerTest.kt @@ -46,7 +46,7 @@ class EnvelopeServerTest { @Test(timeout = 1000) fun doEchoTest() { - val request = Envelope.invoke { + val request = Envelope { type = "test.echo" meta { "test.value" put 22 diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Meta.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Meta.kt index 672a1922..75cae67d 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Meta.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Meta.kt @@ -98,7 +98,7 @@ interface Meta : MetaRepr { */ const val VALUE_KEY = "@value" - val EMPTY: EmptyMeta = EmptyMeta + val EMPTY = EmptyMeta } } @@ -188,7 +188,7 @@ abstract class MetaBase : Meta { override fun hashCode(): Int = items.hashCode() - override fun toString(): String = toJson().toString() + override fun toString(): String = PRETTY_JSON.stringify(MetaSerializer, this) } /** @@ -216,6 +216,7 @@ fun MetaItem<*>.seal(): MetaItem = when (this) { is NodeItem -> NodeItem(node.seal()) } +@Deprecated("Use Meta.EMPTY instead", replaceWith = ReplaceWith("Meta.EMPTY")) object EmptyMeta : MetaBase() { override val items: Map> = emptyMap() } @@ -251,4 +252,4 @@ val MetaItem?.node: M? is NodeItem -> node } -fun Meta.isEmpty() = this === EmptyMeta || this.items.isEmpty() \ No newline at end of file +fun Meta.isEmpty() = this === Meta.EMPTY || this.items.isEmpty() \ No newline at end of file diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MutableMetaDelegate.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MutableMetaDelegate.kt index 3fcdc9da..0cffb3a3 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MutableMetaDelegate.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MutableMetaDelegate.kt @@ -34,9 +34,7 @@ class ReadWriteDelegateWrapper( val reader: (T) -> R, val writer: (R) -> T ) : ReadWriteProperty { - override fun getValue(thisRef: Any?, property: KProperty<*>): R { - return reader(delegate.getValue(thisRef, property)) - } + override fun getValue(thisRef: Any?, property: KProperty<*>): R = reader(delegate.getValue(thisRef, property)) override fun setValue(thisRef: Any?, property: KProperty<*>, value: R) { delegate.setValue(thisRef, property, writer(value)) diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Specification.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Specification.kt index 3724ce38..85cf7b96 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Specification.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Specification.kt @@ -16,7 +16,7 @@ interface Specification { */ fun wrap(config: Config = Config(), defaultProvider: (Name) -> MetaItem<*>? = { null }): T - operator fun invoke(action: T.() -> Unit) = empty().apply(action) + operator fun invoke(action: T.() -> Unit): T = empty().apply(action) } /** @@ -27,12 +27,16 @@ fun Specification.update(config: Config, action: T.() -> U /** * Wrap a configuration using static meta as default */ -fun Specification.wrap(config: Config = Config(), default: Meta): T = wrap(config) { default[it] } +fun Specification.wrap(config: Config = Config(), default: Meta = Meta.EMPTY): T = + wrap(config) { default[it] } /** * Wrap a configuration using static meta as default */ -fun Specification.wrap(default: Meta): T = wrap(Config()) { default[it] } +fun Specification.wrap(source: Meta): T { + val default = source.seal() + return wrap(source.asConfig(), default) +} /** diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/serializationUtils.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/serializationUtils.kt index 16c58bdc..0ac825b9 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/serializationUtils.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/serializationUtils.kt @@ -3,6 +3,8 @@ package hep.dataforge.meta import kotlinx.serialization.* import kotlinx.serialization.builtins.DoubleArraySerializer import kotlinx.serialization.builtins.serializer +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.JsonConfiguration fun SerialDescriptorBuilder.boolean(name: String, isOptional: Boolean = false, vararg annotations: Annotation) = element(name, Boolean.serializer().descriptor, isOptional = isOptional, annotations = annotations.toList()) @@ -62,4 +64,6 @@ inline fun Encoder.encodeStructure( val encoder = beginStructure(desc, *typeParams) encoder.block() encoder.endStructure(desc) -} \ No newline at end of file +} + +val PRETTY_JSON = Json(JsonConfiguration(prettyPrint = true, useArrayPolymorphism = true)) \ No newline at end of file