kscience/dataforge-core
7
0
Fork 0
Code Issues Pull Requests 1 Packages Projects Releases 12 Wiki Activity

Fix Envelope IO with binaries

Browse Source
This commit is contained in:
Alexander Nozik 2020-04-05 21:12:56 +03:00
parent 7efa19920b
commit eebfe534cc
23 changed files with 295 additions and 192 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
dataforge-context/src/commonMain/kotlin/hep/dataforge/context/AbstractPlugin.kt
View File

@ -1,13 +1,12 @@
package hep.dataforge.context
import hep.dataforge.meta.EmptyMeta
import hep.dataforge.meta.Meta
import hep.dataforge.names.Name
import kotlin.properties.ReadOnlyProperty
import kotlin.reflect.KClass
import kotlin.reflect.KProperty
abstract class AbstractPlugin(override val meta: Meta = EmptyMeta) : Plugin {
abstract class AbstractPlugin(override val meta: Meta = Meta.EMPTY) : Plugin {
private var _context: Context? = null
private val dependencies = ArrayList<PluginFactory<*>>()

3
dataforge-context/src/commonMain/kotlin/hep/dataforge/context/Factory.kt
View File

@ -1,8 +1,7 @@
package hep.dataforge.context
import hep.dataforge.meta.EmptyMeta
import hep.dataforge.meta.Meta
interface Factory<out T : Any> {
operator fun invoke(meta: Meta = EmptyMeta, context: Context = Global): T
operator fun invoke(meta: Meta = Meta.EMPTY, context: Context = Global): T
}

3
dataforge-context/src/commonMain/kotlin/hep/dataforge/context/PluginRepository.kt
View File

@ -1,6 +1,5 @@
package hep.dataforge.context
import hep.dataforge.meta.EmptyMeta
import hep.dataforge.meta.Meta
import kotlin.reflect.KClass
@ -23,7 +22,7 @@ expect object PluginRepository {
/**
* Fetch specific plugin and instantiate it with given meta
*/
fun PluginRepository.fetch(tag: PluginTag, meta: Meta = EmptyMeta): Plugin =
fun PluginRepository.fetch(tag: PluginTag, meta: Meta = Meta.EMPTY): Plugin =
list().find { it.tag.matches(tag) }?.invoke(meta = meta)
?: error("Plugin with tag $tag not found in the repository")

26
dataforge-io/dataforge-io-yaml/src/main/kotlin/hep/dataforge/io/yaml/FrontMatterEnvelopeFormat.kt
View File

@ -3,7 +3,6 @@ package hep.dataforge.io.yaml
import hep.dataforge.context.Context
import hep.dataforge.io.*
import hep.dataforge.meta.DFExperimental
import hep.dataforge.meta.EmptyMeta
import hep.dataforge.meta.Meta
import kotlinx.io.*
import kotlinx.io.text.readUtf8Line
@ -13,7 +12,7 @@ import kotlinx.serialization.toUtf8Bytes
@DFExperimental
class FrontMatterEnvelopeFormat(
val io: IOPlugin,
meta: Meta = EmptyMeta
val meta: Meta = Meta.EMPTY
) : EnvelopeFormat {
override fun Input.readPartial(): PartialEnvelope {
@ -26,7 +25,7 @@ class FrontMatterEnvelopeFormat(
val readMetaFormat =
metaTypeRegex.matchEntire(line)?.groupValues?.first()
?.let { io.metaFormat(it) } ?: YamlMetaFormat
?.let { io.resolveMetaFormat(it) } ?: YamlMetaFormat
//TODO replace by preview
val meta = Binary {
@ -51,11 +50,11 @@ class FrontMatterEnvelopeFormat(
val readMetaFormat =
metaTypeRegex.matchEntire(line)?.groupValues?.first()
?.let { io.metaFormat(it) } ?: YamlMetaFormat
?.let { io.resolveMetaFormat(it) } ?: YamlMetaFormat
val meta = Binary {
do {
writeUtf8String(readUtf8Line() + "\r\n")
writeUtf8String(readUtf8Line() + "\r\n")
} while (!line.startsWith(SEPARATOR))
}.read {
readMetaFormat.run {
@ -78,6 +77,11 @@ class FrontMatterEnvelopeFormat(
}
}
override fun toMeta(): Meta = Meta {
IOPlugin.IO_FORMAT_NAME_KEY put name.toString()
IOPlugin.IO_FORMAT_META_KEY put meta
}
companion object : EnvelopeFormatFactory {
const val SEPARATOR = "---"
@ -88,11 +92,13 @@ class FrontMatterEnvelopeFormat(
}
override fun peekFormat(io: IOPlugin, input: Input): EnvelopeFormat? {
val line = input.readUtf8Line()
return if (line.startsWith("---")) {
invoke()
} else {
null
return input.preview {
val line = readUtf8Line()
return@preview if (line.startsWith("---")) {
invoke()
} else {
null
}
}
}

8
dataforge-io/dataforge-io-yaml/src/main/kotlin/hep/dataforge/io/yaml/YamlMetaFormat.kt
View File

@ -1,6 +1,7 @@
package hep.dataforge.io.yaml
import hep.dataforge.context.Context
import hep.dataforge.io.IOPlugin
import hep.dataforge.io.MetaFormat
import hep.dataforge.io.MetaFormatFactory
import hep.dataforge.meta.DFExperimental
@ -11,10 +12,8 @@ import hep.dataforge.meta.toMeta
import kotlinx.io.Input
import kotlinx.io.Output
import kotlinx.io.asInputStream
import kotlinx.io.readUByte
import kotlinx.io.text.writeUtf8String
import org.yaml.snakeyaml.Yaml
import java.io.InputStream
@DFExperimental
class YamlMetaFormat(val meta: Meta) : MetaFormat {
@ -30,6 +29,11 @@ class YamlMetaFormat(val meta: Meta) : MetaFormat {
return map.toMeta(descriptor)
}
override fun toMeta(): Meta = Meta{
IOPlugin.IO_FORMAT_NAME_KEY put FrontMatterEnvelopeFormat.name.toString()
IOPlugin.IO_FORMAT_META_KEY put meta
}
companion object : MetaFormatFactory {
override fun invoke(meta: Meta, context: Context): MetaFormat = YamlMetaFormat(meta)

3
dataforge-io/src/commonMain/kotlin/hep/dataforge/io/EnvelopeFormat.kt
View File

@ -2,7 +2,6 @@ package hep.dataforge.io
import hep.dataforge.context.Context
import hep.dataforge.io.EnvelopeFormatFactory.Companion.ENVELOPE_FORMAT_TYPE
import hep.dataforge.meta.EmptyMeta
import hep.dataforge.meta.Meta
import hep.dataforge.names.Name
import hep.dataforge.names.asName
@ -26,7 +25,7 @@ interface EnvelopeFormat : IOFormat<Envelope> {
fun Output.writeEnvelope(
envelope: Envelope,
metaFormatFactory: MetaFormatFactory = defaultMetaFormat,
formatMeta: Meta = EmptyMeta
formatMeta: Meta = Meta.EMPTY
)
override fun Input.readObject(): Envelope

184
dataforge-io/src/commonMain/kotlin/hep/dataforge/io/EnvelopeParts.kt
View File

@ -1,123 +1,121 @@
package hep.dataforge.io
import hep.dataforge.context.Global
import hep.dataforge.io.EnvelopeParts.FORMAT_META_KEY
import hep.dataforge.io.EnvelopeParts.FORMAT_NAME_KEY
import hep.dataforge.io.EnvelopeParts.INDEX_KEY
import hep.dataforge.io.EnvelopeParts.MULTIPART_DATA_SEPARATOR
import hep.dataforge.io.EnvelopeParts.MULTIPART_DATA_TYPE
import hep.dataforge.io.EnvelopeParts.SIZE_KEY
import hep.dataforge.io.Envelope.Companion.ENVELOPE_NODE_KEY
import hep.dataforge.io.PartDescriptor.Companion.DEFAULT_MULTIPART_DATA_SEPARATOR
import hep.dataforge.io.PartDescriptor.Companion.MULTIPART_DATA_TYPE
import hep.dataforge.io.PartDescriptor.Companion.MULTIPART_KEY
import hep.dataforge.io.PartDescriptor.Companion.PARTS_KEY
import hep.dataforge.io.PartDescriptor.Companion.PART_FORMAT_KEY
import hep.dataforge.io.PartDescriptor.Companion.SEPARATOR_KEY
import hep.dataforge.meta.*
import hep.dataforge.names.asName
import hep.dataforge.names.plus
import hep.dataforge.names.toName
import kotlinx.io.Binary
import kotlinx.io.writeBinary
object EnvelopeParts {
val MULTIPART_KEY = "multipart".asName()
val SIZE_KEY = Envelope.ENVELOPE_NODE_KEY + MULTIPART_KEY + "size"
val INDEX_KEY = Envelope.ENVELOPE_NODE_KEY + MULTIPART_KEY + "index"
val FORMAT_NAME_KEY = Envelope.ENVELOPE_NODE_KEY + MULTIPART_KEY + "format"
val FORMAT_META_KEY = Envelope.ENVELOPE_NODE_KEY + MULTIPART_KEY + "meta"
private class PartDescriptor : Scheme() {
var offset by int(0)
var size by int(0)
var meta by node()
const val MULTIPART_DATA_SEPARATOR = "\r\n#~PART~#\r\n"
companion object : SchemeSpec<PartDescriptor>(::PartDescriptor) {
val MULTIPART_KEY = ENVELOPE_NODE_KEY + "multipart"
val PARTS_KEY = MULTIPART_KEY + "parts"
val SEPARATOR_KEY = MULTIPART_KEY + "separator"
const val MULTIPART_DATA_TYPE = "envelope.multipart"
const val DEFAULT_MULTIPART_DATA_SEPARATOR = "\r\n#~PART~#\r\n"
val PART_FORMAT_KEY = "format".asName()
const val MULTIPART_DATA_TYPE = "envelope.multipart"
}
}
/**
* Append multiple serialized envelopes to the data block. Previous data is erased if it was present
*/
@DFExperimental
data class EnvelopePart(val binary: Binary, val description: Meta?)
typealias EnvelopeParts = List<EnvelopePart>
fun EnvelopeBuilder.multipart(
envelopes: Collection<Envelope>,
format: EnvelopeFormatFactory,
formatMeta: Meta = EmptyMeta
parts: EnvelopeParts,
separator: String = DEFAULT_MULTIPART_DATA_SEPARATOR
) {
dataType = MULTIPART_DATA_TYPE
meta {
SIZE_KEY put envelopes.size
FORMAT_NAME_KEY put format.name.toString()
if (!formatMeta.isEmpty()) {
FORMAT_META_KEY put formatMeta
var offsetCounter = 0
val separatorSize = separator.length
val partDescriptors = parts.map { (binary, description) ->
offsetCounter += separatorSize
PartDescriptor {
offset = offsetCounter
size = binary.size
meta = description
}.also {
offsetCounter += binary.size
}
}
meta {
if (separator != DEFAULT_MULTIPART_DATA_SEPARATOR) {
SEPARATOR_KEY put separator
}
setIndexed(PARTS_KEY, partDescriptors.map { it.toMeta() })
}
data {
format(formatMeta).run {
envelopes.forEach {
writeRawString(MULTIPART_DATA_SEPARATOR)
writeEnvelope(it)
}
parts.forEach {
writeRawString(separator)
writeBinary(it.binary)
}
}
}
/**
* Create a multipart partition in the envelope adding additional name-index mapping in meta
*/
@DFExperimental
fun EnvelopeBuilder.multipart(
envelopes: Map<String, Envelope>,
format: EnvelopeFormatFactory,
formatMeta: Meta = EmptyMeta
fun EnvelopeBuilder.envelopes(
envelopes: List<Envelope>,
format: EnvelopeFormat = TaggedEnvelopeFormat,
separator: String = DEFAULT_MULTIPART_DATA_SEPARATOR
) {
dataType = MULTIPART_DATA_TYPE
meta {
SIZE_KEY put envelopes.size
FORMAT_NAME_KEY put format.name.toString()
if (!formatMeta.isEmpty()) {
FORMAT_META_KEY put formatMeta
}
val parts = envelopes.map {
val binary = format.toBinary(it)
EnvelopePart(binary, null)
}
data {
format.run {
var counter = 0
envelopes.forEach { (key, envelope) ->
writeRawString(MULTIPART_DATA_SEPARATOR)
writeEnvelope(envelope)
meta {
append(INDEX_KEY, Meta {
"key" put key
"index" put counter
})
}
counter++
}
meta{
set(MULTIPART_KEY + PART_FORMAT_KEY, format.toMeta())
}
multipart(parts, separator)
}
fun Envelope.parts(): EnvelopeParts {
if (data == null) return emptyList()
//TODO add zip folder reader
val parts = meta.getIndexed(PARTS_KEY).values.mapNotNull { it.node }.map {
PartDescriptor.wrap(it)
}
return if (parts.isEmpty()) {
listOf(EnvelopePart(data!!, meta[MULTIPART_KEY].node))
} else {
parts.map {
val binary = data!!.view(it.offset, it.size)
val meta = Laminate(it.meta, meta[MULTIPART_KEY].node)
EnvelopePart(binary, meta)
}
}
}
@DFExperimental
fun EnvelopeBuilder.multipart(
formatFactory: EnvelopeFormatFactory,
formatMeta: Meta = EmptyMeta,
builder: suspend SequenceScope<Envelope>.() -> Unit
) = multipart(sequence(builder).toList(), formatFactory, formatMeta)
fun EnvelopePart.envelope(format: EnvelopeFormat): Envelope = binary.readWith(format)
val EnvelopePart.name: String? get() = description?.get("name").string
/**
* If given envelope supports multipart data, return a sequence of those parts (could be empty). Otherwise return null.
* Represent envelope part by an envelope
*/
@DFExperimental
fun Envelope.parts(io: IOPlugin = Global.plugins.fetch(IOPlugin)): Sequence<Envelope>? {
return when (dataType) {
MULTIPART_DATA_TYPE -> {
val size = meta[SIZE_KEY].int ?: error("Unsized parts not supported yet")
val formatName = meta[FORMAT_NAME_KEY].string?.toName()
?: error("Inferring parts format is not supported at the moment")
val formatMeta = meta[FORMAT_META_KEY].node ?: EmptyMeta
val format = io.envelopeFormat(formatName, formatMeta)
?: error("Format $formatName is not resolved by $io")
return format.run {
data?.read {
sequence {
repeat(size) {
val separator = readRawString(MULTIPART_DATA_SEPARATOR.length)
if(separator!= MULTIPART_DATA_SEPARATOR) error("Separator is expected, but $separator found")
yield(readObject())
}
}
} ?: emptySequence()
}
}
else -> null
fun EnvelopePart.envelope(plugin: IOPlugin): Envelope {
val formatItem = description?.get(PART_FORMAT_KEY)
return if (formatItem != null) {
val format: EnvelopeFormat = plugin.resolveEnvelopeFormat(formatItem)
?: error("Envelope format for $formatItem is not resolved")
binary.readWith(format)
} else {
error("Envelope description not found")
//SimpleEnvelope(description ?: Meta.EMPTY, binary)
}
}
}

32
dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOFormat.kt
View File

@ -4,8 +4,10 @@ import hep.dataforge.context.Context
import hep.dataforge.context.Factory
import hep.dataforge.context.Named
import hep.dataforge.io.IOFormatFactory.Companion.IO_FORMAT_TYPE
import hep.dataforge.io.IOPlugin.Companion.IO_FORMAT_NAME_KEY
import hep.dataforge.meta.Meta
import hep.dataforge.meta.MetaItem
import hep.dataforge.meta.MetaRepr
import hep.dataforge.names.Name
import hep.dataforge.names.asName
import hep.dataforge.provider.Type
@ -18,12 +20,20 @@ import kotlin.reflect.KClass
/**
* And interface for reading and writing objects into with IO streams
*/
interface IOFormat<T : Any> {
interface IOFormat<T : Any> : MetaRepr {
fun Output.writeObject(obj: T)
fun Input.readObject(): T
}
fun <T : Any> Input.readWith(format: IOFormat<T>): T = format.run { readObject() }
/**
* Read given binary as object using given format
*/
fun <T : Any> Binary.readWith(format: IOFormat<T>): T = read {
readWith(format)
}
fun <T : Any> Output.writeWith(format: IOFormat<T>, obj: T) = format.run { writeObject(obj) }
class ListIOFormat<T : Any>(val format: IOFormat<T>) : IOFormat<List<T>> {
@ -42,6 +52,11 @@ class ListIOFormat<T : Any>(val format: IOFormat<T>) : IOFormat<List<T>> {
List(size) { readObject() }
}
}
override fun toMeta(): Meta = Meta {
IO_FORMAT_NAME_KEY put "list"
"contentFormat" put format.toMeta()
}
}
val <T : Any> IOFormat<T>.list get() = ListIOFormat(this)
@ -57,12 +72,16 @@ fun ObjectPool<Buffer>.fill(block: Buffer.() -> Unit): Buffer {
}
@Type(IO_FORMAT_TYPE)
interface IOFormatFactory<T : Any> : Factory<IOFormat<T>>, Named {
interface IOFormatFactory<T : Any> : Factory<IOFormat<T>>, Named, MetaRepr {
/**
* Explicit type for dynamic type checks
*/
val type: KClass<out T>
override fun toMeta(): Meta = Meta {
IO_FORMAT_NAME_KEY put name.toString()
}
companion object {
const val IO_FORMAT_TYPE = "io.format"
}
@ -99,13 +118,4 @@ object ValueIOFormat : IOFormat<Value>, IOFormatFactory<Value> {
return (BinaryMetaFormat.run { readMetaItem() } as? MetaItem.ValueItem)?.value
?: error("The item is not a value")
}
}
/**
* Read given binary as object using given format
*/
fun <T : Any> Binary.readWith(format: IOFormat<T>): T = format.run {
read {
readObject()
}
}

46
dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOPlugin.kt
View File

@ -6,29 +6,51 @@ import hep.dataforge.io.IOFormatFactory.Companion.IO_FORMAT_TYPE
import hep.dataforge.io.MetaFormatFactory.Companion.META_FORMAT_TYPE
import hep.dataforge.meta.*
import hep.dataforge.names.Name
import hep.dataforge.names.get
import hep.dataforge.names.asName
import hep.dataforge.names.toName
import kotlin.reflect.KClass
class IOPlugin(meta: Meta) : AbstractPlugin(meta) {
override val tag: PluginTag get() = Companion.tag
val ioFormatFactories by lazy {
context.content<IOFormatFactory<*>>(IO_FORMAT_TYPE).values
}
fun <T : Any> resolveIOFormat(item: MetaItem<*>, type: KClass<out T>): IOFormat<T>? {
val key = item.string ?: item.node[IO_FORMAT_NAME_KEY]?.string ?: error("Format name not defined")
val name = key.toName()
return ioFormatFactories.find { it.name == name }?.let {
@Suppress("UNCHECKED_CAST")
if (it.type != type) error("Format type ${it.type} is not the same as requested type $type")
else it.invoke(item.node[IO_FORMAT_META_KEY].node ?: Meta.EMPTY, context) as IOFormat<T>
}
}
val metaFormatFactories by lazy {
context.content<MetaFormatFactory>(META_FORMAT_TYPE).values
}
fun metaFormat(key: Short, meta: Meta = EmptyMeta): MetaFormat? =
fun resolveMetaFormat(key: Short, meta: Meta = Meta.EMPTY): MetaFormat? =
metaFormatFactories.find { it.key == key }?.invoke(meta)
fun metaFormat(name: String, meta: Meta = EmptyMeta): MetaFormat? =
fun resolveMetaFormat(name: String, meta: Meta = Meta.EMPTY): MetaFormat? =
metaFormatFactories.find { it.shortName == name }?.invoke(meta)
val envelopeFormatFactories by lazy {
context.content<EnvelopeFormatFactory>(ENVELOPE_FORMAT_TYPE).values
}
fun envelopeFormat(name: Name, meta: Meta = EmptyMeta) =
fun resolveEnvelopeFormat(name: Name, meta: Meta = Meta.EMPTY): EnvelopeFormat? =
envelopeFormatFactories.find { it.name == name }?.invoke(meta, context)
fun resolveEnvelopeFormat(item: MetaItem<*>): EnvelopeFormat? {
val name = item.string ?: item.node[IO_FORMAT_NAME_KEY]?.string ?: error("Envelope format name not defined")
val meta = item.node[IO_FORMAT_META_KEY].node ?: Meta.EMPTY
return resolveEnvelopeFormat(name.toName(), meta)
}
override fun provideTop(target: String): Map<Name, Any> {
return when (target) {
META_FORMAT_TYPE -> defaultMetaFormats.toMap()
@ -37,20 +59,10 @@ class IOPlugin(meta: Meta) : AbstractPlugin(meta) {
}
}
val ioFormats: Map<Name, IOFormatFactory<*>> by lazy {
context.content<IOFormatFactory<*>>(IO_FORMAT_TYPE)
}
fun <T : Any> resolveIOFormat(item: MetaItem<*>, type: KClass<out T>): IOFormat<T>? {
val key = item.string ?: item.node["name"]?.string ?: error("Format name not defined")
return ioFormats[key]?.let {
@Suppress("UNCHECKED_CAST")
if (it.type != type) error("Format type ${it.type} is not the same as requested type $type")
else it.invoke(item.node["meta"].node ?: EmptyMeta, context) as IOFormat<T>
}
}
companion object : PluginFactory<IOPlugin> {
val IO_FORMAT_NAME_KEY = "name".asName()
val IO_FORMAT_META_KEY = "meta".asName()
val defaultMetaFormats: List<MetaFormatFactory> = listOf(JsonMetaFormat, BinaryMetaFormat)
val defaultEnvelopeFormats = listOf(TaggedEnvelopeFormat, TaglessEnvelopeFormat)

5
dataforge-io/src/commonMain/kotlin/hep/dataforge/io/JsonMetaFormat.kt
View File

@ -12,7 +12,6 @@ import hep.dataforge.meta.toMetaItem
import kotlinx.io.Input
import kotlinx.io.Output
import kotlinx.io.readByteArray
import kotlinx.io.text.readUtf8String
import kotlinx.io.text.writeUtf8String
import kotlinx.serialization.UnstableDefault
import kotlinx.serialization.json.Json
@ -26,6 +25,10 @@ class JsonMetaFormat(private val json: Json = DEFAULT_JSON) : MetaFormat {
writeUtf8String(json.stringify(JsonObjectSerializer, jsonObject))
}
override fun toMeta(): Meta = Meta{
IOPlugin.IO_FORMAT_NAME_KEY put name.toString()
}
override fun Input.readMeta(descriptor: NodeDescriptor?): Meta {
val str = readByteArray().decodeToString()
val jsonElement = json.parseJson(str)

31
dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt
View File

@ -2,6 +2,7 @@ package hep.dataforge.io
import hep.dataforge.context.Context
import hep.dataforge.meta.Meta
import hep.dataforge.meta.enum
import hep.dataforge.meta.get
import hep.dataforge.meta.string
import hep.dataforge.names.Name
@ -9,7 +10,6 @@ import hep.dataforge.names.plus
import hep.dataforge.names.toName
import kotlinx.io.*
@ExperimentalIoApi
class TaggedEnvelopeFormat(
val io: IOPlugin,
val version: VERSION = VERSION.DF02
@ -58,7 +58,7 @@ class TaggedEnvelopeFormat(
override fun Input.readObject(): Envelope {
val tag = readTag(version)
val metaFormat = io.metaFormat(tag.metaFormatKey)
val metaFormat = io.resolveMetaFormat(tag.metaFormatKey)
?: error("Meta format with key ${tag.metaFormatKey} not found")
val meta: Meta = limit(tag.metaSize.toInt()).run {
@ -67,7 +67,7 @@ class TaggedEnvelopeFormat(
}
}
val data = ByteArray(tag.dataSize.toInt()).also { readByteArray(it) }.asBinary()
val data = readBinary(tag.dataSize.toInt())
return SimpleEnvelope(meta, data)
}
@ -75,7 +75,7 @@ class TaggedEnvelopeFormat(
override fun Input.readPartial(): PartialEnvelope {
val tag = readTag(version)
val metaFormat = io.metaFormat(tag.metaFormatKey)
val metaFormat = io.resolveMetaFormat(tag.metaFormatKey)
?: error("Meta format with key ${tag.metaFormatKey} not found")
val meta: Meta = limit(tag.metaSize.toInt()).run {
@ -98,6 +98,13 @@ class TaggedEnvelopeFormat(
DF03(24u)
}
override fun toMeta(): Meta = Meta {
IOPlugin.IO_FORMAT_NAME_KEY put name.toString()
IOPlugin.IO_FORMAT_META_KEY put {
"version" put version
}
}
companion object : EnvelopeFormatFactory {
private const val START_SEQUENCE = "#~"
private const val END_SEQUENCE = "~#\r\n"
@ -111,7 +118,9 @@ class TaggedEnvelopeFormat(
//Check if appropriate factory exists
io.metaFormatFactories.find { it.name == metaFormatName } ?: error("Meta format could not be resolved")
return TaggedEnvelopeFormat(io)
val version: VERSION = meta["version"].enum<VERSION>() ?: VERSION.DF02
return TaggedEnvelopeFormat(io, version)
}
private fun Input.readTag(version: VERSION): Tag {
@ -132,11 +141,13 @@ class TaggedEnvelopeFormat(
override fun peekFormat(io: IOPlugin, input: Input): EnvelopeFormat? {
return try {
val header = input.readRawString(6)
when (header.substring(2..5)) {
VERSION.DF02.name -> TaggedEnvelopeFormat(io, VERSION.DF02)
VERSION.DF03.name -> TaggedEnvelopeFormat(io, VERSION.DF03)
else -> null
input.preview {
val header = readRawString(6)
return@preview when (header.substring(2..5)) {
VERSION.DF02.name -> TaggedEnvelopeFormat(io, VERSION.DF02)
VERSION.DF03.name -> TaggedEnvelopeFormat(io, VERSION.DF03)
else -> null
}
}
} catch (ex: Exception) {
null

41
dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaglessEnvelopeFormat.kt
View File

@ -1,16 +1,19 @@
package hep.dataforge.io
import hep.dataforge.context.Context
import hep.dataforge.meta.*
import hep.dataforge.meta.Meta
import hep.dataforge.meta.get
import hep.dataforge.meta.isEmpty
import hep.dataforge.meta.string
import hep.dataforge.names.asName
import kotlinx.io.*
import kotlinx.io.text.readUtf8Line
import kotlinx.io.text.writeUtf8String
import kotlin.collections.set
@ExperimentalIoApi
class TaglessEnvelopeFormat(
val io: IOPlugin,
meta: Meta = EmptyMeta
val meta: Meta = Meta.EMPTY
) : EnvelopeFormat {
private val metaStart = meta[META_START_PROPERTY].string ?: DEFAULT_META_START
@ -69,10 +72,10 @@ class TaglessEnvelopeFormat(
line = readUtf8Line()
}
var meta: Meta = EmptyMeta
var meta: Meta = Meta.EMPTY
if (line.startsWith(metaStart)) {
val metaFormat = properties[META_TYPE_PROPERTY]?.let { io.metaFormat(it) } ?: JsonMetaFormat
val metaFormat = properties[META_TYPE_PROPERTY]?.let { io.resolveMetaFormat(it) } ?: JsonMetaFormat
val metaSize = properties[META_LENGTH_PROPERTY]?.toInt()
meta = if (metaSize != null) {
limit(metaSize).run {
@ -95,9 +98,10 @@ class TaglessEnvelopeFormat(
} while (!line.startsWith(dataStart))
val data: Binary? = if (properties.containsKey(DATA_LENGTH_PROPERTY)) {
val bytes = ByteArray(properties[DATA_LENGTH_PROPERTY]!!.toInt())
readByteArray(bytes)
bytes.asBinary()
readBinary(properties[DATA_LENGTH_PROPERTY]!!.toInt())
// val bytes = ByteArray(properties[DATA_LENGTH_PROPERTY]!!.toInt())
// readByteArray(bytes)
// bytes.asBinary()
} else {
Binary {
copyTo(this)
@ -132,10 +136,10 @@ class TaglessEnvelopeFormat(
}
}
var meta: Meta = EmptyMeta
var meta: Meta = Meta.EMPTY
if (line.startsWith(metaStart)) {
val metaFormat = properties[META_TYPE_PROPERTY]?.let { io.metaFormat(it) } ?: JsonMetaFormat
val metaFormat = properties[META_TYPE_PROPERTY]?.let { io.resolveMetaFormat(it) } ?: JsonMetaFormat
val metaSize = properties[META_LENGTH_PROPERTY]?.toInt()
meta = if (metaSize != null) {
offset += metaSize.toUInt()
@ -157,6 +161,11 @@ class TaglessEnvelopeFormat(
return PartialEnvelope(meta, offset, dataSize)
}
override fun toMeta(): Meta = Meta {
IOPlugin.IO_FORMAT_NAME_KEY put name.toString()
IOPlugin.IO_FORMAT_META_KEY put meta
}
companion object : EnvelopeFormatFactory {
private val propertyPattern = "#\\?\\s*(?<key>[\\w.]*)\\s*:\\s*(?<value>[^;]*);?".toRegex()
@ -195,11 +204,13 @@ class TaglessEnvelopeFormat(
override fun peekFormat(io: IOPlugin, input: Input): EnvelopeFormat? {
return try {
val string = input.readRawString(TAGLESS_ENVELOPE_HEADER.length)
return if (string == TAGLESS_ENVELOPE_HEADER) {
TaglessEnvelopeFormat(io)
} else {
null
input.preview {
val string = readRawString(TAGLESS_ENVELOPE_HEADER.length)
return@preview if (string == TAGLESS_ENVELOPE_HEADER) {
TaglessEnvelopeFormat(io)
} else {
null
}
}
} catch (ex: Exception) {
null

24
dataforge-io/src/commonMain/kotlin/hep/dataforge/io/ioMisc.kt
View File

@ -1,6 +1,7 @@
package hep.dataforge.io
import kotlinx.io.*
import kotlin.math.min
fun Output.writeRawString(str: String) {
str.forEach { writeByte(it.toByte()) }
@ -18,5 +19,24 @@ inline fun buildByteArray(expectedSize: Int = 16, block: Output.() -> Unit): Byt
inline fun Binary(expectedSize: Int = 16, block: Output.() -> Unit): Binary =
buildByteArray(expectedSize, block).asBinary()
@Deprecated("To be replaced by Binary.EMPTY",level = DeprecationLevel.WARNING)
val EmptyBinary = ByteArrayBinary(ByteArray(0))
@Deprecated("To be replaced by Binary.EMPTY", level = DeprecationLevel.WARNING)
val EmptyBinary = ByteArrayBinary(ByteArray(0))
/**
* View section of a [Binary] as an independent binary
*/
class BinaryView(private val source: Binary, private val start: Int, override val size: Int) : Binary {
init {
require(start > 0)
require(start + size <= source.size) { "View boundary is outside source binary size" }
}
override fun <R> read(offset: Int, atMost: Int, block: Input.() -> R): R {
return source.read(start + offset, min(size, atMost), block)
}
}
fun Binary.view(start: Int, size: Int) = BinaryView(this, start, size)
operator fun Binary.get(range: IntRange) = view(range.first, range.last - range.first)

20
dataforge-io/src/commonTest/kotlin/hep/dataforge/io/BinaryTest.kt Normal file
View File

@ -0,0 +1,20 @@
package hep.dataforge.io
import kotlinx.io.asBinary
import kotlinx.io.readByte
import kotlinx.io.readInt
import kotlin.test.Test
import kotlin.test.assertEquals
class BinaryTest {
@Test
fun testBinaryAccess(){
val binary = ByteArray(128){it.toByte()}.asBinary()
binary[3..12].read {
readInt()
val res = readByte()
assertEquals(7, res)
}
}
}

18
dataforge-io/src/commonTest/kotlin/hep/dataforge/io/MultipartTest.kt
View File

@ -1,17 +1,19 @@
package hep.dataforge.io
import hep.dataforge.context.Global
import hep.dataforge.meta.DFExperimental
import hep.dataforge.meta.get
import hep.dataforge.meta.int
import kotlinx.io.text.writeUtf8String
import kotlin.test.Test
import kotlin.test.assertEquals
import kotlin.test.assertTrue
@DFExperimental
class MultipartTest {
val envelopes = (0..5).map {
val io: IOPlugin = Global.io
val envelopes = (0 until 5).map {
Envelope {
meta {
"value" put it
@ -26,19 +28,21 @@ class MultipartTest {
}
val partsEnvelope = Envelope {
multipart(envelopes, TaggedEnvelopeFormat)
envelopes(envelopes, TaglessEnvelopeFormat)
}
@Test
fun testParts() {
TaggedEnvelopeFormat.run {
TaglessEnvelopeFormat.run {
val singleEnvelopeData = toBinary(envelopes[0])
val singleEnvelopeSize = singleEnvelopeData.size
val bytes = toBinary(partsEnvelope)
assertTrue(5*singleEnvelopeSize < bytes.size)
assertTrue(envelopes.size * singleEnvelopeSize < bytes.size)
val reconstructed = bytes.readWith(this)
val parts = reconstructed.parts()?.toList() ?: emptyList()
assertEquals(2, parts[2].meta["value"].int)
println(reconstructed.meta)
val parts = reconstructed.parts()
val envelope = parts[2].envelope(io)
assertEquals(2, envelope.meta["value"].int)
println(reconstructed.data!!.size)
}
}

7
dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/fileIO.kt
View File

@ -1,7 +1,6 @@
package hep.dataforge.io
import hep.dataforge.meta.DFExperimental
import hep.dataforge.meta.EmptyMeta
import hep.dataforge.meta.Meta
import hep.dataforge.meta.descriptors.NodeDescriptor
import hep.dataforge.meta.isEmpty
@ -60,7 +59,7 @@ fun Path.readEnvelope(format: EnvelopeFormat): Envelope {
@Suppress("UNCHECKED_CAST")
@DFExperimental
inline fun <reified T : Any> IOPlugin.resolveIOFormat(): IOFormat<T>? {
return ioFormats.values.find { it.type.isSuperclassOf(T::class) } as IOFormat<T>?
return ioFormatFactories.find { it.type.isSuperclassOf(T::class) } as IOFormat<T>?
}
/**
@ -78,7 +77,7 @@ fun IOPlugin.readMetaFile(path: Path, formatOverride: MetaFormat? = null, descri
}
val extension = actualPath.fileName.toString().substringAfterLast('.')
val metaFormat = formatOverride ?: metaFormat(extension) ?: error("Can't resolve meta format $extension")
val metaFormat = formatOverride ?: resolveMetaFormat(extension) ?: error("Can't resolve meta format $extension")
return metaFormat.run {
actualPath.read {
readMeta(descriptor)
@ -157,7 +156,7 @@ fun IOPlugin.readEnvelopeFile(
.singleOrNull { it.fileName.toString().startsWith(IOPlugin.META_FILE_NAME) }
val meta = if (metaFile == null) {
EmptyMeta
Meta.EMPTY
} else {
readMetaFile(metaFile)
}

2
dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/functionsJVM.kt
View File

@ -9,7 +9,7 @@ import kotlin.reflect.full.isSuperclassOf
fun IOPlugin.resolveIOFormatName(type: KClass<*>): Name {
return ioFormats.entries.find { it.value.type.isSuperclassOf(type) }?.key
return ioFormatFactories.find { it.type.isSuperclassOf(type) }?.name
?: error("Can't resolve IOFormat for type $type")
}

2
dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/FileEnvelopeTest.kt
View File

@ -1,12 +1,14 @@
package hep.dataforge.io
import hep.dataforge.context.Global
import hep.dataforge.meta.DFExperimental
import kotlinx.io.writeDouble
import java.nio.file.Files
import kotlin.test.Test
import kotlin.test.assertTrue
@DFExperimental
class FileEnvelopeTest {
val envelope = Envelope {
meta {

2
dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/tcp/EnvelopeServerTest.kt
View File

@ -46,7 +46,7 @@ class EnvelopeServerTest {
@Test(timeout = 1000)
fun doEchoTest() {
val request = Envelope.invoke {
val request = Envelope {
type = "test.echo"
meta {
"test.value" put 22

7
dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Meta.kt
View File

@ -98,7 +98,7 @@ interface Meta : MetaRepr {
*/
const val VALUE_KEY = "@value"
val EMPTY: EmptyMeta = EmptyMeta
val EMPTY = EmptyMeta
}
}
@ -188,7 +188,7 @@ abstract class MetaBase : Meta {
override fun hashCode(): Int = items.hashCode()
override fun toString(): String = toJson().toString()
override fun toString(): String = PRETTY_JSON.stringify(MetaSerializer, this)
}
/**
@ -216,6 +216,7 @@ fun MetaItem<*>.seal(): MetaItem<SealedMeta> = when (this) {
is NodeItem -> NodeItem(node.seal())
}
@Deprecated("Use Meta.EMPTY instead", replaceWith = ReplaceWith("Meta.EMPTY"))
object EmptyMeta : MetaBase() {
override val items: Map<NameToken, MetaItem<*>> = emptyMap()
}
@ -251,4 +252,4 @@ val <M : Meta> MetaItem<M>?.node: M?
is NodeItem -> node
}
fun Meta.isEmpty() = this === EmptyMeta || this.items.isEmpty()
fun Meta.isEmpty() = this === Meta.EMPTY || this.items.isEmpty()

4
dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MutableMetaDelegate.kt
View File

@ -34,9 +34,7 @@ class ReadWriteDelegateWrapper<T, R>(
val reader: (T) -> R,
val writer: (R) -> T
) : ReadWriteProperty<Any?, R> {
override fun getValue(thisRef: Any?, property: KProperty<*>): R {
return reader(delegate.getValue(thisRef, property))
}
override fun getValue(thisRef: Any?, property: KProperty<*>): R = reader(delegate.getValue(thisRef, property))
override fun setValue(thisRef: Any?, property: KProperty<*>, value: R) {
delegate.setValue(thisRef, property, writer(value))

10
dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Specification.kt
View File

@ -16,7 +16,7 @@ interface Specification<T : Configurable> {
*/
fun wrap(config: Config = Config(), defaultProvider: (Name) -> MetaItem<*>? = { null }): T
operator fun invoke(action: T.() -> Unit) = empty().apply(action)
operator fun invoke(action: T.() -> Unit): T = empty().apply(action)
}
/**
@ -27,12 +27,16 @@ fun <T : Configurable> Specification<T>.update(config: Config, action: T.() -> U
/**
* Wrap a configuration using static meta as default
*/
fun <T : Configurable> Specification<T>.wrap(config: Config = Config(), default: Meta): T = wrap(config) { default[it] }
fun <T : Configurable> Specification<T>.wrap(config: Config = Config(), default: Meta = Meta.EMPTY): T =
wrap(config) { default[it] }
/**
* Wrap a configuration using static meta as default
*/
fun <T : Configurable> Specification<T>.wrap(default: Meta): T = wrap(Config()) { default[it] }
fun <T : Configurable> Specification<T>.wrap(source: Meta): T {
val default = source.seal()
return wrap(source.asConfig(), default)
}
/**

6
dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/serializationUtils.kt
View File

@ -3,6 +3,8 @@ package hep.dataforge.meta
import kotlinx.serialization.*
import kotlinx.serialization.builtins.DoubleArraySerializer
import kotlinx.serialization.builtins.serializer
import kotlinx.serialization.json.Json
import kotlinx.serialization.json.JsonConfiguration
fun SerialDescriptorBuilder.boolean(name: String, isOptional: Boolean = false, vararg annotations: Annotation) =
element(name, Boolean.serializer().descriptor, isOptional = isOptional, annotations = annotations.toList())
@ -62,4 +64,6 @@ inline fun Encoder.encodeStructure(
val encoder = beginStructure(desc, *typeParams)
encoder.block()
encoder.endStructure(desc)
}
}
val PRETTY_JSON = Json(JsonConfiguration(prettyPrint = true, useArrayPolymorphism = true))