kscience/dataforge-core
8
0
Fork 0
Code Issues Pull Requests 1 Packages Projects Releases 12 Wiki Activity

Envelope IO

Browse Source
This commit is contained in:
Alexander Nozik 2019-05-23 22:16:34 +03:00
parent 54c7b55bc4
commit 7c38cadddd
9 changed files with 191 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

60
dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Binary.kt Normal file
View File

@ -0,0 +1,60 @@
package hep.dataforge.io
import kotlinx.io.core.ByteReadPacket
import kotlinx.io.core.Input
import kotlinx.io.core.readBytes
/**
* A source of binary data
*/
interface Binary {
/**
* The size of binary in bytes
*/
val size: ULong
/**
* Read continuous [Input] from this binary stating from the beginning.
* The input is automatically closed on scope close.
* Some implementation may forbid this to be called twice. In this case second call will throw an exception.
*/
fun <R> read(block: Input.() -> R): R
}
/**
* A [Binary] with addition random access functionality. It by default allows multiple [read] operations.
*/
interface RandomAccessBinary : Binary {
/**
* Read at most [size] of bytes starting at [from] offset from the beginning of the binary.
* This method could be called multiple times simultaneously.
*/
fun <R> read(from: UInt, size: UInt = UInt.MAX_VALUE, block: Input.() -> R): R
override fun <R> read(block: Input.() -> R): R = read(0.toUInt(), UInt.MAX_VALUE, block)
}
fun Binary.readAll(): ByteReadPacket = read {
ByteReadPacket(this.readBytes())
}
fun RandomAccessBinary.readPacket(from: UInt, size: UInt): ByteReadPacket = read(from, size) {
ByteReadPacket(this.readBytes())
}
object EmptyBinary : RandomAccessBinary {
override val size: ULong = 0.toULong()
override fun <R> read(from: UInt, size: UInt, block: Input.() -> R): R {
error("The binary is empty")
}
}
class ArrayBinary(val array: ByteArray) : RandomAccessBinary {
override val size: ULong = array.size.toULong()
override fun <R> read(from: UInt, size: UInt, block: Input.() -> R): R {
return ByteReadPacket(array, from.toInt(), size.toInt()).block()
}
}

20
dataforge-io/src/commonMain/kotlin/hep/dataforge/io/BinaryMetaFormat.kt
View File

@ -8,12 +8,11 @@ import kotlinx.io.core.readText
import kotlinx.io.core.writeText
object BinaryMetaFormat : MetaFormat {
override fun write(obj: Meta, out: Output) {
out.writeMeta(obj)
}
override val name: String = "bin"
override val key: Short = 0x4249//BI
override fun read(input: Input): Meta {
return (input.readMetaItem() as MetaItem.NodeItem).node
override fun Input.readObject(): Meta {
return (readMetaItem() as MetaItem.NodeItem).node
}
private fun Output.writeChar(char: Char) = writeByte(char.toByte())
@ -70,7 +69,7 @@ object BinaryMetaFormat : MetaFormat {
}
}
private fun Output.writeMeta(meta: Meta) {
override fun Output.writeObject(meta: Meta) {
writeChar('M')
writeInt(meta.items.size)
meta.items.forEach { (key, item) ->
@ -80,7 +79,7 @@ object BinaryMetaFormat : MetaFormat {
writeValue(item.value)
}
is MetaItem.NodeItem -> {
writeMeta(item.node)
writeObject(item.node)
}
}
}
@ -122,11 +121,4 @@ object BinaryMetaFormat : MetaFormat {
else -> error("Unknown serialization key character: $keyChar")
}
}
}
class BinaryMetaFormatFactory : MetaFormatFactory {
override val name: String = "bin"
override val key: Short = 0x4249//BI
override fun build(): MetaFormat = BinaryMetaFormat
}

9
dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Envelope.kt
View File

@ -7,7 +7,7 @@ import kotlinx.io.core.Input
interface Envelope {
val meta: Meta
val data: Input?
val data: Binary?
companion object {
@ -23,11 +23,7 @@ interface Envelope {
}
}
class SimpleEnvelope(override val meta: Meta, val dataProvider: () -> Input?) : Envelope{
override val data: Input?
get() = dataProvider()
}
class SimpleEnvelope(override val meta: Meta, override val data: Binary?) : Envelope
/**
* The purpose of the envelope
@ -50,3 +46,4 @@ val Envelope.dataType: String? get() = meta[Envelope.ENVELOPE_DATA_TYPE_KEY].str
*/
val Envelope.description: String? get() = meta[Envelope.ENVELOPE_DESCRIPTION_KEY].string
typealias EnvelopeFormat = IOFormat<Envelope>

16
dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOFormat.kt
View File

@ -1,10 +1,14 @@
package hep.dataforge.io
import kotlinx.io.core.Input
import kotlinx.io.core.Output
import kotlinx.io.core.*
/**
* And interface for serialization facilities
*/
interface IOFormat<T : Any> {
fun write(obj: T, out: Output)
fun read(input: Input): T
}
fun Output.writeObject(obj: T)
fun Input.readObject(): T
}
fun <T : Any> IOFormat<T>.writePacket(obj: T): ByteReadPacket = buildPacket { writeObject(obj) }
fun <T : Any> IOFormat<T>.writeBytes(obj: T): ByteArray = buildPacket { writeObject(obj) }.readBytes()

18
dataforge-io/src/commonMain/kotlin/hep/dataforge/io/JsonMetaFormat.kt
View File

@ -14,13 +14,16 @@ import kotlinx.serialization.json.*
object JsonMetaFormat : MetaFormat {
override fun write(obj: Meta, out: Output) {
override val name: String = "json"
override val key: Short = 0x4a53//"JS"
override fun Output.writeObject(obj: Meta) {
val str = obj.toJson().toString()
out.writeText(str)
writeText(str)
}
override fun read(input: Input): Meta {
val str = input.readText()
override fun Input.readObject(): Meta {
val str = readText()
val json = Json.plain.parseJson(str)
if (json is JsonObject) {
@ -97,11 +100,4 @@ class JsonMeta(val json: JsonObject) : Meta {
json.forEach { (key, value) -> map[key] = value }
map.mapKeys { it.key.toName().first()!! }
}
}
class JsonMetaFormatFactory : MetaFormatFactory {
override val name: String = "json"
override val key: Short = 0x4a53//"JS"
override fun build() = JsonMetaFormat
}

19
dataforge-io/src/commonMain/kotlin/hep/dataforge/io/MetaFormat.kt
View File

@ -1,33 +1,26 @@
package hep.dataforge.io
import hep.dataforge.meta.Meta
import kotlinx.io.core.BytePacketBuilder
import kotlinx.io.core.ByteReadPacket
import kotlinx.io.core.buildPacket
import kotlinx.io.core.toByteArray
/**
* A format for meta serialization
*/
interface MetaFormat: IOFormat<Meta>
/**
* ServiceLoader compatible factory
*/
interface MetaFormatFactory {
interface MetaFormat : IOFormat<Meta> {
val name: String
val key: Short
fun build(): MetaFormat
}
fun Meta.asString(format: MetaFormat = JsonMetaFormat): String {
val builder = BytePacketBuilder()
format.write(this, builder)
return builder.build().readText()
return buildPacket {
format.run { writeObject(this@asString) }
}.readText()
}
fun MetaFormat.parse(str: String): Meta {
return read(ByteReadPacket(str.toByteArray()))
return ByteReadPacket(str.toByteArray()).readObject()
}

80
dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt Normal file
View File

@ -0,0 +1,80 @@
package hep.dataforge.io
import kotlinx.io.core.*
class TaggedEnvelopeFormat(
val metaFormats: Collection<MetaFormat>,
val outputMetaFormat: MetaFormat = metaFormats.first()
) : EnvelopeFormat {
override fun Output.writeObject(obj: Envelope) {
write(obj, this, outputMetaFormat)
}
/**
* Read an envelope from input into memory
*
* @param input an input to read from
* @param metaFormats a collection of meta formats to resolve
*/
override fun Input.readObject(): Envelope = read(this, metaFormats)
private data class Tag(
val metaFormatKey: Short,
val metaSize: UInt,
val dataSize: ULong
)
companion object {
private const val VERSION = "DF03"
private const val START_SEQUENCE = "#~"
private const val END_SEQUENCE = "~#\r\n"
private fun Tag.toBytes(): ByteReadPacket = buildPacket(24) {
writeText(START_SEQUENCE)
writeText(VERSION)
writeShort(metaFormatKey)
writeUInt(metaSize)
writeULong(dataSize)
writeText(END_SEQUENCE)
}
private fun Input.readTag(): Tag {
val start = readTextExactBytes(2)
if (start != START_SEQUENCE) error("The input is not an envelope")
val version = readTextExactBytes(4)
if (version != VERSION) error("Wrong version of DataForge: expected $VERSION but found $version")
val metaFormatKey = readShort()
val metaLength = readUInt()
val dataLength = readULong()
return Tag(metaFormatKey, metaLength, dataLength)
}
fun read(input: Input, metaFormats: Collection<MetaFormat>): Envelope {
val tag = input.readTag()
val metaFormat = metaFormats.find { it.key == tag.metaFormatKey }
?: error("Meta format with key ${tag.metaFormatKey} not found")
val metaPacket = ByteReadPacket(input.readBytes(tag.metaSize.toInt()))
val meta = metaFormat.run { metaPacket.readObject() }
val dataBytes = input.readBytes(tag.dataSize.toInt())
return SimpleEnvelope(meta, ArrayBinary(dataBytes))
}
fun write(obj: Envelope, out: Output, metaFormat: MetaFormat) {
val metaBytes = metaFormat.writeBytes(obj.meta)
val tag = Tag(metaFormat.key, metaBytes.size.toUInt(), obj.data?.size ?: 0.toULong())
out.writePacket(tag.toBytes())
out.writeFully(metaBytes)
obj.data?.read {
while (!endOfInput){
out.writeByte(readByte())
}
}
}
}
}

16
dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileBinary.kt Normal file
View File

@ -0,0 +1,16 @@
package hep.dataforge.io
import kotlinx.io.core.ByteReadPacket
import kotlinx.io.core.Input
import java.nio.channels.FileChannel
import java.nio.file.Path
import java.nio.file.StandardOpenOption
class FileBinary(val path: Path, private val offset: Int = 0) : RandomAccessBinary {
override fun <R> read(from: Long, size: Long, block: Input.() -> R): R {
FileChannel.open(path, StandardOpenOption.READ).use {
val buffer = it.map(FileChannel.MapMode.READ_ONLY, from + offset, size)
return ByteReadPacket(buffer).block()
}
}
}

5
dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MutableMeta.kt
View File

@ -95,12 +95,13 @@ abstract class MutableMetaNode<M : MutableMetaNode<M>> : AbstractMetaNode<M>(),
fun <M : MutableMeta<M>> MutableMeta<M>.remove(name: Name) = set(name, null)
fun <M : MutableMeta<M>> MutableMeta<M>.remove(name: String) = remove(name.toName())
fun <M : MutableMeta<M>> MutableMeta<M>.setValue(name: Name, value: Value) = set(name, MetaItem.ValueItem(value))
//fun <M : MutableMeta<M>> MutableMeta<M>.setItem(name: String, item: MetaItem<M>) = set(name.toName(), item)
fun <M : MutableMeta<M>> MutableMeta<M>.setValue(name: Name, value: Value) =
set(name, MetaItem.ValueItem(value))
fun <M : MutableMeta<M>> MutableMeta<M>.setValue(name: String, value: Value) =
set(name.toName(), MetaItem.ValueItem(value))
//fun <M : MutableMeta<M>> MutableMeta<M>.setItem(token: NameToken, item: MetaItem<M>?) = set(token.asName(), item)
//fun <M : MutableMeta<M>> MutableMeta<M>.setItem(name: String, item: MetaItem<M>) = set(name.toName(), item)
fun <M : MutableMetaNode<M>> MutableMetaNode<M>.setItem(name: Name, item: MetaItem<*>) {
when (item) {