kscience/dataforge-core
7
0
Fork 0
Code Issues Pull Requests 1 Packages Projects Releases 12 Wiki Activity

Merge remote-tracking branch 'remotes/origin/dev' into dev-zelenyy

Browse Source
This commit is contained in:
Mikhail Zelenyy 2019-11-05 03:00:59 +03:00
commit be79272ad2
15 changed files with 294 additions and 127 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
dataforge-data/src/commonMain/kotlin/hep/dataforge/data/MapAction.kt
View File

@ -1,16 +1,26 @@
package hep.dataforge.data package hep.dataforge.data
import hep.dataforge.meta.* import hep.dataforge.meta.Meta
import hep.dataforge.meta.MetaBuilder
import hep.dataforge.meta.builder
import hep.dataforge.meta.seal
import hep.dataforge.names.Name import hep.dataforge.names.Name
import kotlin.reflect.KClass import kotlin.reflect.KClass
class ActionEnv(val name: Name, val meta: Meta) /**
* Action environment includes data name, data meta and action configuration meta
*/
data class ActionEnv(
val name: Name,
val meta: Meta,
val actionMeta: Meta
)
/** /**
* Action environment * Action environment
*/ */
class MapActionBuilder<T, R>(var name: Name, var meta: MetaBuilder) { class MapActionBuilder<T, R>(var name: Name, var meta: MetaBuilder, val actionMeta: Meta) {
lateinit var result: suspend ActionEnv.(T) -> R lateinit var result: suspend ActionEnv.(T) -> R
/** /**
@ -33,16 +43,24 @@ class MapAction<T : Any, out R : Any>(
return DataNode.invoke(outputType) { return DataNode.invoke(outputType) {
node.dataSequence().forEach { (name, data) -> node.dataSequence().forEach { (name, data) ->
//merging data meta with action meta (data meta is primary) /*
val oldMeta = meta.builder().apply { update(data.meta) } * Creating a new environment for action using **old** name, old meta and task meta
// creating environment from old meta and name */
val env = ActionEnv(name, oldMeta) val env = ActionEnv(name, data.meta, meta)
//applying transformation from builder //applying transformation from builder
val builder = MapActionBuilder<T, R>(name, oldMeta).apply(block) val builder = MapActionBuilder<T, R>(
name,
data.meta.builder(), // using data meta
meta
).apply(block)
//getting new name //getting new name
val newName = builder.name val newName = builder.name
//getting new meta //getting new meta
val newMeta = builder.meta.seal() val newMeta = builder.meta.seal()
val newData = data.map(outputType, meta = newMeta) { builder.result(env, it) } val newData = data.map(outputType, meta = newMeta) { builder.result(env, it) }
//setting the data node //setting the data node
this[newName] = newData this[newName] = newData

15
dataforge-data/src/commonMain/kotlin/hep/dataforge/data/ReduceAction.kt
View File

@ -1,9 +1,7 @@
package hep.dataforge.data package hep.dataforge.data
import hep.dataforge.meta.Laminate
import hep.dataforge.meta.Meta import hep.dataforge.meta.Meta
import hep.dataforge.meta.MetaBuilder import hep.dataforge.meta.MetaBuilder
import hep.dataforge.meta.builder
import hep.dataforge.names.Name import hep.dataforge.names.Name
import hep.dataforge.names.toName import hep.dataforge.names.toName
import kotlin.reflect.KClass import kotlin.reflect.KClass
@ -84,15 +82,20 @@ class ReduceAction<T : Any, R : Any>(
return DataNode.invoke(outputType) { return DataNode.invoke(outputType) {
ReduceGroupBuilder<T, R>(meta).apply(action).buildGroups(node).forEach { group -> ReduceGroupBuilder<T, R>(meta).apply(action).buildGroups(node).forEach { group ->
val laminate = Laminate(group.meta, meta) //val laminate = Laminate(group.meta, meta)
val dataMap = group.node.dataSequence().associate { it } val dataMap = group.node.dataSequence().associate { it }
val groupName: String = group.name; val groupName: String = group.name
val env = ActionEnv(groupName.toName(), laminate.builder()) val groupMeta = group.meta
val res: DynamicData<R> = dataMap.reduce(outputType, meta = laminate) { group.result.invoke(env, it) } val env = ActionEnv(groupName.toName(), groupMeta, meta)
val res: DynamicData<R> = dataMap.reduce(
outputType,
meta = groupMeta
) { group.result.invoke(env, it) }
set(env.name, res) set(env.name, res)
} }

18
dataforge-io/dataforge-io-yaml/src/main/kotlin/hep/dataforge/io/yaml/FrontMatterEnvelopeFormat.kt
View File

@ -2,20 +2,18 @@ package hep.dataforge.io.yaml
import hep.dataforge.context.Context import hep.dataforge.context.Context
import hep.dataforge.io.* import hep.dataforge.io.*
import hep.dataforge.meta.* import hep.dataforge.meta.DFExperimental
import hep.dataforge.meta.EmptyMeta
import hep.dataforge.meta.Meta
import kotlinx.io.core.* import kotlinx.io.core.*
import kotlinx.serialization.toUtf8Bytes import kotlinx.serialization.toUtf8Bytes
@DFExperimental @DFExperimental
class FrontMatterEnvelopeFormat( class FrontMatterEnvelopeFormat(
val io: IOPlugin, val io: IOPlugin,
val metaType: String = YamlMetaFormat.name.toString(),
meta: Meta = EmptyMeta meta: Meta = EmptyMeta
) : EnvelopeFormat { ) : EnvelopeFormat {
val metaFormat = io.metaFormat(metaType, meta)
?: error("Meta format with type $metaType could not be resolved in $io")
override fun Input.readPartial(): PartialEnvelope { override fun Input.readPartial(): PartialEnvelope {
var line: String = "" var line: String = ""
var offset = 0u var offset = 0u
@ -60,11 +58,12 @@ class FrontMatterEnvelopeFormat(
return SimpleEnvelope(meta, data) return SimpleEnvelope(meta, data)
} }
override fun Output.writeObject(obj: Envelope) { override fun Output.writeEnvelope(envelope: Envelope, metaFormatFactory: MetaFormatFactory, formatMeta: Meta) {
val metaFormat = metaFormatFactory(formatMeta, io.context)
writeText("$SEPARATOR\r\n") writeText("$SEPARATOR\r\n")
metaFormat.run { writeObject(obj.meta) } metaFormat.run { writeObject(envelope.meta) }
writeText("$SEPARATOR\r\n") writeText("$SEPARATOR\r\n")
obj.data?.read { copyTo(this@writeObject) } envelope.data?.read { copyTo(this@writeEnvelope) }
} }
companion object : EnvelopeFormatFactory { companion object : EnvelopeFormatFactory {
@ -73,8 +72,7 @@ class FrontMatterEnvelopeFormat(
private val metaTypeRegex = "---(\\w*)\\s*".toRegex() private val metaTypeRegex = "---(\\w*)\\s*".toRegex()
override fun invoke(meta: Meta, context: Context): EnvelopeFormat { override fun invoke(meta: Meta, context: Context): EnvelopeFormat {
val metaFormatName: String = meta["name"].string ?: YamlMetaFormat.name.toString() return FrontMatterEnvelopeFormat(context.io, meta)
return FrontMatterEnvelopeFormat(context.io, metaFormatName, meta)
} }
override fun peekFormat(io: IOPlugin, input: Input): EnvelopeFormat? { override fun peekFormat(io: IOPlugin, input: Input): EnvelopeFormat? {

11
dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Binary.kt
View File

@ -1,9 +1,6 @@
package hep.dataforge.io package hep.dataforge.io
import kotlinx.io.core.ByteReadPacket import kotlinx.io.core.*
import kotlinx.io.core.Input
import kotlinx.io.core.buildPacket
import kotlinx.io.core.readBytes
import kotlin.math.min import kotlin.math.min
/** /**
@ -45,7 +42,7 @@ fun Binary.toBytes(): ByteArray = read {
@ExperimentalUnsignedTypes @ExperimentalUnsignedTypes
fun RandomAccessBinary.readPacket(from: UInt, size: UInt): ByteReadPacket = read(from, size) { fun RandomAccessBinary.readPacket(from: UInt, size: UInt): ByteReadPacket = read(from, size) {
ByteReadPacket(this.readBytes()) buildPacket { copyTo(this) }
} }
@ExperimentalUnsignedTypes @ExperimentalUnsignedTypes
@ -65,7 +62,9 @@ inline class ArrayBinary(val array: ByteArray) : RandomAccessBinary {
override fun <R> read(from: UInt, size: UInt, block: Input.() -> R): R { override fun <R> read(from: UInt, size: UInt, block: Input.() -> R): R {
val theSize = min(size, array.size.toUInt() - from) val theSize = min(size, array.size.toUInt() - from)
return ByteReadPacket(array, from.toInt(), theSize.toInt()).block() return buildPacket {
writeFully(array, from.toInt(), theSize.toInt())
}.block()
} }
} }

7
dataforge-io/src/commonMain/kotlin/hep/dataforge/io/EnvelopeFormat.kt
View File

@ -2,6 +2,7 @@ package hep.dataforge.io
import hep.dataforge.context.Context import hep.dataforge.context.Context
import hep.dataforge.io.EnvelopeFormatFactory.Companion.ENVELOPE_FORMAT_TYPE import hep.dataforge.io.EnvelopeFormatFactory.Companion.ENVELOPE_FORMAT_TYPE
import hep.dataforge.meta.EmptyMeta
import hep.dataforge.meta.Meta import hep.dataforge.meta.Meta
import hep.dataforge.names.Name import hep.dataforge.names.Name
import hep.dataforge.names.asName import hep.dataforge.names.asName
@ -18,11 +19,15 @@ data class PartialEnvelope(val meta: Meta, val dataOffset: UInt, val dataSize: U
interface EnvelopeFormat : IOFormat<Envelope> { interface EnvelopeFormat : IOFormat<Envelope> {
val defaultMetaFormat: MetaFormatFactory get() = JsonMetaFormat
fun Input.readPartial(): PartialEnvelope fun Input.readPartial(): PartialEnvelope
fun Output.writeEnvelope(envelope: Envelope, metaFormatFactory: MetaFormatFactory, formatMeta: Meta = EmptyMeta)
override fun Input.readObject(): Envelope override fun Input.readObject(): Envelope
override fun Output.writeObject(obj: Envelope) override fun Output.writeObject(obj: Envelope): Unit = writeEnvelope(obj, defaultMetaFormat)
} }
@Type(ENVELOPE_FORMAT_TYPE) @Type(ENVELOPE_FORMAT_TYPE)

2
dataforge-io/src/commonMain/kotlin/hep/dataforge/io/MetaFormat.kt
View File

@ -52,7 +52,7 @@ fun Meta.toBytes(format: MetaFormat = JsonMetaFormat.default): ByteReadPacket =
} }
fun MetaFormat.parse(str: String): Meta { fun MetaFormat.parse(str: String): Meta {
return ByteReadPacket(str.toByteArray()).readObject() return buildPacket { writeText(str) }.readObject()
} }
fun MetaFormatFactory.parse(str: String): Meta = invoke().parse(str) fun MetaFormatFactory.parse(str: String): Meta = invoke().parse(str)

69
dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt
View File

@ -7,33 +7,44 @@ import hep.dataforge.meta.string
import hep.dataforge.names.Name import hep.dataforge.names.Name
import hep.dataforge.names.plus import hep.dataforge.names.plus
import hep.dataforge.names.toName import hep.dataforge.names.toName
import kotlinx.io.charsets.Charsets
import kotlinx.io.core.* import kotlinx.io.core.*
@ExperimentalUnsignedTypes @ExperimentalUnsignedTypes
class TaggedEnvelopeFormat( class TaggedEnvelopeFormat(
val io: IOPlugin, val io: IOPlugin,
private val metaFormatKey: Short val version: VERSION = TaggedEnvelopeFormat.VERSION.DF02
) : EnvelopeFormat { ) : EnvelopeFormat {
private val metaFormat = io.metaFormat(metaFormatKey) // private val metaFormat = io.metaFormat(metaFormatKey)
?: error("Meta format with key $metaFormatKey could not be resolved in $io") // ?: error("Meta format with key $metaFormatKey could not be resolved in $io")
private fun Tag.toBytes(): ByteReadPacket = buildPacket(24) { private fun Tag.toBytes(): ByteReadPacket = buildPacket(24) {
writeText(START_SEQUENCE) writeText(START_SEQUENCE)
writeText(VERSION) writeText(version.name)
writeShort(metaFormatKey) writeShort(metaFormatKey)
writeUInt(metaSize) writeUInt(metaSize)
when (version) {
TaggedEnvelopeFormat.VERSION.DF02 -> {
writeUInt(dataSize.toUInt())
}
TaggedEnvelopeFormat.VERSION.DF03 -> {
writeULong(dataSize) writeULong(dataSize)
}
}
writeText(END_SEQUENCE) writeText(END_SEQUENCE)
} }
override fun Output.writeObject(obj: Envelope) { override fun Output.writeEnvelope(envelope: Envelope, metaFormatFactory: MetaFormatFactory, formatMeta: Meta) {
val metaBytes = metaFormat.writeBytes(obj.meta) val metaFormat = metaFormatFactory.invoke(formatMeta, io.context)
val tag = Tag(metaFormatKey, metaBytes.size.toUInt(), obj.data?.size ?: 0.toULong()) val metaBytes = metaFormat.writeBytes(envelope.meta)
val tag = Tag(metaFormatFactory.key, metaBytes.size.toUInt() + 2u, envelope.data?.size ?: 0.toULong())
writePacket(tag.toBytes()) writePacket(tag.toBytes())
writeFully(metaBytes) writeFully(metaBytes)
obj.data?.read { copyTo(this@writeObject) } writeText("\r\n")
envelope.data?.read { copyTo(this@writeEnvelope) }
flush()
} }
/** /**
@ -43,7 +54,7 @@ class TaggedEnvelopeFormat(
* @param formats a collection of meta formats to resolve * @param formats a collection of meta formats to resolve
*/ */
override fun Input.readObject(): Envelope { override fun Input.readObject(): Envelope {
val tag = readTag() val tag = readTag(version)
val metaFormat = io.metaFormat(tag.metaFormatKey) val metaFormat = io.metaFormat(tag.metaFormatKey)
?: error("Meta format with key ${tag.metaFormatKey} not found") ?: error("Meta format with key ${tag.metaFormatKey} not found")
@ -56,7 +67,7 @@ class TaggedEnvelopeFormat(
} }
override fun Input.readPartial(): PartialEnvelope { override fun Input.readPartial(): PartialEnvelope {
val tag = readTag() val tag = readTag(version)
val metaFormat = io.metaFormat(tag.metaFormatKey) val metaFormat = io.metaFormat(tag.metaFormatKey)
?: error("Meta format with key ${tag.metaFormatKey} not found") ?: error("Meta format with key ${tag.metaFormatKey} not found")
@ -64,7 +75,7 @@ class TaggedEnvelopeFormat(
val metaPacket = ByteReadPacket(readBytes(tag.metaSize.toInt())) val metaPacket = ByteReadPacket(readBytes(tag.metaSize.toInt()))
val meta = metaFormat.run { metaPacket.readObject() } val meta = metaFormat.run { metaPacket.readObject() }
return PartialEnvelope(meta, TAG_SIZE + tag.metaSize, tag.dataSize) return PartialEnvelope(meta, version.tagSize + tag.metaSize, tag.dataSize)
} }
private data class Tag( private data class Tag(
@ -73,13 +84,16 @@ class TaggedEnvelopeFormat(
val dataSize: ULong val dataSize: ULong
) )
enum class VERSION(val tagSize: UInt) {
DF02(20u),
DF03(24u)
}
companion object : EnvelopeFormatFactory { companion object : EnvelopeFormatFactory {
const val VERSION = "DF03"
private const val START_SEQUENCE = "#~" private const val START_SEQUENCE = "#~"
private const val END_SEQUENCE = "~#\r\n" private const val END_SEQUENCE = "~#\r\n"
private const val TAG_SIZE = 24u
override val name: Name = super.name + VERSION override val name: Name = super.name + "tagged"
override fun invoke(meta: Meta, context: Context): EnvelopeFormat { override fun invoke(meta: Meta, context: Context): EnvelopeFormat {
val io = context.io val io = context.io
@ -88,32 +102,39 @@ class TaggedEnvelopeFormat(
val metaFormatFactory = io.metaFormatFactories.find { it.name == metaFormatName } val metaFormatFactory = io.metaFormatFactories.find { it.name == metaFormatName }
?: error("Meta format could not be resolved") ?: error("Meta format could not be resolved")
return TaggedEnvelopeFormat(io, metaFormatFactory.key) return TaggedEnvelopeFormat(io)
} }
private fun Input.readTag(): Tag { private fun Input.readTag(version: VERSION): Tag {
val start = readTextExactBytes(2) val start = readTextExactBytes(2, charset = Charsets.ISO_8859_1)
if (start != START_SEQUENCE) error("The input is not an envelope") if (start != START_SEQUENCE) error("The input is not an envelope")
val version = readTextExactBytes(4) val versionString = readTextExactBytes(4, charset = Charsets.ISO_8859_1)
if (version != VERSION) error("Wrong version of DataForge: expected $VERSION but found $version") if (version.name != versionString) error("Wrong version of DataForge: expected $version but found $versionString")
val metaFormatKey = readShort() val metaFormatKey = readShort()
val metaLength = readUInt() val metaLength = readUInt()
val dataLength = readULong() val dataLength: ULong = when (version) {
val end = readTextExactBytes(4) VERSION.DF02 -> readUInt().toULong()
VERSION.DF03 -> readULong()
}
val end = readTextExactBytes(4, charset = Charsets.ISO_8859_1)
if (end != END_SEQUENCE) error("The input is not an envelope") if (end != END_SEQUENCE) error("The input is not an envelope")
return Tag(metaFormatKey, metaLength, dataLength) return Tag(metaFormatKey, metaLength, dataLength)
} }
override fun peekFormat(io: IOPlugin, input: Input): EnvelopeFormat? { override fun peekFormat(io: IOPlugin, input: Input): EnvelopeFormat? {
return try { return try {
val tag = input.readTag() val header = input.readTextExactBytes(6)
TaggedEnvelopeFormat(io, tag.metaFormatKey) when (header.substring(2..5)) {
VERSION.DF02.name -> TaggedEnvelopeFormat(io, VERSION.DF02)
VERSION.DF03.name -> TaggedEnvelopeFormat(io, VERSION.DF03)
else -> null
}
} catch (ex: Exception) { } catch (ex: Exception) {
null null
} }
} }
val default by lazy { invoke()} val default by lazy { invoke() }
} }
} }

28
dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaglessEnvelopeFormat.kt
View File

@ -8,33 +8,30 @@ import kotlinx.serialization.toUtf8Bytes
class TaglessEnvelopeFormat( class TaglessEnvelopeFormat(
val io: IOPlugin, val io: IOPlugin,
val metaType: String = JsonMetaFormat.name.toString(),
meta: Meta = EmptyMeta meta: Meta = EmptyMeta
) : EnvelopeFormat { ) : EnvelopeFormat {
private val metaStart = meta[META_START_PROPERTY].string ?: DEFAULT_META_START private val metaStart = meta[META_START_PROPERTY].string ?: DEFAULT_META_START
private val dataStart = meta[DATA_START_PROPERTY].string ?: DEFAULT_DATA_START private val dataStart = meta[DATA_START_PROPERTY].string ?: DEFAULT_DATA_START
val metaFormat = io.metaFormat(metaType, meta)
?: error("Meta format with type $metaType could not be resolved in $io")
private fun Output.writeProperty(key: String, value: Any) { private fun Output.writeProperty(key: String, value: Any) {
writeText("#? $key: $value;\r\n") writeText("#? $key: $value;\r\n")
} }
override fun Output.writeObject(obj: Envelope) { override fun Output.writeEnvelope(envelope: Envelope, metaFormatFactory: MetaFormatFactory, formatMeta: Meta) {
val metaFormat = metaFormatFactory(formatMeta, io.context)
//printing header //printing header
writeText(TAGLESS_ENVELOPE_HEADER + "\r\n") writeText(TAGLESS_ENVELOPE_HEADER + "\r\n")
//printing all properties //printing all properties
writeProperty(META_TYPE_PROPERTY, metaType) writeProperty(META_TYPE_PROPERTY, metaFormatFactory.type)
//TODO add optional metaFormat properties //TODO add optional metaFormat properties
writeProperty(DATA_LENGTH_PROPERTY, obj.data?.size ?: 0) writeProperty(DATA_LENGTH_PROPERTY, envelope.data?.size ?: 0)
//Printing meta //Printing meta
if (!obj.meta.isEmpty()) { if (!envelope.meta.isEmpty()) {
val metaBytes = metaFormat.writeBytes(obj.meta) val metaBytes = metaFormat.writeBytes(envelope.meta)
writeProperty(META_LENGTH_PROPERTY, metaBytes.size) writeProperty(META_LENGTH_PROPERTY, metaBytes.size)
writeText(metaStart + "\r\n") writeText(metaStart + "\r\n")
writeFully(metaBytes) writeFully(metaBytes)
@ -42,7 +39,7 @@ class TaglessEnvelopeFormat(
} }
//Printing data //Printing data
obj.data?.let { data -> envelope.data?.let { data ->
writeText(dataStart + "\r\n") writeText(dataStart + "\r\n")
writeFully(data.toBytes()) writeFully(data.toBytes())
} }
@ -72,7 +69,9 @@ class TaglessEnvelopeFormat(
val metaFormat = properties[META_TYPE_PROPERTY]?.let { io.metaFormat(it) } ?: JsonMetaFormat.default val metaFormat = properties[META_TYPE_PROPERTY]?.let { io.metaFormat(it) } ?: JsonMetaFormat.default
val metaSize = properties.get(META_LENGTH_PROPERTY)?.toInt() val metaSize = properties.get(META_LENGTH_PROPERTY)?.toInt()
meta = if (metaSize != null) { meta = if (metaSize != null) {
val metaPacket = ByteReadPacket(readBytes(metaSize)) val metaPacket = buildPacket {
writeFully(readBytes(metaSize))
}
metaFormat.run { metaPacket.readObject() } metaFormat.run { metaPacket.readObject() }
} else { } else {
metaFormat.run { metaFormat.run {
@ -126,7 +125,9 @@ class TaglessEnvelopeFormat(
val metaSize = properties.get(META_LENGTH_PROPERTY)?.toInt() val metaSize = properties.get(META_LENGTH_PROPERTY)?.toInt()
meta = if (metaSize != null) { meta = if (metaSize != null) {
val metaPacket = ByteReadPacket(readBytes(metaSize)) val metaPacket = buildPacket {
writeFully(readBytes(metaSize))
}
offset += metaSize.toUInt() offset += metaSize.toUInt()
metaFormat.run { metaPacket.readObject() } metaFormat.run { metaPacket.readObject() }
} else { } else {
@ -166,8 +167,7 @@ class TaglessEnvelopeFormat(
override val name = TAGLESS_ENVELOPE_TYPE.asName() override val name = TAGLESS_ENVELOPE_TYPE.asName()
override fun invoke(meta: Meta, context: Context): EnvelopeFormat { override fun invoke(meta: Meta, context: Context): EnvelopeFormat {
val metaFormatName: String = meta["name"].string ?: JsonMetaFormat.name.toString() return TaglessEnvelopeFormat(context.io, meta)
return TaglessEnvelopeFormat(context.io, metaFormatName, meta)
} }
val default by lazy { invoke() } val default by lazy { invoke() }

34
dataforge-io/src/commonMain/kotlin/hep/dataforge/io/serialization/MetaSerializer.kt
View File

@ -1,6 +1,7 @@
package hep.dataforge.io package hep.dataforge.io.serialization
import hep.dataforge.io.serialization.descriptor import hep.dataforge.io.toJson
import hep.dataforge.io.toMeta
import hep.dataforge.meta.* import hep.dataforge.meta.*
import hep.dataforge.names.NameToken import hep.dataforge.names.NameToken
import hep.dataforge.values.* import hep.dataforge.values.*
@ -14,12 +15,12 @@ import kotlinx.serialization.json.JsonOutput
@Serializer(Value::class) @Serializer(Value::class)
object ValueSerializer : KSerializer<Value> { object ValueSerializer : KSerializer<Value> {
private val valueTypeSerializer = EnumSerializer(ValueType::class) private val valueTypeSerializer = EnumSerializer(ValueType::class)
private val listSerializer by lazy { ArrayListSerializer(ValueSerializer)} private val listSerializer by lazy { ArrayListSerializer(ValueSerializer) }
override val descriptor: SerialDescriptor = descriptor("Value") { override val descriptor: SerialDescriptor = descriptor("hep.dataforge.values.Value") {
element("isList") boolean("isList")
element("valueType") enum<ValueType>("valueType")
element("value") element("value", null)
} }
private fun Decoder.decodeValue(): Value { private fun Decoder.decodeValue(): Value {
@ -68,10 +69,11 @@ object ValueSerializer : KSerializer<Value> {
@Serializer(MetaItem::class) @Serializer(MetaItem::class)
object MetaItemSerializer : KSerializer<MetaItem<*>> { object MetaItemSerializer : KSerializer<MetaItem<*>> {
override val descriptor: SerialDescriptor = descriptor("MetaItem") { override val descriptor: SerialDescriptor = descriptor("MetaItem") {
element("isNode") boolean("isNode")
element("value") element("value", null)
} }
override fun deserialize(decoder: Decoder): MetaItem<*> { override fun deserialize(decoder: Decoder): MetaItem<*> {
val isNode = decoder.decodeBoolean() val isNode = decoder.decodeBoolean()
return if (isNode) { return if (isNode) {
@ -92,17 +94,21 @@ object MetaItemSerializer : KSerializer<MetaItem<*>> {
private class DeserializedMeta(override val items: Map<NameToken, MetaItem<*>>) : MetaBase() private class DeserializedMeta(override val items: Map<NameToken, MetaItem<*>>) : MetaBase()
/** /**
* Serialized for meta * Serialized for meta
*/ */
@Serializer(Meta::class) @Serializer(Meta::class)
object MetaSerializer : KSerializer<Meta> { object MetaSerializer : KSerializer<Meta> {
private val mapSerializer = private val mapSerializer = HashMapSerializer(
HashMapSerializer(StringSerializer, MetaItemSerializer) StringSerializer,
MetaItemSerializer
)
override val descriptor: SerialDescriptor = override val descriptor: SerialDescriptor = NamedMapClassDescriptor(
NamedMapClassDescriptor("Meta", StringSerializer.descriptor, MetaItemSerializer.descriptor) "hep.dataforge.meta.Meta",
StringSerializer.descriptor,
MetaItemSerializer.descriptor
)
override fun deserialize(decoder: Decoder): Meta { override fun deserialize(decoder: Decoder): Meta {
return if (decoder is JsonInput) { return if (decoder is JsonInput) {

64
dataforge-io/src/commonMain/kotlin/hep/dataforge/io/serialization/serializationUtils.kt
View File

@ -4,23 +4,71 @@ import kotlinx.serialization.CompositeDecoder
import kotlinx.serialization.Decoder import kotlinx.serialization.Decoder
import kotlinx.serialization.KSerializer import kotlinx.serialization.KSerializer
import kotlinx.serialization.SerialDescriptor import kotlinx.serialization.SerialDescriptor
import kotlinx.serialization.internal.SerialClassDescImpl import kotlinx.serialization.internal.*
/**
* A convenience builder for serial descriptors
*/
inline class SerialDescriptorBuilder(private val impl: SerialClassDescImpl) { inline class SerialDescriptorBuilder(private val impl: SerialClassDescImpl) {
fun element(name: String, isOptional: Boolean = false) = impl.addElement(name, isOptional) fun element(
name: String,
descriptor: SerialDescriptor?,
isOptional: Boolean = false,
vararg annotations: Annotation
) {
impl.addElement(name, isOptional)
descriptor?.let { impl.pushDescriptor(descriptor) }
annotations.forEach {
impl.pushAnnotation(it)
}
}
fun annotation(a: Annotation) = impl.pushAnnotation(a) fun element(
name: String,
isOptional: Boolean = false,
vararg annotations: Annotation,
block: SerialDescriptorBuilder.() -> Unit
) {
impl.addElement(name, isOptional)
impl.pushDescriptor(SerialDescriptorBuilder(SerialClassDescImpl(name)).apply(block).build())
annotations.forEach {
impl.pushAnnotation(it)
}
}
fun boolean(name: String, isOptional: Boolean = false, vararg annotations: Annotation) =
element(name, BooleanDescriptor, isOptional, *annotations)
fun string(name: String, isOptional: Boolean = false, vararg annotations: Annotation) =
element(name, StringDescriptor, isOptional, *annotations)
fun int(name: String, isOptional: Boolean = false, vararg annotations: Annotation) =
element(name, IntDescriptor, isOptional, *annotations)
fun double(name: String, isOptional: Boolean = false, vararg annotations: Annotation) =
element(name, DoubleDescriptor, isOptional, *annotations)
fun float(name: String, isOptional: Boolean = false, vararg annotations: Annotation) =
element(name, FloatDescriptor, isOptional, *annotations)
fun long(name: String, isOptional: Boolean = false, vararg annotations: Annotation) =
element(name, LongDescriptor, isOptional, *annotations)
fun doubleArray(name: String, isOptional: Boolean = false, vararg annotations: Annotation) =
element(name, DoubleArraySerializer.descriptor, isOptional, *annotations)
inline fun <reified E : Enum<E>> enum(name: String, isOptional: Boolean = false, vararg annotations: Annotation) =
element(name, EnumSerializer(E::class).descriptor, isOptional, *annotations)
fun classAnnotation(a: Annotation) = impl.pushClassAnnotation(a) fun classAnnotation(a: Annotation) = impl.pushClassAnnotation(a)
fun descriptor(name: String, block: SerialDescriptorBuilder.() -> Unit) = impl.pushDescriptor(
SerialDescriptorBuilder(SerialClassDescImpl(name)).apply(block).build()
)
fun build(): SerialDescriptor = impl fun build(): SerialDescriptor = impl
} }
inline fun KSerializer<*>.descriptor(name: String, block: SerialDescriptorBuilder.() -> Unit): SerialDescriptor = inline fun <reified T : Any> KSerializer<T>.descriptor(
name: String,
block: SerialDescriptorBuilder.() -> Unit
): SerialDescriptor =
SerialDescriptorBuilder(SerialClassDescImpl(name)).apply(block).build() SerialDescriptorBuilder(SerialClassDescImpl(name)).apply(block).build()
fun Decoder.decodeStructure( fun Decoder.decodeStructure(

7
dataforge-io/src/commonTest/kotlin/hep/dataforge/io/MetaSerializerTest.kt
View File

@ -1,5 +1,7 @@
package hep.dataforge.io package hep.dataforge.io
import hep.dataforge.io.serialization.MetaItemSerializer
import hep.dataforge.io.serialization.MetaSerializer
import hep.dataforge.io.serialization.NameSerializer import hep.dataforge.io.serialization.NameSerializer
import hep.dataforge.meta.buildMeta import hep.dataforge.meta.buildMeta
import hep.dataforge.names.toName import hep.dataforge.names.toName
@ -51,4 +53,9 @@ class MetaSerializerTest {
val restored = Json.plain.parse(NameSerializer, string) val restored = Json.plain.parse(NameSerializer, string)
assertEquals(restored, name) assertEquals(restored, name)
} }
@Test
fun testMetaItemDescriptor(){
val descriptor = MetaItemSerializer.descriptor.getElementDescriptor(0)
}
} }

10
dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileBinary.kt
View File

@ -1,7 +1,7 @@
package hep.dataforge.io package hep.dataforge.io
import kotlinx.io.core.ByteReadPacket
import kotlinx.io.core.Input import kotlinx.io.core.Input
import kotlinx.io.core.buildPacket
import java.nio.channels.FileChannel import java.nio.channels.FileChannel
import java.nio.file.Files import java.nio.file.Files
import java.nio.file.Path import java.nio.file.Path
@ -13,11 +13,17 @@ class FileBinary(val path: Path, private val offset: UInt = 0u, size: ULong? = n
override val size: ULong = size ?: (Files.size(path).toULong() - offset).toULong() override val size: ULong = size ?: (Files.size(path).toULong() - offset).toULong()
init {
if( size != null && Files.size(path) < offset.toLong() + size.toLong()){
error("Can't read binary from file. File is to short.")
}
}
override fun <R> read(from: UInt, size: UInt, block: Input.() -> R): R { override fun <R> read(from: UInt, size: UInt, block: Input.() -> R): R {
FileChannel.open(path, StandardOpenOption.READ).use { FileChannel.open(path, StandardOpenOption.READ).use {
val theSize: UInt = min(size, Files.size(path).toUInt() - offset) val theSize: UInt = min(size, Files.size(path).toUInt() - offset)
val buffer = it.map(FileChannel.MapMode.READ_ONLY, (from + offset).toLong(), theSize.toLong()) val buffer = it.map(FileChannel.MapMode.READ_ONLY, (from + offset).toLong(), theSize.toLong())
return ByteReadPacket(buffer).block() return buildPacket { writeFully(buffer) }.block()
} }
} }
} }

2
dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileEnvelope.kt
View File

@ -15,7 +15,7 @@ class FileEnvelope internal constructor(val path: Path, val format: EnvelopeForm
init { init {
val input = Files.newByteChannel(path, StandardOpenOption.READ).asInput() val input = Files.newByteChannel(path, StandardOpenOption.READ).asInput()
partialEnvelope = format.run { input.readPartial() } partialEnvelope = format.run { input.use { it.readPartial()} }
} }
override val meta: Meta get() = partialEnvelope.meta override val meta: Meta get() = partialEnvelope.meta

56
dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/FileBinaryTest.kt Normal file
View File

@ -0,0 +1,56 @@
package hep.dataforge.io
import hep.dataforge.context.Global
import java.nio.file.Files
import kotlin.test.Test
import kotlin.test.assertEquals
class FileBinaryTest {
val envelope = Envelope {
meta {
"a" put "AAA"
"b" put 22.2
}
dataType = "hep.dataforge.test"
dataID = "myData" // добавил только что
data {
writeDouble(16.7)
}
}
@Test
fun testSize() {
val binary = envelope.data
assertEquals(binary?.size?.toInt(), binary?.toBytes()?.size)
}
@Test
fun testFileData(){
val dataFile = Files.createTempFile("dataforge_test_bin", ".bin")
dataFile.toFile().writeText("This is my binary")
val envelopeFromFile = Envelope {
meta {
"a" put "AAA"
"b" put 22.2
}
dataType = "hep.dataforge.satellite"
dataID = "cellDepositTest" // добавил только что
data = dataFile.asBinary()
}
val binary = envelopeFromFile.data!!
println(binary.toBytes().size)
assertEquals(binary.size.toInt(), binary.toBytes().size)
}
@Test
fun testFileDataSizeRewriting() {
println(System.getProperty("user.dir"))
val tmpPath = Files.createTempFile("dataforge_test", ".df")
Global.io.writeEnvelopeFile(tmpPath, envelope)
val binary = Global.io.readEnvelopeFile(tmpPath).data!!
assertEquals(binary.size.toInt(), binary.toBytes().size)
}
}

60
dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MetaBuilder.kt
View File

@ -15,55 +15,55 @@ class MetaBuilder : AbstractMutableMeta<MetaBuilder>() {
override fun wrapNode(meta: Meta): MetaBuilder = if (meta is MetaBuilder) meta else meta.builder() override fun wrapNode(meta: Meta): MetaBuilder = if (meta is MetaBuilder) meta else meta.builder()
override fun empty(): MetaBuilder = MetaBuilder() override fun empty(): MetaBuilder = MetaBuilder()
infix fun String.put(value: Value){ infix fun String.put(value: Value?) {
set(this,value) set(this, value)
} }
infix fun String.put(string: String){ infix fun String.put(string: String?) {
set(this,string.asValue()) set(this, string?.asValue())
} }
infix fun String.put(number: Number){ infix fun String.put(number: Number?) {
set(this,number.asValue()) set(this, number?.asValue())
} }
infix fun String.put(boolean: Boolean){ infix fun String.put(boolean: Boolean?) {
set(this, boolean.asValue()) set(this, boolean?.asValue())
} }
infix fun String.put(enum: Enum<*>){ infix fun String.put(enum: Enum<*>) {
set(this, EnumValue(enum)) set(this, EnumValue(enum))
} }
@JvmName("putValues") @JvmName("putValues")
infix fun String.put(iterable: Iterable<Value>){ infix fun String.put(iterable: Iterable<Value>) {
set(this, iterable.asValue()) set(this, iterable.asValue())
} }
@JvmName("putNumbers") @JvmName("putNumbers")
infix fun String.put(iterable: Iterable<Number>){ infix fun String.put(iterable: Iterable<Number>) {
set(this, iterable.map { it.asValue() }.asValue()) set(this, iterable.map { it.asValue() }.asValue())
} }
@JvmName("putStrings") @JvmName("putStrings")
infix fun String.put(iterable: Iterable<String>){ infix fun String.put(iterable: Iterable<String>) {
set(this, iterable.map { it.asValue() }.asValue()) set(this, iterable.map { it.asValue() }.asValue())
} }
infix fun String.put(array: DoubleArray){ infix fun String.put(array: DoubleArray) {
set(this, array.asValue()) set(this, array.asValue())
} }
infix fun String.putValue(any: Any?){ infix fun String.putValue(any: Any?) {
set(this, Value.of(any)) set(this, Value.of(any))
} }
infix fun String.put(meta: Meta) { infix fun String.put(meta: Meta?) {
this@MetaBuilder[this] = meta this@MetaBuilder[this] = meta
} }
infix fun String.put(repr: MetaRepr){ infix fun String.put(repr: MetaRepr?) {
set(this,repr.toMeta()) set(this, repr?.toMeta())
} }
@JvmName("putMetas") @JvmName("putMetas")
@ -75,37 +75,37 @@ class MetaBuilder : AbstractMutableMeta<MetaBuilder>() {
this@MetaBuilder[this] = MetaBuilder().apply(metaBuilder) this@MetaBuilder[this] = MetaBuilder().apply(metaBuilder)
} }
infix fun Name.put(value: Value){ infix fun Name.put(value: Value?) {
set(this,value) set(this, value)
} }
infix fun Name.put(string: String){ infix fun Name.put(string: String?) {
set(this,string.asValue()) set(this, string?.asValue())
} }
infix fun Name.put(number: Number){ infix fun Name.put(number: Number?) {
set(this,number.asValue()) set(this, number?.asValue())
} }
infix fun Name.put(boolean: Boolean){ infix fun Name.put(boolean: Boolean?) {
set(this, boolean.asValue()) set(this, boolean?.asValue())
} }
infix fun Name.put(enum: Enum<*>){ infix fun Name.put(enum: Enum<*>) {
set(this, EnumValue(enum)) set(this, EnumValue(enum))
} }
@JvmName("putValues") @JvmName("putValues")
infix fun Name.put(iterable: Iterable<Value>){ infix fun Name.put(iterable: Iterable<Value>) {
set(this, iterable.asValue()) set(this, iterable.asValue())
} }
infix fun Name.put(meta: Meta) { infix fun Name.put(meta: Meta?) {
this@MetaBuilder[this] = meta this@MetaBuilder[this] = meta
} }
infix fun Name.put(repr: MetaRepr){ infix fun Name.put(repr: MetaRepr?) {
set(this,repr.toMeta()) set(this, repr?.toMeta())
} }
@JvmName("putMetas") @JvmName("putMetas")