From 8c32eaeb6b66cc03501254b5d5c50686d08c6644 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Sun, 11 Aug 2019 18:35:03 +0300 Subject: [PATCH] Binary and file data update --- dataforge-data/build.gradle.kts | 2 - dataforge-io/build.gradle.kts | 4 +- .../kotlin/hep/dataforge/io/Binary.kt | 23 ++++++++- .../kotlin/hep/dataforge/io/Envelope.kt | 37 ++++++-------- .../kotlin/hep/dataforge/io/EnvelopeFormat.kt | 31 ++++++++++++ .../kotlin/hep/dataforge/io/FileEnvelope.kt | 3 +- .../hep/dataforge/workspace/dataUtils.kt | 14 ++++++ .../hep/dataforge/workspace/fileData.kt | 50 +++++++++++++------ 8 files changed, 121 insertions(+), 43 deletions(-) create mode 100644 dataforge-io/src/commonMain/kotlin/hep/dataforge/io/EnvelopeFormat.kt create mode 100644 dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/dataUtils.kt diff --git a/dataforge-data/build.gradle.kts b/dataforge-data/build.gradle.kts index 1faff685..793f551b 100644 --- a/dataforge-data/build.gradle.kts +++ b/dataforge-data/build.gradle.kts @@ -5,8 +5,6 @@ plugins { val coroutinesVersion: String = Scientifik.coroutinesVersion kotlin { - jvm() - js() sourceSets { val commonMain by getting{ dependencies { diff --git a/dataforge-io/build.gradle.kts b/dataforge-io/build.gradle.kts index 3ed7ab62..b4306efc 100644 --- a/dataforge-io/build.gradle.kts +++ b/dataforge-io/build.gradle.kts @@ -12,12 +12,12 @@ scientifik{ kotlin { sourceSets { - val commonMain by getting{ + commonMain{ dependencies { api(project(":dataforge-context")) } } - val jsMain by getting{ + jsMain{ dependencies{ api(npm("text-encoding")) } diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Binary.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Binary.kt index a0374968..d8fe9ad7 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Binary.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Binary.kt @@ -2,6 +2,7 @@ package hep.dataforge.io import kotlinx.io.core.ByteReadPacket import kotlinx.io.core.Input +import kotlinx.io.core.buildPacket import kotlinx.io.core.readBytes /** @@ -52,9 +53,29 @@ object EmptyBinary : RandomAccessBinary { } class ArrayBinary(val array: ByteArray) : RandomAccessBinary { - override val size: ULong = array.size.toULong() + override val size: ULong get() = array.size.toULong() override fun read(from: UInt, size: UInt, block: Input.() -> R): R { return ByteReadPacket(array, from.toInt(), size.toInt()).block() } +} + +/** + * Read given binary as object using given format + */ +fun Binary.readWith(format: IOFormat): T = format.run { + read { + readThis() + } +} + +/** + * Write this object to a binary + * TODO make a lazy binary that does not use intermediate array + */ +fun T.writeWith(format: IOFormat): Binary = format.run{ + val packet = buildPacket { + writeThis(this@writeWith) + } + return@run ArrayBinary(packet.readBytes()) } \ No newline at end of file diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Envelope.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Envelope.kt index bfbe40bf..c2abca21 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Envelope.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Envelope.kt @@ -1,14 +1,9 @@ package hep.dataforge.io -import hep.dataforge.context.Named -import hep.dataforge.io.EnvelopeFormat.Companion.ENVELOPE_FORMAT_TYPE -import hep.dataforge.io.IOPlugin.Companion.defaultMetaFormats +import hep.dataforge.meta.Laminate import hep.dataforge.meta.Meta import hep.dataforge.meta.get import hep.dataforge.meta.string -import hep.dataforge.provider.Type -import kotlinx.io.core.Input -import kotlinx.io.core.Output interface Envelope { val meta: Meta @@ -52,24 +47,20 @@ val Envelope.dataType: String? get() = meta[Envelope.ENVELOPE_DATA_TYPE_KEY].str val Envelope.description: String? get() = meta[Envelope.ENVELOPE_DESCRIPTION_KEY].string /** - * A partially read envelope with meta, but without data + * An envelope, which wraps existing envelope and adds one or several additional layers of meta */ -@ExperimentalUnsignedTypes -data class PartialEnvelope(val meta: Meta, val dataOffset: UInt, val dataSize: ULong?) +class ProxyEnvelope(val source: Envelope, vararg meta: Meta) : Envelope { + override val meta: Laminate = Laminate(*meta, source.meta) + override val data: Binary? get() = source.data +} -@Type(ENVELOPE_FORMAT_TYPE) -interface EnvelopeFormat : IOFormat, Named { - fun Input.readPartial(formats: Collection = defaultMetaFormats): PartialEnvelope - - fun Input.readEnvelope(formats: Collection = defaultMetaFormats): Envelope - - override fun Input.readThis(): Envelope = readEnvelope() - - fun Output.writeEnvelope(envelope: Envelope, format: MetaFormat = JsonMetaFormat) - - override fun Output.writeThis(obj: Envelope) = writeEnvelope(obj) - - companion object { - const val ENVELOPE_FORMAT_TYPE = "envelopeFormat" +/** + * Add few meta layers to existing envelope + */ +fun Envelope.withMetaLayers(vararg layers: Meta): Envelope { + return when { + layers.isEmpty() -> this + this is ProxyEnvelope -> ProxyEnvelope(source, *layers, *this.meta.layers.toTypedArray()) + else -> ProxyEnvelope(this, *layers) } } \ No newline at end of file diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/EnvelopeFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/EnvelopeFormat.kt new file mode 100644 index 00000000..24217e14 --- /dev/null +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/EnvelopeFormat.kt @@ -0,0 +1,31 @@ +package hep.dataforge.io + +import hep.dataforge.context.Named +import hep.dataforge.io.EnvelopeFormat.Companion.ENVELOPE_FORMAT_TYPE +import hep.dataforge.meta.Meta +import hep.dataforge.provider.Type +import kotlinx.io.core.Input +import kotlinx.io.core.Output + +/** + * A partially read envelope with meta, but without data + */ +@ExperimentalUnsignedTypes +data class PartialEnvelope(val meta: Meta, val dataOffset: UInt, val dataSize: ULong?) + +@Type(ENVELOPE_FORMAT_TYPE) +interface EnvelopeFormat : IOFormat, Named { + fun Input.readPartial(formats: Collection = IOPlugin.defaultMetaFormats): PartialEnvelope + + fun Input.readEnvelope(formats: Collection = IOPlugin.defaultMetaFormats): Envelope + + override fun Input.readThis(): Envelope = readEnvelope() + + fun Output.writeEnvelope(envelope: Envelope, format: MetaFormat = JsonMetaFormat) + + override fun Output.writeThis(obj: Envelope) = writeEnvelope(obj) + + companion object { + const val ENVELOPE_FORMAT_TYPE = "envelopeFormat" + } +} \ No newline at end of file diff --git a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileEnvelope.kt b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileEnvelope.kt index fdc4acc0..0c54012c 100644 --- a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileEnvelope.kt +++ b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileEnvelope.kt @@ -39,4 +39,5 @@ fun Path.writeEnvelope( with(format) { output.writeEnvelope(envelope, metaFormat) } -} \ No newline at end of file +} + diff --git a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/dataUtils.kt b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/dataUtils.kt new file mode 100644 index 00000000..f6d27774 --- /dev/null +++ b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/dataUtils.kt @@ -0,0 +1,14 @@ +package hep.dataforge.workspace + +import hep.dataforge.data.Data +import hep.dataforge.io.Envelope +import hep.dataforge.io.IOFormat +import hep.dataforge.io.readWith +import kotlin.reflect.KClass + +/** + * Convert an [Envelope] to a data via given format. The actual parsing is done lazily. + */ +fun Envelope.toData(type: KClass, format: IOFormat): Data = Data(type, meta) { + data?.readWith(format) ?: error("Can't convert envelope without data to Data") +} \ No newline at end of file diff --git a/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/fileData.kt b/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/fileData.kt index 1367998d..f20e18cf 100644 --- a/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/fileData.kt +++ b/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/fileData.kt @@ -1,11 +1,8 @@ package hep.dataforge.workspace -import hep.dataforge.context.Context import hep.dataforge.data.Data import hep.dataforge.descriptors.NodeDescriptor -import hep.dataforge.io.IOFormat -import hep.dataforge.io.JsonMetaFormat -import hep.dataforge.io.MetaFormat +import hep.dataforge.io.* import hep.dataforge.meta.EmptyMeta import hep.dataforge.meta.Meta import kotlinx.coroutines.Dispatchers @@ -44,11 +41,19 @@ suspend fun Meta.write(path: Path, format: MetaFormat, descriptor: NodeDescripto } } -suspend fun Context.readData( +/** + * Read data with supported envelope format and binary format. If envelope format is null, then read binary directly from file. + * @param type explicit type of data read + * @param format binary format + * @param envelopeFormat the format of envelope. If null, file is read directly + * @param metaFile the relative file for optional meta override + * @param metaFileFormat the meta format for override + */ +suspend fun Path.readData( type: KClass, - path: Path, format: IOFormat, - metaFile: Path = path.resolveSibling("${path.fileName}.meta"), + envelopeFormat: EnvelopeFormat? = null, + metaFile: Path = resolveSibling("$fileName.meta"), metaFileFormat: MetaFormat = JsonMetaFormat ): Data { return coroutineScope { @@ -57,14 +62,31 @@ suspend fun Context.readData( } else { null } - Data(type, externalMeta ?: EmptyMeta){ - withContext(Dispatchers.IO) { - format.run { - Files.newByteChannel(path, StandardOpenOption.READ) - .asInput() - .readThis() + if (envelopeFormat == null) { + Data(type, externalMeta ?: EmptyMeta) { + withContext(Dispatchers.IO) { + format.run { + Files.newByteChannel(this@readData, StandardOpenOption.READ) + .asInput() + .readThis() + } } } + } else { + withContext(Dispatchers.IO) { + readEnvelope(envelopeFormat).let { + if (externalMeta == null) { + it + } else { + it.withMetaLayers(externalMeta) + } + }.toData(type, format) + } } } -} \ No newline at end of file +} + +//suspend fun Path.writeData( +// data: Data, +// format: IOFormat, +// ) \ No newline at end of file