Binary and file data update

This commit is contained in:
Alexander Nozik 2019-08-11 18:35:03 +03:00
parent a8c9100539
commit 8c32eaeb6b
8 changed files with 121 additions and 43 deletions

View File

@ -5,8 +5,6 @@ plugins {
val coroutinesVersion: String = Scientifik.coroutinesVersion val coroutinesVersion: String = Scientifik.coroutinesVersion
kotlin { kotlin {
jvm()
js()
sourceSets { sourceSets {
val commonMain by getting{ val commonMain by getting{
dependencies { dependencies {

View File

@ -12,12 +12,12 @@ scientifik{
kotlin { kotlin {
sourceSets { sourceSets {
val commonMain by getting{ commonMain{
dependencies { dependencies {
api(project(":dataforge-context")) api(project(":dataforge-context"))
} }
} }
val jsMain by getting{ jsMain{
dependencies{ dependencies{
api(npm("text-encoding")) api(npm("text-encoding"))
} }

View File

@ -2,6 +2,7 @@ package hep.dataforge.io
import kotlinx.io.core.ByteReadPacket import kotlinx.io.core.ByteReadPacket
import kotlinx.io.core.Input import kotlinx.io.core.Input
import kotlinx.io.core.buildPacket
import kotlinx.io.core.readBytes import kotlinx.io.core.readBytes
/** /**
@ -52,9 +53,29 @@ object EmptyBinary : RandomAccessBinary {
} }
class ArrayBinary(val array: ByteArray) : RandomAccessBinary { class ArrayBinary(val array: ByteArray) : RandomAccessBinary {
override val size: ULong = array.size.toULong() override val size: ULong get() = array.size.toULong()
override fun <R> read(from: UInt, size: UInt, block: Input.() -> R): R { override fun <R> read(from: UInt, size: UInt, block: Input.() -> R): R {
return ByteReadPacket(array, from.toInt(), size.toInt()).block() return ByteReadPacket(array, from.toInt(), size.toInt()).block()
} }
}
/**
* Read given binary as object using given format
*/
fun <T : Any> Binary.readWith(format: IOFormat<T>): T = format.run {
read {
readThis()
}
}
/**
* Write this object to a binary
* TODO make a lazy binary that does not use intermediate array
*/
fun <T: Any> T.writeWith(format: IOFormat<T>): Binary = format.run{
val packet = buildPacket {
writeThis(this@writeWith)
}
return@run ArrayBinary(packet.readBytes())
} }

View File

@ -1,14 +1,9 @@
package hep.dataforge.io package hep.dataforge.io
import hep.dataforge.context.Named import hep.dataforge.meta.Laminate
import hep.dataforge.io.EnvelopeFormat.Companion.ENVELOPE_FORMAT_TYPE
import hep.dataforge.io.IOPlugin.Companion.defaultMetaFormats
import hep.dataforge.meta.Meta import hep.dataforge.meta.Meta
import hep.dataforge.meta.get import hep.dataforge.meta.get
import hep.dataforge.meta.string import hep.dataforge.meta.string
import hep.dataforge.provider.Type
import kotlinx.io.core.Input
import kotlinx.io.core.Output
interface Envelope { interface Envelope {
val meta: Meta val meta: Meta
@ -52,24 +47,20 @@ val Envelope.dataType: String? get() = meta[Envelope.ENVELOPE_DATA_TYPE_KEY].str
val Envelope.description: String? get() = meta[Envelope.ENVELOPE_DESCRIPTION_KEY].string val Envelope.description: String? get() = meta[Envelope.ENVELOPE_DESCRIPTION_KEY].string
/** /**
* A partially read envelope with meta, but without data * An envelope, which wraps existing envelope and adds one or several additional layers of meta
*/ */
@ExperimentalUnsignedTypes class ProxyEnvelope(val source: Envelope, vararg meta: Meta) : Envelope {
data class PartialEnvelope(val meta: Meta, val dataOffset: UInt, val dataSize: ULong?) override val meta: Laminate = Laminate(*meta, source.meta)
override val data: Binary? get() = source.data
}
@Type(ENVELOPE_FORMAT_TYPE) /**
interface EnvelopeFormat : IOFormat<Envelope>, Named { * Add few meta layers to existing envelope
fun Input.readPartial(formats: Collection<MetaFormat> = defaultMetaFormats): PartialEnvelope */
fun Envelope.withMetaLayers(vararg layers: Meta): Envelope {
fun Input.readEnvelope(formats: Collection<MetaFormat> = defaultMetaFormats): Envelope return when {
layers.isEmpty() -> this
override fun Input.readThis(): Envelope = readEnvelope() this is ProxyEnvelope -> ProxyEnvelope(source, *layers, *this.meta.layers.toTypedArray())
else -> ProxyEnvelope(this, *layers)
fun Output.writeEnvelope(envelope: Envelope, format: MetaFormat = JsonMetaFormat)
override fun Output.writeThis(obj: Envelope) = writeEnvelope(obj)
companion object {
const val ENVELOPE_FORMAT_TYPE = "envelopeFormat"
} }
} }

View File

@ -0,0 +1,31 @@
package hep.dataforge.io
import hep.dataforge.context.Named
import hep.dataforge.io.EnvelopeFormat.Companion.ENVELOPE_FORMAT_TYPE
import hep.dataforge.meta.Meta
import hep.dataforge.provider.Type
import kotlinx.io.core.Input
import kotlinx.io.core.Output
/**
* A partially read envelope with meta, but without data
*/
@ExperimentalUnsignedTypes
data class PartialEnvelope(val meta: Meta, val dataOffset: UInt, val dataSize: ULong?)
@Type(ENVELOPE_FORMAT_TYPE)
interface EnvelopeFormat : IOFormat<Envelope>, Named {
fun Input.readPartial(formats: Collection<MetaFormat> = IOPlugin.defaultMetaFormats): PartialEnvelope
fun Input.readEnvelope(formats: Collection<MetaFormat> = IOPlugin.defaultMetaFormats): Envelope
override fun Input.readThis(): Envelope = readEnvelope()
fun Output.writeEnvelope(envelope: Envelope, format: MetaFormat = JsonMetaFormat)
override fun Output.writeThis(obj: Envelope) = writeEnvelope(obj)
companion object {
const val ENVELOPE_FORMAT_TYPE = "envelopeFormat"
}
}

View File

@ -39,4 +39,5 @@ fun Path.writeEnvelope(
with(format) { with(format) {
output.writeEnvelope(envelope, metaFormat) output.writeEnvelope(envelope, metaFormat)
} }
} }

View File

@ -0,0 +1,14 @@
package hep.dataforge.workspace
import hep.dataforge.data.Data
import hep.dataforge.io.Envelope
import hep.dataforge.io.IOFormat
import hep.dataforge.io.readWith
import kotlin.reflect.KClass
/**
* Convert an [Envelope] to a data via given format. The actual parsing is done lazily.
*/
fun <T : Any> Envelope.toData(type: KClass<out T>, format: IOFormat<T>): Data<T> = Data(type, meta) {
data?.readWith(format) ?: error("Can't convert envelope without data to Data")
}

View File

@ -1,11 +1,8 @@
package hep.dataforge.workspace package hep.dataforge.workspace
import hep.dataforge.context.Context
import hep.dataforge.data.Data import hep.dataforge.data.Data
import hep.dataforge.descriptors.NodeDescriptor import hep.dataforge.descriptors.NodeDescriptor
import hep.dataforge.io.IOFormat import hep.dataforge.io.*
import hep.dataforge.io.JsonMetaFormat
import hep.dataforge.io.MetaFormat
import hep.dataforge.meta.EmptyMeta import hep.dataforge.meta.EmptyMeta
import hep.dataforge.meta.Meta import hep.dataforge.meta.Meta
import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.Dispatchers
@ -44,11 +41,19 @@ suspend fun Meta.write(path: Path, format: MetaFormat, descriptor: NodeDescripto
} }
} }
suspend fun <T : Any> Context.readData( /**
* Read data with supported envelope format and binary format. If envelope format is null, then read binary directly from file.
* @param type explicit type of data read
* @param format binary format
* @param envelopeFormat the format of envelope. If null, file is read directly
* @param metaFile the relative file for optional meta override
* @param metaFileFormat the meta format for override
*/
suspend fun <T : Any> Path.readData(
type: KClass<out T>, type: KClass<out T>,
path: Path,
format: IOFormat<T>, format: IOFormat<T>,
metaFile: Path = path.resolveSibling("${path.fileName}.meta"), envelopeFormat: EnvelopeFormat? = null,
metaFile: Path = resolveSibling("$fileName.meta"),
metaFileFormat: MetaFormat = JsonMetaFormat metaFileFormat: MetaFormat = JsonMetaFormat
): Data<T> { ): Data<T> {
return coroutineScope { return coroutineScope {
@ -57,14 +62,31 @@ suspend fun <T : Any> Context.readData(
} else { } else {
null null
} }
Data(type, externalMeta ?: EmptyMeta){ if (envelopeFormat == null) {
withContext(Dispatchers.IO) { Data(type, externalMeta ?: EmptyMeta) {
format.run { withContext(Dispatchers.IO) {
Files.newByteChannel(path, StandardOpenOption.READ) format.run {
.asInput() Files.newByteChannel(this@readData, StandardOpenOption.READ)
.readThis() .asInput()
.readThis()
}
} }
} }
} else {
withContext(Dispatchers.IO) {
readEnvelope(envelopeFormat).let {
if (externalMeta == null) {
it
} else {
it.withMetaLayers(externalMeta)
}
}.toData(type, format)
}
} }
} }
} }
//suspend fun <T : Any> Path.writeData(
// data: Data<T>,
// format: IOFormat<T>,
// )