Binary and file data update
This commit is contained in:
parent
a8c9100539
commit
8c32eaeb6b
@ -5,8 +5,6 @@ plugins {
|
||||
val coroutinesVersion: String = Scientifik.coroutinesVersion
|
||||
|
||||
kotlin {
|
||||
jvm()
|
||||
js()
|
||||
sourceSets {
|
||||
val commonMain by getting{
|
||||
dependencies {
|
||||
|
@ -12,12 +12,12 @@ scientifik{
|
||||
|
||||
kotlin {
|
||||
sourceSets {
|
||||
val commonMain by getting{
|
||||
commonMain{
|
||||
dependencies {
|
||||
api(project(":dataforge-context"))
|
||||
}
|
||||
}
|
||||
val jsMain by getting{
|
||||
jsMain{
|
||||
dependencies{
|
||||
api(npm("text-encoding"))
|
||||
}
|
||||
|
@ -2,6 +2,7 @@ package hep.dataforge.io
|
||||
|
||||
import kotlinx.io.core.ByteReadPacket
|
||||
import kotlinx.io.core.Input
|
||||
import kotlinx.io.core.buildPacket
|
||||
import kotlinx.io.core.readBytes
|
||||
|
||||
/**
|
||||
@ -52,9 +53,29 @@ object EmptyBinary : RandomAccessBinary {
|
||||
}
|
||||
|
||||
class ArrayBinary(val array: ByteArray) : RandomAccessBinary {
|
||||
override val size: ULong = array.size.toULong()
|
||||
override val size: ULong get() = array.size.toULong()
|
||||
|
||||
override fun <R> read(from: UInt, size: UInt, block: Input.() -> R): R {
|
||||
return ByteReadPacket(array, from.toInt(), size.toInt()).block()
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Read given binary as object using given format
|
||||
*/
|
||||
fun <T : Any> Binary.readWith(format: IOFormat<T>): T = format.run {
|
||||
read {
|
||||
readThis()
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Write this object to a binary
|
||||
* TODO make a lazy binary that does not use intermediate array
|
||||
*/
|
||||
fun <T: Any> T.writeWith(format: IOFormat<T>): Binary = format.run{
|
||||
val packet = buildPacket {
|
||||
writeThis(this@writeWith)
|
||||
}
|
||||
return@run ArrayBinary(packet.readBytes())
|
||||
}
|
@ -1,14 +1,9 @@
|
||||
package hep.dataforge.io
|
||||
|
||||
import hep.dataforge.context.Named
|
||||
import hep.dataforge.io.EnvelopeFormat.Companion.ENVELOPE_FORMAT_TYPE
|
||||
import hep.dataforge.io.IOPlugin.Companion.defaultMetaFormats
|
||||
import hep.dataforge.meta.Laminate
|
||||
import hep.dataforge.meta.Meta
|
||||
import hep.dataforge.meta.get
|
||||
import hep.dataforge.meta.string
|
||||
import hep.dataforge.provider.Type
|
||||
import kotlinx.io.core.Input
|
||||
import kotlinx.io.core.Output
|
||||
|
||||
interface Envelope {
|
||||
val meta: Meta
|
||||
@ -52,24 +47,20 @@ val Envelope.dataType: String? get() = meta[Envelope.ENVELOPE_DATA_TYPE_KEY].str
|
||||
val Envelope.description: String? get() = meta[Envelope.ENVELOPE_DESCRIPTION_KEY].string
|
||||
|
||||
/**
|
||||
* A partially read envelope with meta, but without data
|
||||
* An envelope, which wraps existing envelope and adds one or several additional layers of meta
|
||||
*/
|
||||
@ExperimentalUnsignedTypes
|
||||
data class PartialEnvelope(val meta: Meta, val dataOffset: UInt, val dataSize: ULong?)
|
||||
class ProxyEnvelope(val source: Envelope, vararg meta: Meta) : Envelope {
|
||||
override val meta: Laminate = Laminate(*meta, source.meta)
|
||||
override val data: Binary? get() = source.data
|
||||
}
|
||||
|
||||
@Type(ENVELOPE_FORMAT_TYPE)
|
||||
interface EnvelopeFormat : IOFormat<Envelope>, Named {
|
||||
fun Input.readPartial(formats: Collection<MetaFormat> = defaultMetaFormats): PartialEnvelope
|
||||
|
||||
fun Input.readEnvelope(formats: Collection<MetaFormat> = defaultMetaFormats): Envelope
|
||||
|
||||
override fun Input.readThis(): Envelope = readEnvelope()
|
||||
|
||||
fun Output.writeEnvelope(envelope: Envelope, format: MetaFormat = JsonMetaFormat)
|
||||
|
||||
override fun Output.writeThis(obj: Envelope) = writeEnvelope(obj)
|
||||
|
||||
companion object {
|
||||
const val ENVELOPE_FORMAT_TYPE = "envelopeFormat"
|
||||
/**
|
||||
* Add few meta layers to existing envelope
|
||||
*/
|
||||
fun Envelope.withMetaLayers(vararg layers: Meta): Envelope {
|
||||
return when {
|
||||
layers.isEmpty() -> this
|
||||
this is ProxyEnvelope -> ProxyEnvelope(source, *layers, *this.meta.layers.toTypedArray())
|
||||
else -> ProxyEnvelope(this, *layers)
|
||||
}
|
||||
}
|
@ -0,0 +1,31 @@
|
||||
package hep.dataforge.io
|
||||
|
||||
import hep.dataforge.context.Named
|
||||
import hep.dataforge.io.EnvelopeFormat.Companion.ENVELOPE_FORMAT_TYPE
|
||||
import hep.dataforge.meta.Meta
|
||||
import hep.dataforge.provider.Type
|
||||
import kotlinx.io.core.Input
|
||||
import kotlinx.io.core.Output
|
||||
|
||||
/**
|
||||
* A partially read envelope with meta, but without data
|
||||
*/
|
||||
@ExperimentalUnsignedTypes
|
||||
data class PartialEnvelope(val meta: Meta, val dataOffset: UInt, val dataSize: ULong?)
|
||||
|
||||
@Type(ENVELOPE_FORMAT_TYPE)
|
||||
interface EnvelopeFormat : IOFormat<Envelope>, Named {
|
||||
fun Input.readPartial(formats: Collection<MetaFormat> = IOPlugin.defaultMetaFormats): PartialEnvelope
|
||||
|
||||
fun Input.readEnvelope(formats: Collection<MetaFormat> = IOPlugin.defaultMetaFormats): Envelope
|
||||
|
||||
override fun Input.readThis(): Envelope = readEnvelope()
|
||||
|
||||
fun Output.writeEnvelope(envelope: Envelope, format: MetaFormat = JsonMetaFormat)
|
||||
|
||||
override fun Output.writeThis(obj: Envelope) = writeEnvelope(obj)
|
||||
|
||||
companion object {
|
||||
const val ENVELOPE_FORMAT_TYPE = "envelopeFormat"
|
||||
}
|
||||
}
|
@ -40,3 +40,4 @@ fun Path.writeEnvelope(
|
||||
output.writeEnvelope(envelope, metaFormat)
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -0,0 +1,14 @@
|
||||
package hep.dataforge.workspace
|
||||
|
||||
import hep.dataforge.data.Data
|
||||
import hep.dataforge.io.Envelope
|
||||
import hep.dataforge.io.IOFormat
|
||||
import hep.dataforge.io.readWith
|
||||
import kotlin.reflect.KClass
|
||||
|
||||
/**
|
||||
* Convert an [Envelope] to a data via given format. The actual parsing is done lazily.
|
||||
*/
|
||||
fun <T : Any> Envelope.toData(type: KClass<out T>, format: IOFormat<T>): Data<T> = Data(type, meta) {
|
||||
data?.readWith(format) ?: error("Can't convert envelope without data to Data")
|
||||
}
|
@ -1,11 +1,8 @@
|
||||
package hep.dataforge.workspace
|
||||
|
||||
import hep.dataforge.context.Context
|
||||
import hep.dataforge.data.Data
|
||||
import hep.dataforge.descriptors.NodeDescriptor
|
||||
import hep.dataforge.io.IOFormat
|
||||
import hep.dataforge.io.JsonMetaFormat
|
||||
import hep.dataforge.io.MetaFormat
|
||||
import hep.dataforge.io.*
|
||||
import hep.dataforge.meta.EmptyMeta
|
||||
import hep.dataforge.meta.Meta
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
@ -44,11 +41,19 @@ suspend fun Meta.write(path: Path, format: MetaFormat, descriptor: NodeDescripto
|
||||
}
|
||||
}
|
||||
|
||||
suspend fun <T : Any> Context.readData(
|
||||
/**
|
||||
* Read data with supported envelope format and binary format. If envelope format is null, then read binary directly from file.
|
||||
* @param type explicit type of data read
|
||||
* @param format binary format
|
||||
* @param envelopeFormat the format of envelope. If null, file is read directly
|
||||
* @param metaFile the relative file for optional meta override
|
||||
* @param metaFileFormat the meta format for override
|
||||
*/
|
||||
suspend fun <T : Any> Path.readData(
|
||||
type: KClass<out T>,
|
||||
path: Path,
|
||||
format: IOFormat<T>,
|
||||
metaFile: Path = path.resolveSibling("${path.fileName}.meta"),
|
||||
envelopeFormat: EnvelopeFormat? = null,
|
||||
metaFile: Path = resolveSibling("$fileName.meta"),
|
||||
metaFileFormat: MetaFormat = JsonMetaFormat
|
||||
): Data<T> {
|
||||
return coroutineScope {
|
||||
@ -57,14 +62,31 @@ suspend fun <T : Any> Context.readData(
|
||||
} else {
|
||||
null
|
||||
}
|
||||
Data(type, externalMeta ?: EmptyMeta){
|
||||
withContext(Dispatchers.IO) {
|
||||
format.run {
|
||||
Files.newByteChannel(path, StandardOpenOption.READ)
|
||||
.asInput()
|
||||
.readThis()
|
||||
if (envelopeFormat == null) {
|
||||
Data(type, externalMeta ?: EmptyMeta) {
|
||||
withContext(Dispatchers.IO) {
|
||||
format.run {
|
||||
Files.newByteChannel(this@readData, StandardOpenOption.READ)
|
||||
.asInput()
|
||||
.readThis()
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
withContext(Dispatchers.IO) {
|
||||
readEnvelope(envelopeFormat).let {
|
||||
if (externalMeta == null) {
|
||||
it
|
||||
} else {
|
||||
it.withMetaLayers(externalMeta)
|
||||
}
|
||||
}.toData(type, format)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
//suspend fun <T : Any> Path.writeData(
|
||||
// data: Data<T>,
|
||||
// format: IOFormat<T>,
|
||||
// )
|
Loading…
Reference in New Issue
Block a user