Binary and file data update
This commit is contained in:
parent
a8c9100539
commit
8c32eaeb6b
@ -5,8 +5,6 @@ plugins {
|
|||||||
val coroutinesVersion: String = Scientifik.coroutinesVersion
|
val coroutinesVersion: String = Scientifik.coroutinesVersion
|
||||||
|
|
||||||
kotlin {
|
kotlin {
|
||||||
jvm()
|
|
||||||
js()
|
|
||||||
sourceSets {
|
sourceSets {
|
||||||
val commonMain by getting{
|
val commonMain by getting{
|
||||||
dependencies {
|
dependencies {
|
||||||
|
@ -12,12 +12,12 @@ scientifik{
|
|||||||
|
|
||||||
kotlin {
|
kotlin {
|
||||||
sourceSets {
|
sourceSets {
|
||||||
val commonMain by getting{
|
commonMain{
|
||||||
dependencies {
|
dependencies {
|
||||||
api(project(":dataforge-context"))
|
api(project(":dataforge-context"))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
val jsMain by getting{
|
jsMain{
|
||||||
dependencies{
|
dependencies{
|
||||||
api(npm("text-encoding"))
|
api(npm("text-encoding"))
|
||||||
}
|
}
|
||||||
|
@ -2,6 +2,7 @@ package hep.dataforge.io
|
|||||||
|
|
||||||
import kotlinx.io.core.ByteReadPacket
|
import kotlinx.io.core.ByteReadPacket
|
||||||
import kotlinx.io.core.Input
|
import kotlinx.io.core.Input
|
||||||
|
import kotlinx.io.core.buildPacket
|
||||||
import kotlinx.io.core.readBytes
|
import kotlinx.io.core.readBytes
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@ -52,9 +53,29 @@ object EmptyBinary : RandomAccessBinary {
|
|||||||
}
|
}
|
||||||
|
|
||||||
class ArrayBinary(val array: ByteArray) : RandomAccessBinary {
|
class ArrayBinary(val array: ByteArray) : RandomAccessBinary {
|
||||||
override val size: ULong = array.size.toULong()
|
override val size: ULong get() = array.size.toULong()
|
||||||
|
|
||||||
override fun <R> read(from: UInt, size: UInt, block: Input.() -> R): R {
|
override fun <R> read(from: UInt, size: UInt, block: Input.() -> R): R {
|
||||||
return ByteReadPacket(array, from.toInt(), size.toInt()).block()
|
return ByteReadPacket(array, from.toInt(), size.toInt()).block()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Read given binary as object using given format
|
||||||
|
*/
|
||||||
|
fun <T : Any> Binary.readWith(format: IOFormat<T>): T = format.run {
|
||||||
|
read {
|
||||||
|
readThis()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Write this object to a binary
|
||||||
|
* TODO make a lazy binary that does not use intermediate array
|
||||||
|
*/
|
||||||
|
fun <T: Any> T.writeWith(format: IOFormat<T>): Binary = format.run{
|
||||||
|
val packet = buildPacket {
|
||||||
|
writeThis(this@writeWith)
|
||||||
|
}
|
||||||
|
return@run ArrayBinary(packet.readBytes())
|
||||||
|
}
|
@ -1,14 +1,9 @@
|
|||||||
package hep.dataforge.io
|
package hep.dataforge.io
|
||||||
|
|
||||||
import hep.dataforge.context.Named
|
import hep.dataforge.meta.Laminate
|
||||||
import hep.dataforge.io.EnvelopeFormat.Companion.ENVELOPE_FORMAT_TYPE
|
|
||||||
import hep.dataforge.io.IOPlugin.Companion.defaultMetaFormats
|
|
||||||
import hep.dataforge.meta.Meta
|
import hep.dataforge.meta.Meta
|
||||||
import hep.dataforge.meta.get
|
import hep.dataforge.meta.get
|
||||||
import hep.dataforge.meta.string
|
import hep.dataforge.meta.string
|
||||||
import hep.dataforge.provider.Type
|
|
||||||
import kotlinx.io.core.Input
|
|
||||||
import kotlinx.io.core.Output
|
|
||||||
|
|
||||||
interface Envelope {
|
interface Envelope {
|
||||||
val meta: Meta
|
val meta: Meta
|
||||||
@ -52,24 +47,20 @@ val Envelope.dataType: String? get() = meta[Envelope.ENVELOPE_DATA_TYPE_KEY].str
|
|||||||
val Envelope.description: String? get() = meta[Envelope.ENVELOPE_DESCRIPTION_KEY].string
|
val Envelope.description: String? get() = meta[Envelope.ENVELOPE_DESCRIPTION_KEY].string
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* A partially read envelope with meta, but without data
|
* An envelope, which wraps existing envelope and adds one or several additional layers of meta
|
||||||
*/
|
*/
|
||||||
@ExperimentalUnsignedTypes
|
class ProxyEnvelope(val source: Envelope, vararg meta: Meta) : Envelope {
|
||||||
data class PartialEnvelope(val meta: Meta, val dataOffset: UInt, val dataSize: ULong?)
|
override val meta: Laminate = Laminate(*meta, source.meta)
|
||||||
|
override val data: Binary? get() = source.data
|
||||||
|
}
|
||||||
|
|
||||||
@Type(ENVELOPE_FORMAT_TYPE)
|
/**
|
||||||
interface EnvelopeFormat : IOFormat<Envelope>, Named {
|
* Add few meta layers to existing envelope
|
||||||
fun Input.readPartial(formats: Collection<MetaFormat> = defaultMetaFormats): PartialEnvelope
|
*/
|
||||||
|
fun Envelope.withMetaLayers(vararg layers: Meta): Envelope {
|
||||||
fun Input.readEnvelope(formats: Collection<MetaFormat> = defaultMetaFormats): Envelope
|
return when {
|
||||||
|
layers.isEmpty() -> this
|
||||||
override fun Input.readThis(): Envelope = readEnvelope()
|
this is ProxyEnvelope -> ProxyEnvelope(source, *layers, *this.meta.layers.toTypedArray())
|
||||||
|
else -> ProxyEnvelope(this, *layers)
|
||||||
fun Output.writeEnvelope(envelope: Envelope, format: MetaFormat = JsonMetaFormat)
|
|
||||||
|
|
||||||
override fun Output.writeThis(obj: Envelope) = writeEnvelope(obj)
|
|
||||||
|
|
||||||
companion object {
|
|
||||||
const val ENVELOPE_FORMAT_TYPE = "envelopeFormat"
|
|
||||||
}
|
}
|
||||||
}
|
}
|
@ -0,0 +1,31 @@
|
|||||||
|
package hep.dataforge.io
|
||||||
|
|
||||||
|
import hep.dataforge.context.Named
|
||||||
|
import hep.dataforge.io.EnvelopeFormat.Companion.ENVELOPE_FORMAT_TYPE
|
||||||
|
import hep.dataforge.meta.Meta
|
||||||
|
import hep.dataforge.provider.Type
|
||||||
|
import kotlinx.io.core.Input
|
||||||
|
import kotlinx.io.core.Output
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A partially read envelope with meta, but without data
|
||||||
|
*/
|
||||||
|
@ExperimentalUnsignedTypes
|
||||||
|
data class PartialEnvelope(val meta: Meta, val dataOffset: UInt, val dataSize: ULong?)
|
||||||
|
|
||||||
|
@Type(ENVELOPE_FORMAT_TYPE)
|
||||||
|
interface EnvelopeFormat : IOFormat<Envelope>, Named {
|
||||||
|
fun Input.readPartial(formats: Collection<MetaFormat> = IOPlugin.defaultMetaFormats): PartialEnvelope
|
||||||
|
|
||||||
|
fun Input.readEnvelope(formats: Collection<MetaFormat> = IOPlugin.defaultMetaFormats): Envelope
|
||||||
|
|
||||||
|
override fun Input.readThis(): Envelope = readEnvelope()
|
||||||
|
|
||||||
|
fun Output.writeEnvelope(envelope: Envelope, format: MetaFormat = JsonMetaFormat)
|
||||||
|
|
||||||
|
override fun Output.writeThis(obj: Envelope) = writeEnvelope(obj)
|
||||||
|
|
||||||
|
companion object {
|
||||||
|
const val ENVELOPE_FORMAT_TYPE = "envelopeFormat"
|
||||||
|
}
|
||||||
|
}
|
@ -40,3 +40,4 @@ fun Path.writeEnvelope(
|
|||||||
output.writeEnvelope(envelope, metaFormat)
|
output.writeEnvelope(envelope, metaFormat)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -0,0 +1,14 @@
|
|||||||
|
package hep.dataforge.workspace
|
||||||
|
|
||||||
|
import hep.dataforge.data.Data
|
||||||
|
import hep.dataforge.io.Envelope
|
||||||
|
import hep.dataforge.io.IOFormat
|
||||||
|
import hep.dataforge.io.readWith
|
||||||
|
import kotlin.reflect.KClass
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Convert an [Envelope] to a data via given format. The actual parsing is done lazily.
|
||||||
|
*/
|
||||||
|
fun <T : Any> Envelope.toData(type: KClass<out T>, format: IOFormat<T>): Data<T> = Data(type, meta) {
|
||||||
|
data?.readWith(format) ?: error("Can't convert envelope without data to Data")
|
||||||
|
}
|
@ -1,11 +1,8 @@
|
|||||||
package hep.dataforge.workspace
|
package hep.dataforge.workspace
|
||||||
|
|
||||||
import hep.dataforge.context.Context
|
|
||||||
import hep.dataforge.data.Data
|
import hep.dataforge.data.Data
|
||||||
import hep.dataforge.descriptors.NodeDescriptor
|
import hep.dataforge.descriptors.NodeDescriptor
|
||||||
import hep.dataforge.io.IOFormat
|
import hep.dataforge.io.*
|
||||||
import hep.dataforge.io.JsonMetaFormat
|
|
||||||
import hep.dataforge.io.MetaFormat
|
|
||||||
import hep.dataforge.meta.EmptyMeta
|
import hep.dataforge.meta.EmptyMeta
|
||||||
import hep.dataforge.meta.Meta
|
import hep.dataforge.meta.Meta
|
||||||
import kotlinx.coroutines.Dispatchers
|
import kotlinx.coroutines.Dispatchers
|
||||||
@ -44,11 +41,19 @@ suspend fun Meta.write(path: Path, format: MetaFormat, descriptor: NodeDescripto
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
suspend fun <T : Any> Context.readData(
|
/**
|
||||||
|
* Read data with supported envelope format and binary format. If envelope format is null, then read binary directly from file.
|
||||||
|
* @param type explicit type of data read
|
||||||
|
* @param format binary format
|
||||||
|
* @param envelopeFormat the format of envelope. If null, file is read directly
|
||||||
|
* @param metaFile the relative file for optional meta override
|
||||||
|
* @param metaFileFormat the meta format for override
|
||||||
|
*/
|
||||||
|
suspend fun <T : Any> Path.readData(
|
||||||
type: KClass<out T>,
|
type: KClass<out T>,
|
||||||
path: Path,
|
|
||||||
format: IOFormat<T>,
|
format: IOFormat<T>,
|
||||||
metaFile: Path = path.resolveSibling("${path.fileName}.meta"),
|
envelopeFormat: EnvelopeFormat? = null,
|
||||||
|
metaFile: Path = resolveSibling("$fileName.meta"),
|
||||||
metaFileFormat: MetaFormat = JsonMetaFormat
|
metaFileFormat: MetaFormat = JsonMetaFormat
|
||||||
): Data<T> {
|
): Data<T> {
|
||||||
return coroutineScope {
|
return coroutineScope {
|
||||||
@ -57,14 +62,31 @@ suspend fun <T : Any> Context.readData(
|
|||||||
} else {
|
} else {
|
||||||
null
|
null
|
||||||
}
|
}
|
||||||
|
if (envelopeFormat == null) {
|
||||||
Data(type, externalMeta ?: EmptyMeta) {
|
Data(type, externalMeta ?: EmptyMeta) {
|
||||||
withContext(Dispatchers.IO) {
|
withContext(Dispatchers.IO) {
|
||||||
format.run {
|
format.run {
|
||||||
Files.newByteChannel(path, StandardOpenOption.READ)
|
Files.newByteChannel(this@readData, StandardOpenOption.READ)
|
||||||
.asInput()
|
.asInput()
|
||||||
.readThis()
|
.readThis()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
} else {
|
||||||
|
withContext(Dispatchers.IO) {
|
||||||
|
readEnvelope(envelopeFormat).let {
|
||||||
|
if (externalMeta == null) {
|
||||||
|
it
|
||||||
|
} else {
|
||||||
|
it.withMetaLayers(externalMeta)
|
||||||
|
}
|
||||||
|
}.toData(type, format)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
//suspend fun <T : Any> Path.writeData(
|
||||||
|
// data: Data<T>,
|
||||||
|
// format: IOFormat<T>,
|
||||||
|
// )
|
Loading…
Reference in New Issue
Block a user