kscience/dataforge-core
7
0
Fork 0
Code Issues Pull Requests 1 Packages Projects Releases 12 Wiki Activity

Binary and file data update

Browse Source
This commit is contained in:
Alexander Nozik 2019-08-11 18:35:03 +03:00
parent a8c9100539
commit 8c32eaeb6b
8 changed files with 121 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
dataforge-data/build.gradle.kts
View File

@ -5,8 +5,6 @@ plugins {
val coroutinesVersion: String = Scientifik.coroutinesVersion
kotlin {
jvm()
js()
sourceSets {
val commonMain by getting{
dependencies {

4
dataforge-io/build.gradle.kts
View File

@ -12,12 +12,12 @@ scientifik{
kotlin {
sourceSets {
val commonMain by getting{
commonMain{
dependencies {
api(project(":dataforge-context"))
}
}
val jsMain by getting{
jsMain{
dependencies{
api(npm("text-encoding"))
}

23
dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Binary.kt
View File

@ -2,6 +2,7 @@ package hep.dataforge.io
import kotlinx.io.core.ByteReadPacket
import kotlinx.io.core.Input
import kotlinx.io.core.buildPacket
import kotlinx.io.core.readBytes
/**
@ -52,9 +53,29 @@ object EmptyBinary : RandomAccessBinary {
}
class ArrayBinary(val array: ByteArray) : RandomAccessBinary {
override val size: ULong = array.size.toULong()
override val size: ULong get() = array.size.toULong()
override fun <R> read(from: UInt, size: UInt, block: Input.() -> R): R {
return ByteReadPacket(array, from.toInt(), size.toInt()).block()
}
}
/**
* Read given binary as object using given format
*/
fun <T : Any> Binary.readWith(format: IOFormat<T>): T = format.run {
read {
readThis()
}
}
/**
* Write this object to a binary
* TODO make a lazy binary that does not use intermediate array
*/
fun <T: Any> T.writeWith(format: IOFormat<T>): Binary = format.run{
val packet = buildPacket {
writeThis(this@writeWith)
}
return@run ArrayBinary(packet.readBytes())
}

37
dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Envelope.kt
View File

@ -1,14 +1,9 @@
package hep.dataforge.io
import hep.dataforge.context.Named
import hep.dataforge.io.EnvelopeFormat.Companion.ENVELOPE_FORMAT_TYPE
import hep.dataforge.io.IOPlugin.Companion.defaultMetaFormats
import hep.dataforge.meta.Laminate
import hep.dataforge.meta.Meta
import hep.dataforge.meta.get
import hep.dataforge.meta.string
import hep.dataforge.provider.Type
import kotlinx.io.core.Input
import kotlinx.io.core.Output
interface Envelope {
val meta: Meta
@ -52,24 +47,20 @@ val Envelope.dataType: String? get() = meta[Envelope.ENVELOPE_DATA_TYPE_KEY].str
val Envelope.description: String? get() = meta[Envelope.ENVELOPE_DESCRIPTION_KEY].string
/**
* A partially read envelope with meta, but without data
* An envelope, which wraps existing envelope and adds one or several additional layers of meta
*/
@ExperimentalUnsignedTypes
data class PartialEnvelope(val meta: Meta, val dataOffset: UInt, val dataSize: ULong?)
class ProxyEnvelope(val source: Envelope, vararg meta: Meta) : Envelope {
override val meta: Laminate = Laminate(*meta, source.meta)
override val data: Binary? get() = source.data
}
@Type(ENVELOPE_FORMAT_TYPE)
interface EnvelopeFormat : IOFormat<Envelope>, Named {
fun Input.readPartial(formats: Collection<MetaFormat> = defaultMetaFormats): PartialEnvelope
fun Input.readEnvelope(formats: Collection<MetaFormat> = defaultMetaFormats): Envelope
override fun Input.readThis(): Envelope = readEnvelope()
fun Output.writeEnvelope(envelope: Envelope, format: MetaFormat = JsonMetaFormat)
override fun Output.writeThis(obj: Envelope) = writeEnvelope(obj)
companion object {
const val ENVELOPE_FORMAT_TYPE = "envelopeFormat"
/**
* Add few meta layers to existing envelope
*/
fun Envelope.withMetaLayers(vararg layers: Meta): Envelope {
return when {
layers.isEmpty() -> this
this is ProxyEnvelope -> ProxyEnvelope(source, *layers, *this.meta.layers.toTypedArray())
else -> ProxyEnvelope(this, *layers)
}
}

31
dataforge-io/src/commonMain/kotlin/hep/dataforge/io/EnvelopeFormat.kt Normal file
View File

@ -0,0 +1,31 @@
package hep.dataforge.io
import hep.dataforge.context.Named
import hep.dataforge.io.EnvelopeFormat.Companion.ENVELOPE_FORMAT_TYPE
import hep.dataforge.meta.Meta
import hep.dataforge.provider.Type
import kotlinx.io.core.Input
import kotlinx.io.core.Output
/**
* A partially read envelope with meta, but without data
*/
@ExperimentalUnsignedTypes
data class PartialEnvelope(val meta: Meta, val dataOffset: UInt, val dataSize: ULong?)
@Type(ENVELOPE_FORMAT_TYPE)
interface EnvelopeFormat : IOFormat<Envelope>, Named {
fun Input.readPartial(formats: Collection<MetaFormat> = IOPlugin.defaultMetaFormats): PartialEnvelope
fun Input.readEnvelope(formats: Collection<MetaFormat> = IOPlugin.defaultMetaFormats): Envelope
override fun Input.readThis(): Envelope = readEnvelope()
fun Output.writeEnvelope(envelope: Envelope, format: MetaFormat = JsonMetaFormat)
override fun Output.writeThis(obj: Envelope) = writeEnvelope(obj)
companion object {
const val ENVELOPE_FORMAT_TYPE = "envelopeFormat"
}
}

1
dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileEnvelope.kt
View File

@ -40,3 +40,4 @@ fun Path.writeEnvelope(
output.writeEnvelope(envelope, metaFormat)
}
}

14
dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/dataUtils.kt Normal file
View File

@ -0,0 +1,14 @@
package hep.dataforge.workspace
import hep.dataforge.data.Data
import hep.dataforge.io.Envelope
import hep.dataforge.io.IOFormat
import hep.dataforge.io.readWith
import kotlin.reflect.KClass
/**
* Convert an [Envelope] to a data via given format. The actual parsing is done lazily.
*/
fun <T : Any> Envelope.toData(type: KClass<out T>, format: IOFormat<T>): Data<T> = Data(type, meta) {
data?.readWith(format) ?: error("Can't convert envelope without data to Data")
}

40
dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/fileData.kt
View File

@ -1,11 +1,8 @@
package hep.dataforge.workspace
import hep.dataforge.context.Context
import hep.dataforge.data.Data
import hep.dataforge.descriptors.NodeDescriptor
import hep.dataforge.io.IOFormat
import hep.dataforge.io.JsonMetaFormat
import hep.dataforge.io.MetaFormat
import hep.dataforge.io.*
import hep.dataforge.meta.EmptyMeta
import hep.dataforge.meta.Meta
import kotlinx.coroutines.Dispatchers
@ -44,11 +41,19 @@ suspend fun Meta.write(path: Path, format: MetaFormat, descriptor: NodeDescripto
}
}
suspend fun <T : Any> Context.readData(
/**
* Read data with supported envelope format and binary format. If envelope format is null, then read binary directly from file.
* @param type explicit type of data read
* @param format binary format
* @param envelopeFormat the format of envelope. If null, file is read directly
* @param metaFile the relative file for optional meta override
* @param metaFileFormat the meta format for override
*/
suspend fun <T : Any> Path.readData(
type: KClass<out T>,
path: Path,
format: IOFormat<T>,
metaFile: Path = path.resolveSibling("${path.fileName}.meta"),
envelopeFormat: EnvelopeFormat? = null,
metaFile: Path = resolveSibling("$fileName.meta"),
metaFileFormat: MetaFormat = JsonMetaFormat
): Data<T> {
return coroutineScope {
@ -57,14 +62,31 @@ suspend fun <T : Any> Context.readData(
} else {
null
}
Data(type, externalMeta ?: EmptyMeta){
if (envelopeFormat == null) {
Data(type, externalMeta ?: EmptyMeta) {
withContext(Dispatchers.IO) {
format.run {
Files.newByteChannel(path, StandardOpenOption.READ)
Files.newByteChannel(this@readData, StandardOpenOption.READ)
.asInput()
.readThis()
}
}
}
} else {
withContext(Dispatchers.IO) {
readEnvelope(envelopeFormat).let {
if (externalMeta == null) {
it
} else {
it.withMetaLayers(externalMeta)
}
}.toData(type, format)
}
}
}
}
//suspend fun <T : Any> Path.writeData(
// data: Data<T>,
// format: IOFormat<T>,
// )