kscience/dataforge-core
8
0
Fork 0
Code Issues Pull Requests 1 Packages Projects Releases 12 Wiki Activity

Data and DataNode file io update

Browse Source
This commit is contained in:
Alexander Nozik 2019-09-18 16:36:35 +03:00
parent 6c1c49d15e
commit 71536051aa
14 changed files with 117 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
build.gradle.kts
View File

@ -1,9 +1,9 @@
plugins {
id("scientifik.mpp") version "0.1.7-dev" apply false
id("scientifik.publish") version "0.1.7-dev" apply false
id("scientifik.mpp") version "0.1.7" apply false
id("scientifik.publish") version "0.1.7" apply false
}
val dataforgeVersion by extra("0.1.4-dev-2")
val dataforgeVersion by extra("0.1.4-dev-3")
val bintrayRepo by extra("dataforge")
val githubProject by extra("dataforge-core")

2
dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataNode.kt
View File

@ -140,7 +140,7 @@ private sealed class DataTreeBuilderItem<out T : Any> {
/**
* A builder for a DataTree.
*/
class DataTreeBuilder<T : Any>(private val type: KClass<out T>) {
class DataTreeBuilder<T : Any>(val type: KClass<out T>) {
private val map = HashMap<NameToken, DataTreeBuilderItem<T>>()
operator fun set(token: NameToken, node: DataTreeBuilder<out T>) {

4
dataforge-io/build.gradle.kts
View File

@ -5,8 +5,8 @@ plugins {
description = "IO module"
scientifik{
serialization = true
io = true
withSerialization()
withIO()
}

3
dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Envelope.kt
View File

@ -26,7 +26,7 @@ interface Envelope {
/**
* Build a static envelope using provided builder
*/
fun build(block: EnvelopeBuilder.() -> Unit) = EnvelopeBuilder().apply(block).build()
operator fun invoke(block: EnvelopeBuilder.() -> Unit) = EnvelopeBuilder().apply(block).build()
}
}
@ -97,6 +97,7 @@ class EnvelopeBuilder {
var type by metaBuilder.string(key = Envelope.ENVELOPE_TYPE_KEY)
var dataType by metaBuilder.string(key = Envelope.ENVELOPE_DATA_TYPE_KEY)
var dataID by metaBuilder.string(key = Envelope.ENVELOPE_DATA_ID_KEY)
var description by metaBuilder.string(key = Envelope.ENVELOPE_DESCRIPTION_KEY)
/**

4
dataforge-io/src/commonMain/kotlin/hep/dataforge/io/functions/RemoteFunctionClient.kt
View File

@ -12,7 +12,7 @@ class RemoteFunctionClient(override val context: Context, val responder: Respond
private fun <T : Any> IOPlugin.encodeOne(
meta: Meta,
value: T
): Envelope = Envelope.build {
): Envelope = Envelope.invoke {
meta(meta)
type = REQUEST_TYPE
data {
@ -26,7 +26,7 @@ class RemoteFunctionClient(override val context: Context, val responder: Respond
private fun <T : Any> IOPlugin.encodeMany(
meta: Meta,
values: List<T>
): Envelope = Envelope.build {
): Envelope = Envelope.invoke {
meta(meta)
type = REQUEST_TYPE
meta {

2
dataforge-io/src/commonMain/kotlin/hep/dataforge/io/functions/RemoteFunctionServer.kt
View File

@ -40,7 +40,7 @@ class RemoteFunctionServer(
input
)
return Envelope.build {
return Envelope.invoke {
meta {
meta(request.meta)
}

2
dataforge-io/src/commonTest/kotlin/hep/dataforge/io/EnvelopeFormatTest.kt
View File

@ -5,7 +5,7 @@ import kotlin.test.assertEquals
class EnvelopeFormatTest {
val envelope = Envelope.build {
val envelope = Envelope.invoke {
type = "test.format"
meta{
"d" to 22.2

4
dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileBinary.kt
View File

@ -20,4 +20,6 @@ class FileBinary(val path: Path, private val offset: UInt = 0u, size: ULong? = n
return ByteReadPacket(buffer).block()
}
}
}
}
fun Path.asBinary(offset: UInt = 0u, size: ULong? = null): FileBinary = FileBinary(this, offset, size)

2
dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileEnvelope.kt
View File

@ -22,7 +22,7 @@ class FileEnvelope internal constructor(val path: Path, val format: EnvelopeForm
override val data: Binary? = FileBinary(path, partialEnvelope.dataOffset, partialEnvelope.dataSize)
}
fun Path.readEnvelope(format: EnvelopeFormat) = FileEnvelope(this, format)
fun Path.readEnvelope(format: EnvelopeFormat = TaggedEnvelopeFormat) = FileEnvelope(this, format)
fun Path.writeEnvelope(
envelope: Envelope,

16
dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/ioFormatsJVM.kt
View File

@ -1,5 +1,6 @@
package hep.dataforge.io
import hep.dataforge.descriptors.NodeDescriptor
import hep.dataforge.io.functions.FunctionServer
import hep.dataforge.io.functions.FunctionServer.Companion.FUNCTION_NAME_KEY
import hep.dataforge.io.functions.FunctionServer.Companion.INPUT_FORMAT_KEY
@ -7,6 +8,10 @@ import hep.dataforge.io.functions.FunctionServer.Companion.OUTPUT_FORMAT_KEY
import hep.dataforge.meta.Meta
import hep.dataforge.meta.buildMeta
import hep.dataforge.names.Name
import kotlinx.io.nio.asOutput
import java.nio.file.Files
import java.nio.file.Path
import java.nio.file.StandardOpenOption
import kotlin.reflect.KClass
import kotlin.reflect.full.isSuperclassOf
@ -31,4 +36,15 @@ inline fun <reified T : Any, reified R : Any> FunctionServer.function(
val plugin = context.plugins.get<IOPlugin>() ?: error("IO plugin not loaded")
val meta = plugin.generateFunctionMeta<T, R>(functionName)
return function(meta)
}
/**
* Write meta to file in a given [format]
*/
fun Meta.write(path: Path, format: MetaFormat, descriptor: NodeDescriptor? = null) {
format.run {
Files.newByteChannel(path, StandardOpenOption.WRITE, StandardOpenOption.CREATE_NEW)
.asOutput()
.writeMeta(this@write, descriptor)
}
}

2
dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/tcp/EnvelopeClient.kt
View File

@ -34,7 +34,7 @@ class EnvelopeClient(
suspend fun close() {
try {
respond(
Envelope.build {
Envelope.invoke {
type = EnvelopeServer.SHUTDOWN_ENVELOPE_TYPE
}
)

30
dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/FileEnvelopeTest.kt Normal file
View File

@ -0,0 +1,30 @@
package hep.dataforge.io
import java.nio.file.Files
import kotlin.test.Test
import kotlin.test.assertTrue
class FileEnvelopeTest {
val envelope = Envelope {
meta {
"a" to "AAA"
"b" to 22.2
}
dataType = "hep.dataforge.test"
dataID = "myData" // добавил только что
data {
writeDouble(16.7)
}
}
@Test
fun testFileWriteRead() {
val tmpPath = Files.createTempFile("dataforge_test", ".df")
tmpPath.writeEnvelope(envelope)
println(tmpPath.toUri())
val restored: Envelope = tmpPath.readEnvelope()
assertTrue { envelope.contentEquals(restored) }
}
}

2
dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/tcp/EnvelopeServerTest.kt
View File

@ -46,7 +46,7 @@ class EnvelopeServerTest {
@Test
fun doEchoTest() {
val request = Envelope.build {
val request = Envelope.invoke {
type = "test.echo"
meta {
"test.value" to 22

99
dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/fileData.kt
View File

@ -1,15 +1,16 @@
package hep.dataforge.workspace
import hep.dataforge.data.Data
import hep.dataforge.data.DataNode
import hep.dataforge.data.DataTreeBuilder
import hep.dataforge.data.datum
import hep.dataforge.descriptors.NodeDescriptor
import hep.dataforge.io.*
import hep.dataforge.meta.EmptyMeta
import hep.dataforge.meta.Meta
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.coroutineScope
import kotlinx.coroutines.withContext
import kotlinx.io.nio.asInput
import kotlinx.io.nio.asOutput
import java.nio.file.Files
import java.nio.file.Path
import java.nio.file.StandardOpenOption
@ -18,69 +19,75 @@ import kotlin.reflect.KClass
/**
* Read meta from file in a given [format]
*/
suspend fun Path.readMeta(format: MetaFormat, descriptor: NodeDescriptor? = null): Meta {
return withContext(Dispatchers.IO) {
format.run {
Files.newByteChannel(this@readMeta, StandardOpenOption.READ)
.asInput()
.readMeta(descriptor)
}
}
}
/**
* Write meta to file in a given [format]
*/
suspend fun Meta.write(path: Path, format: MetaFormat, descriptor: NodeDescriptor? = null) {
withContext(Dispatchers.IO) {
format.run {
Files.newByteChannel(path, StandardOpenOption.WRITE, StandardOpenOption.CREATE_NEW)
.asOutput()
.writeMeta(this@write, descriptor)
}
fun Path.readMeta(format: MetaFormat, descriptor: NodeDescriptor? = null): Meta {
return format.run {
Files.newByteChannel(this@readMeta, StandardOpenOption.READ)
.asInput()
.readMeta(descriptor)
}
}
/**
* Read data with supported envelope format and binary format. If envelope format is null, then read binary directly from file.
* The operation is blocking since it must read meta header. The reading of envelope body is lazy
* @param type explicit type of data read
* @param format binary format
* @param envelopeFormat the format of envelope. If null, file is read directly
* @param metaFile the relative file for optional meta override
* @param metaFileFormat the meta format for override
*/
suspend fun <T : Any> Path.readData(
fun <T : Any> Path.readData(
type: KClass<out T>,
format: IOFormat<T>,
envelopeFormat: EnvelopeFormat? = null,
metaFile: Path = resolveSibling("$fileName.meta"),
metaFileFormat: MetaFormat = JsonMetaFormat
): Data<T> {
return coroutineScope {
val externalMeta = if (Files.exists(metaFile)) {
metaFile.readMeta(metaFileFormat)
} else {
null
}
if (envelopeFormat == null) {
Data(type, externalMeta ?: EmptyMeta) {
withContext(Dispatchers.IO) {
format.run {
Files.newByteChannel(this@readData, StandardOpenOption.READ)
.asInput()
.readThis()
}
val externalMeta = if (Files.exists(metaFile)) {
metaFile.readMeta(metaFileFormat)
} else {
null
}
return if (envelopeFormat == null) {
Data(type, externalMeta ?: EmptyMeta) {
withContext(Dispatchers.IO) {
format.run {
Files.newByteChannel(this@readData, StandardOpenOption.READ)
.asInput()
.readThis()
}
}
} else {
withContext(Dispatchers.IO) {
readEnvelope(envelopeFormat).let {
if (externalMeta == null) {
it
} else {
it.withMetaLayers(externalMeta)
}
}.toData(type, format)
}
} else {
readEnvelope(envelopeFormat).let {
if (externalMeta == null) {
it
} else {
it.withMetaLayers(externalMeta)
}
}.toData(type, format)
}
}
fun <T : Any> DataTreeBuilder<T>.file(path: Path, format: IOFormat<T>, envelopeFormat: EnvelopeFormat? = null) {
val data = path.readData(type, format, envelopeFormat)
val name = path.fileName.toString().replace(".df", "")
datum(name, data)
}
/**
* Read the directory as a data node
*/
fun <T : Any> Path.readDataNode(
type: KClass<out T>,
format: IOFormat<T>,
envelopeFormat: EnvelopeFormat? = null
): DataNode<T> {
if (!Files.isDirectory(this)) error("Provided path $this is not a directory")
return DataNode(type) {
Files.list(this@readDataNode).forEach { path ->
if (!path.fileName.toString().endsWith(".meta")) {
file(path, format, envelopeFormat)
}
}
}