From 71536051aae9b4991abfcd39ecd33cb0a4174473 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Wed, 18 Sep 2019 16:36:35 +0300 Subject: [PATCH] Data and DataNode file io update --- build.gradle.kts | 6 +- .../kotlin/hep/dataforge/data/DataNode.kt | 2 +- dataforge-io/build.gradle.kts | 4 +- .../kotlin/hep/dataforge/io/Envelope.kt | 3 +- .../io/functions/RemoteFunctionClient.kt | 4 +- .../io/functions/RemoteFunctionServer.kt | 2 +- .../hep/dataforge/io/EnvelopeFormatTest.kt | 2 +- .../kotlin/hep/dataforge/io/FileBinary.kt | 4 +- .../kotlin/hep/dataforge/io/FileEnvelope.kt | 2 +- .../kotlin/hep/dataforge/io/ioFormatsJVM.kt | 16 +++ .../hep/dataforge/io/tcp/EnvelopeClient.kt | 2 +- .../hep/dataforge/io/FileEnvelopeTest.kt | 30 ++++++ .../dataforge/io/tcp/EnvelopeServerTest.kt | 2 +- .../hep/dataforge/workspace/fileData.kt | 99 ++++++++++--------- 14 files changed, 117 insertions(+), 61 deletions(-) create mode 100644 dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/FileEnvelopeTest.kt diff --git a/build.gradle.kts b/build.gradle.kts index 41cf03f2..b0adec0e 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -1,9 +1,9 @@ plugins { - id("scientifik.mpp") version "0.1.7-dev" apply false - id("scientifik.publish") version "0.1.7-dev" apply false + id("scientifik.mpp") version "0.1.7" apply false + id("scientifik.publish") version "0.1.7" apply false } -val dataforgeVersion by extra("0.1.4-dev-2") +val dataforgeVersion by extra("0.1.4-dev-3") val bintrayRepo by extra("dataforge") val githubProject by extra("dataforge-core") diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataNode.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataNode.kt index d7158dab..5e220758 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataNode.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataNode.kt @@ -140,7 +140,7 @@ private sealed class DataTreeBuilderItem { /** * A builder for a DataTree. */ -class DataTreeBuilder(private val type: KClass) { +class DataTreeBuilder(val type: KClass) { private val map = HashMap>() operator fun set(token: NameToken, node: DataTreeBuilder) { diff --git a/dataforge-io/build.gradle.kts b/dataforge-io/build.gradle.kts index 8a997017..083e9d53 100644 --- a/dataforge-io/build.gradle.kts +++ b/dataforge-io/build.gradle.kts @@ -5,8 +5,8 @@ plugins { description = "IO module" scientifik{ - serialization = true - io = true + withSerialization() + withIO() } diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Envelope.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Envelope.kt index 6c3b45b9..80e07b56 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Envelope.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Envelope.kt @@ -26,7 +26,7 @@ interface Envelope { /** * Build a static envelope using provided builder */ - fun build(block: EnvelopeBuilder.() -> Unit) = EnvelopeBuilder().apply(block).build() + operator fun invoke(block: EnvelopeBuilder.() -> Unit) = EnvelopeBuilder().apply(block).build() } } @@ -97,6 +97,7 @@ class EnvelopeBuilder { var type by metaBuilder.string(key = Envelope.ENVELOPE_TYPE_KEY) var dataType by metaBuilder.string(key = Envelope.ENVELOPE_DATA_TYPE_KEY) + var dataID by metaBuilder.string(key = Envelope.ENVELOPE_DATA_ID_KEY) var description by metaBuilder.string(key = Envelope.ENVELOPE_DESCRIPTION_KEY) /** diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/functions/RemoteFunctionClient.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/functions/RemoteFunctionClient.kt index 0ba90282..770e5578 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/functions/RemoteFunctionClient.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/functions/RemoteFunctionClient.kt @@ -12,7 +12,7 @@ class RemoteFunctionClient(override val context: Context, val responder: Respond private fun IOPlugin.encodeOne( meta: Meta, value: T - ): Envelope = Envelope.build { + ): Envelope = Envelope.invoke { meta(meta) type = REQUEST_TYPE data { @@ -26,7 +26,7 @@ class RemoteFunctionClient(override val context: Context, val responder: Respond private fun IOPlugin.encodeMany( meta: Meta, values: List - ): Envelope = Envelope.build { + ): Envelope = Envelope.invoke { meta(meta) type = REQUEST_TYPE meta { diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/functions/RemoteFunctionServer.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/functions/RemoteFunctionServer.kt index 41a2a271..b2fb6cd0 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/functions/RemoteFunctionServer.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/functions/RemoteFunctionServer.kt @@ -40,7 +40,7 @@ class RemoteFunctionServer( input ) - return Envelope.build { + return Envelope.invoke { meta { meta(request.meta) } diff --git a/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/EnvelopeFormatTest.kt b/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/EnvelopeFormatTest.kt index 059c0f19..5c1d82f9 100644 --- a/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/EnvelopeFormatTest.kt +++ b/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/EnvelopeFormatTest.kt @@ -5,7 +5,7 @@ import kotlin.test.assertEquals class EnvelopeFormatTest { - val envelope = Envelope.build { + val envelope = Envelope.invoke { type = "test.format" meta{ "d" to 22.2 diff --git a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileBinary.kt b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileBinary.kt index b69969ed..2c357a47 100644 --- a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileBinary.kt +++ b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileBinary.kt @@ -20,4 +20,6 @@ class FileBinary(val path: Path, private val offset: UInt = 0u, size: ULong? = n return ByteReadPacket(buffer).block() } } -} \ No newline at end of file +} + +fun Path.asBinary(offset: UInt = 0u, size: ULong? = null): FileBinary = FileBinary(this, offset, size) \ No newline at end of file diff --git a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileEnvelope.kt b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileEnvelope.kt index 0c54012c..88ebf285 100644 --- a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileEnvelope.kt +++ b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileEnvelope.kt @@ -22,7 +22,7 @@ class FileEnvelope internal constructor(val path: Path, val format: EnvelopeForm override val data: Binary? = FileBinary(path, partialEnvelope.dataOffset, partialEnvelope.dataSize) } -fun Path.readEnvelope(format: EnvelopeFormat) = FileEnvelope(this, format) +fun Path.readEnvelope(format: EnvelopeFormat = TaggedEnvelopeFormat) = FileEnvelope(this, format) fun Path.writeEnvelope( envelope: Envelope, diff --git a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/ioFormatsJVM.kt b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/ioFormatsJVM.kt index 8ba97fc3..6d075284 100644 --- a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/ioFormatsJVM.kt +++ b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/ioFormatsJVM.kt @@ -1,5 +1,6 @@ package hep.dataforge.io +import hep.dataforge.descriptors.NodeDescriptor import hep.dataforge.io.functions.FunctionServer import hep.dataforge.io.functions.FunctionServer.Companion.FUNCTION_NAME_KEY import hep.dataforge.io.functions.FunctionServer.Companion.INPUT_FORMAT_KEY @@ -7,6 +8,10 @@ import hep.dataforge.io.functions.FunctionServer.Companion.OUTPUT_FORMAT_KEY import hep.dataforge.meta.Meta import hep.dataforge.meta.buildMeta import hep.dataforge.names.Name +import kotlinx.io.nio.asOutput +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.StandardOpenOption import kotlin.reflect.KClass import kotlin.reflect.full.isSuperclassOf @@ -31,4 +36,15 @@ inline fun FunctionServer.function( val plugin = context.plugins.get() ?: error("IO plugin not loaded") val meta = plugin.generateFunctionMeta(functionName) return function(meta) +} + +/** + * Write meta to file in a given [format] + */ +fun Meta.write(path: Path, format: MetaFormat, descriptor: NodeDescriptor? = null) { + format.run { + Files.newByteChannel(path, StandardOpenOption.WRITE, StandardOpenOption.CREATE_NEW) + .asOutput() + .writeMeta(this@write, descriptor) + } } \ No newline at end of file diff --git a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/tcp/EnvelopeClient.kt b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/tcp/EnvelopeClient.kt index 17b0429d..9011ae1e 100644 --- a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/tcp/EnvelopeClient.kt +++ b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/tcp/EnvelopeClient.kt @@ -34,7 +34,7 @@ class EnvelopeClient( suspend fun close() { try { respond( - Envelope.build { + Envelope.invoke { type = EnvelopeServer.SHUTDOWN_ENVELOPE_TYPE } ) diff --git a/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/FileEnvelopeTest.kt b/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/FileEnvelopeTest.kt new file mode 100644 index 00000000..c76b446e --- /dev/null +++ b/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/FileEnvelopeTest.kt @@ -0,0 +1,30 @@ +package hep.dataforge.io + +import java.nio.file.Files +import kotlin.test.Test +import kotlin.test.assertTrue + + +class FileEnvelopeTest { + val envelope = Envelope { + meta { + "a" to "AAA" + "b" to 22.2 + } + dataType = "hep.dataforge.test" + dataID = "myData" // добавил только что + data { + writeDouble(16.7) + + } + } + + @Test + fun testFileWriteRead() { + val tmpPath = Files.createTempFile("dataforge_test", ".df") + tmpPath.writeEnvelope(envelope) + println(tmpPath.toUri()) + val restored: Envelope = tmpPath.readEnvelope() + assertTrue { envelope.contentEquals(restored) } + } +} \ No newline at end of file diff --git a/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/tcp/EnvelopeServerTest.kt b/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/tcp/EnvelopeServerTest.kt index d48b42ef..5691f0a0 100644 --- a/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/tcp/EnvelopeServerTest.kt +++ b/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/tcp/EnvelopeServerTest.kt @@ -46,7 +46,7 @@ class EnvelopeServerTest { @Test fun doEchoTest() { - val request = Envelope.build { + val request = Envelope.invoke { type = "test.echo" meta { "test.value" to 22 diff --git a/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/fileData.kt b/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/fileData.kt index f20e18cf..08959298 100644 --- a/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/fileData.kt +++ b/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/fileData.kt @@ -1,15 +1,16 @@ package hep.dataforge.workspace import hep.dataforge.data.Data +import hep.dataforge.data.DataNode +import hep.dataforge.data.DataTreeBuilder +import hep.dataforge.data.datum import hep.dataforge.descriptors.NodeDescriptor import hep.dataforge.io.* import hep.dataforge.meta.EmptyMeta import hep.dataforge.meta.Meta import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.coroutineScope import kotlinx.coroutines.withContext import kotlinx.io.nio.asInput -import kotlinx.io.nio.asOutput import java.nio.file.Files import java.nio.file.Path import java.nio.file.StandardOpenOption @@ -18,69 +19,75 @@ import kotlin.reflect.KClass /** * Read meta from file in a given [format] */ -suspend fun Path.readMeta(format: MetaFormat, descriptor: NodeDescriptor? = null): Meta { - return withContext(Dispatchers.IO) { - format.run { - Files.newByteChannel(this@readMeta, StandardOpenOption.READ) - .asInput() - .readMeta(descriptor) - } - } -} - -/** - * Write meta to file in a given [format] - */ -suspend fun Meta.write(path: Path, format: MetaFormat, descriptor: NodeDescriptor? = null) { - withContext(Dispatchers.IO) { - format.run { - Files.newByteChannel(path, StandardOpenOption.WRITE, StandardOpenOption.CREATE_NEW) - .asOutput() - .writeMeta(this@write, descriptor) - } +fun Path.readMeta(format: MetaFormat, descriptor: NodeDescriptor? = null): Meta { + return format.run { + Files.newByteChannel(this@readMeta, StandardOpenOption.READ) + .asInput() + .readMeta(descriptor) } } /** * Read data with supported envelope format and binary format. If envelope format is null, then read binary directly from file. + * The operation is blocking since it must read meta header. The reading of envelope body is lazy * @param type explicit type of data read * @param format binary format * @param envelopeFormat the format of envelope. If null, file is read directly * @param metaFile the relative file for optional meta override * @param metaFileFormat the meta format for override */ -suspend fun Path.readData( +fun Path.readData( type: KClass, format: IOFormat, envelopeFormat: EnvelopeFormat? = null, metaFile: Path = resolveSibling("$fileName.meta"), metaFileFormat: MetaFormat = JsonMetaFormat ): Data { - return coroutineScope { - val externalMeta = if (Files.exists(metaFile)) { - metaFile.readMeta(metaFileFormat) - } else { - null - } - if (envelopeFormat == null) { - Data(type, externalMeta ?: EmptyMeta) { - withContext(Dispatchers.IO) { - format.run { - Files.newByteChannel(this@readData, StandardOpenOption.READ) - .asInput() - .readThis() - } + val externalMeta = if (Files.exists(metaFile)) { + metaFile.readMeta(metaFileFormat) + } else { + null + } + return if (envelopeFormat == null) { + Data(type, externalMeta ?: EmptyMeta) { + withContext(Dispatchers.IO) { + format.run { + Files.newByteChannel(this@readData, StandardOpenOption.READ) + .asInput() + .readThis() } } - } else { - withContext(Dispatchers.IO) { - readEnvelope(envelopeFormat).let { - if (externalMeta == null) { - it - } else { - it.withMetaLayers(externalMeta) - } - }.toData(type, format) + } + } else { + readEnvelope(envelopeFormat).let { + if (externalMeta == null) { + it + } else { + it.withMetaLayers(externalMeta) + } + }.toData(type, format) + } +} + +fun DataTreeBuilder.file(path: Path, format: IOFormat, envelopeFormat: EnvelopeFormat? = null) { + val data = path.readData(type, format, envelopeFormat) + val name = path.fileName.toString().replace(".df", "") + datum(name, data) +} + +/** + * Read the directory as a data node + */ +fun Path.readDataNode( + type: KClass, + format: IOFormat, + envelopeFormat: EnvelopeFormat? = null +): DataNode { + if (!Files.isDirectory(this)) error("Provided path $this is not a directory") + return DataNode(type) { + Files.list(this@readDataNode).forEach { path -> + if (!path.fileName.toString().endsWith(".meta")) { + file(path, format, envelopeFormat) } } }