From 6450696157998ad65922b9c164dac4a9f9915567 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Mon, 6 Apr 2020 22:50:51 +0300 Subject: [PATCH] Fix zip writer and name refactoring --- .../hep/dataforge/context/PluginManager.kt | 6 +- .../kotlin/hep/dataforge/data/Data.kt | 18 +++-- .../kotlin/hep/dataforge/data/DataNode.kt | 2 +- dataforge-io/build.gradle.kts | 2 +- .../io/yaml/FrontMatterEnvelopeFormat.kt | 6 +- .../hep/dataforge/io/yaml/YamlMetaFormat.kt | 7 +- .../dataforge/io/yaml/YamlMetaFormatTest.kt | 2 + .../kotlin/hep/dataforge/io/IOFormat.kt | 11 ++- .../kotlin/hep/dataforge/io/IOPlugin.kt | 14 ++-- .../kotlin/hep/dataforge/io/JsonMetaFormat.kt | 3 +- .../hep/dataforge/io/TaggedEnvelopeFormat.kt | 6 +- .../hep/dataforge/io/TaglessEnvelopeFormat.kt | 6 +- .../kotlin/hep/dataforge/io/ioMisc.kt | 3 - .../hep/dataforge/io/MetaSerializerTest.kt | 13 ++-- .../hep/dataforge/io/tcp/EnvelopeClient.kt | 3 +- .../hep/dataforge/io/tcp/EnvelopeServer.kt | 5 +- .../kotlin/hep/dataforge/meta/Meta.kt | 2 +- .../hep/dataforge/meta/serializationUtils.kt | 3 +- .../hep/dataforge/meta/DynamicMetaTest.kt | 2 +- .../hep/dataforge/output/OutputManager.kt | 7 +- .../kotlin/hep/dataforge/output/Renderer.kt | 3 +- .../dataforge/tables/io/textTableEnvelope.kt | 5 +- .../hep/dataforge/workspace/TaskModel.kt | 2 +- .../hep/dataforge/workspace/fileData.kt | 76 ++++++++++++++----- .../hep/dataforge/workspace/FileDataTest.kt | 9 ++- 25 files changed, 132 insertions(+), 84 deletions(-) diff --git a/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/PluginManager.kt b/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/PluginManager.kt index 5f35d876..e6e8464e 100644 --- a/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/PluginManager.kt +++ b/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/PluginManager.kt @@ -1,9 +1,7 @@ package hep.dataforge.context -import hep.dataforge.meta.EmptyMeta import hep.dataforge.meta.Meta import hep.dataforge.meta.MetaBuilder -import hep.dataforge.meta.buildMeta import kotlin.reflect.KClass /** @@ -98,7 +96,7 @@ class PluginManager(override val context: Context) : ContextAware, Iterable load(factory: PluginFactory, meta: Meta = EmptyMeta): T = + fun load(factory: PluginFactory, meta: Meta = Meta.EMPTY): T = load(factory(meta, context)) fun load(factory: PluginFactory, metaBuilder: MetaBuilder.() -> Unit): T = @@ -121,7 +119,7 @@ class PluginManager(override val context: Context) : ContextAware, Iterable fetch(factory: PluginFactory, recursive: Boolean = true, meta: Meta = EmptyMeta): T { + fun fetch(factory: PluginFactory, recursive: Boolean = true, meta: Meta = Meta.EMPTY): T { val loaded = get(factory.type, factory.tag, recursive) return when { loaded == null -> load(factory(meta, context)) diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/Data.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/Data.kt index 4d787b69..233ea164 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/Data.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/Data.kt @@ -1,6 +1,8 @@ package hep.dataforge.data -import hep.dataforge.meta.* +import hep.dataforge.meta.Meta +import hep.dataforge.meta.MetaRepr +import hep.dataforge.meta.isEmpty import kotlinx.coroutines.CoroutineScope import kotlin.coroutines.CoroutineContext import kotlin.coroutines.EmptyCoroutineContext @@ -31,14 +33,14 @@ interface Data : Goal, MetaRepr{ operator fun invoke( type: KClass, - meta: Meta = EmptyMeta, + meta: Meta = Meta.EMPTY, context: CoroutineContext = EmptyCoroutineContext, dependencies: Collection> = emptyList(), block: suspend CoroutineScope.() -> T ): Data = DynamicData(type, meta, context, dependencies, block) inline operator fun invoke( - meta: Meta = EmptyMeta, + meta: Meta = Meta.EMPTY, context: CoroutineContext = EmptyCoroutineContext, dependencies: Collection> = emptyList(), noinline block: suspend CoroutineScope.() -> T @@ -47,7 +49,7 @@ interface Data : Goal, MetaRepr{ operator fun invoke( name: String, type: KClass, - meta: Meta = EmptyMeta, + meta: Meta = Meta.EMPTY, context: CoroutineContext = EmptyCoroutineContext, dependencies: Collection> = emptyList(), block: suspend CoroutineScope.() -> T @@ -55,14 +57,14 @@ interface Data : Goal, MetaRepr{ inline operator fun invoke( name: String, - meta: Meta = EmptyMeta, + meta: Meta = Meta.EMPTY, context: CoroutineContext = EmptyCoroutineContext, dependencies: Collection> = emptyList(), noinline block: suspend CoroutineScope.() -> T ): Data = invoke(name, T::class, meta, context, dependencies, block) - fun static(value: T, meta: Meta = EmptyMeta): Data = + fun static(value: T, meta: Meta = Meta.EMPTY): Data = StaticData(value, meta) } } @@ -70,7 +72,7 @@ interface Data : Goal, MetaRepr{ class DynamicData( override val type: KClass, - override val meta: Meta = EmptyMeta, + override val meta: Meta = Meta.EMPTY, context: CoroutineContext = EmptyCoroutineContext, dependencies: Collection> = emptyList(), block: suspend CoroutineScope.() -> T @@ -78,7 +80,7 @@ class DynamicData( class StaticData( value: T, - override val meta: Meta = EmptyMeta + override val meta: Meta = Meta.EMPTY ) : Data, StaticGoal(value) { override val type: KClass get() = value::class } diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataNode.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataNode.kt index ee8d9c4d..246d9796 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataNode.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataNode.kt @@ -250,7 +250,7 @@ fun DataTreeBuilder.datum(name: String, data: Data) { this[name.toName()] = data } -fun DataTreeBuilder.static(name: Name, data: T, meta: Meta = EmptyMeta) { +fun DataTreeBuilder.static(name: Name, data: T, meta: Meta = Meta.EMPTY) { this[name] = Data.static(data, meta) } diff --git a/dataforge-io/build.gradle.kts b/dataforge-io/build.gradle.kts index 5a94bae5..221d0d09 100644 --- a/dataforge-io/build.gradle.kts +++ b/dataforge-io/build.gradle.kts @@ -11,7 +11,7 @@ serialization(sourceSet = TEST){ cbor() } -val ioVersion by rootProject.extra("0.2.0-npm-dev-6") +val ioVersion by rootProject.extra("0.2.0-npm-dev-7") kotlin { sourceSets { diff --git a/dataforge-io/dataforge-io-yaml/src/main/kotlin/hep/dataforge/io/yaml/FrontMatterEnvelopeFormat.kt b/dataforge-io/dataforge-io-yaml/src/main/kotlin/hep/dataforge/io/yaml/FrontMatterEnvelopeFormat.kt index 8a837d74..0fffdb7e 100644 --- a/dataforge-io/dataforge-io-yaml/src/main/kotlin/hep/dataforge/io/yaml/FrontMatterEnvelopeFormat.kt +++ b/dataforge-io/dataforge-io-yaml/src/main/kotlin/hep/dataforge/io/yaml/FrontMatterEnvelopeFormat.kt @@ -2,6 +2,8 @@ package hep.dataforge.io.yaml import hep.dataforge.context.Context import hep.dataforge.io.* +import hep.dataforge.io.IOFormat.Companion.META_KEY +import hep.dataforge.io.IOFormat.Companion.NAME_KEY import hep.dataforge.meta.DFExperimental import hep.dataforge.meta.Meta import kotlinx.io.* @@ -78,8 +80,8 @@ class FrontMatterEnvelopeFormat( } override fun toMeta(): Meta = Meta { - IOPlugin.IO_FORMAT_NAME_KEY put name.toString() - IOPlugin.IO_FORMAT_META_KEY put meta + NAME_KEY put name.toString() + META_KEY put meta } companion object : EnvelopeFormatFactory { diff --git a/dataforge-io/dataforge-io-yaml/src/main/kotlin/hep/dataforge/io/yaml/YamlMetaFormat.kt b/dataforge-io/dataforge-io-yaml/src/main/kotlin/hep/dataforge/io/yaml/YamlMetaFormat.kt index 1799d072..5806abf8 100644 --- a/dataforge-io/dataforge-io-yaml/src/main/kotlin/hep/dataforge/io/yaml/YamlMetaFormat.kt +++ b/dataforge-io/dataforge-io-yaml/src/main/kotlin/hep/dataforge/io/yaml/YamlMetaFormat.kt @@ -1,7 +1,8 @@ package hep.dataforge.io.yaml import hep.dataforge.context.Context -import hep.dataforge.io.IOPlugin +import hep.dataforge.io.IOFormat.Companion.META_KEY +import hep.dataforge.io.IOFormat.Companion.NAME_KEY import hep.dataforge.io.MetaFormat import hep.dataforge.io.MetaFormatFactory import hep.dataforge.meta.DFExperimental @@ -30,8 +31,8 @@ class YamlMetaFormat(val meta: Meta) : MetaFormat { } override fun toMeta(): Meta = Meta{ - IOPlugin.IO_FORMAT_NAME_KEY put FrontMatterEnvelopeFormat.name.toString() - IOPlugin.IO_FORMAT_META_KEY put meta + NAME_KEY put FrontMatterEnvelopeFormat.name.toString() + META_KEY put meta } companion object : MetaFormatFactory { diff --git a/dataforge-io/dataforge-io-yaml/src/test/kotlin/hep/dataforge/io/yaml/YamlMetaFormatTest.kt b/dataforge-io/dataforge-io-yaml/src/test/kotlin/hep/dataforge/io/yaml/YamlMetaFormatTest.kt index 0bb1c5d0..83f6d3a6 100644 --- a/dataforge-io/dataforge-io-yaml/src/test/kotlin/hep/dataforge/io/yaml/YamlMetaFormatTest.kt +++ b/dataforge-io/dataforge-io-yaml/src/test/kotlin/hep/dataforge/io/yaml/YamlMetaFormatTest.kt @@ -2,6 +2,7 @@ package hep.dataforge.io.yaml import hep.dataforge.io.parse import hep.dataforge.io.toString +import hep.dataforge.meta.DFExperimental import hep.dataforge.meta.Meta import hep.dataforge.meta.get import hep.dataforge.meta.seal @@ -9,6 +10,7 @@ import kotlin.test.Test import kotlin.test.assertEquals +@DFExperimental class YamlMetaFormatTest { @Test fun testYamlMetaFormat() { diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOFormat.kt index 474d882e..7946c536 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOFormat.kt @@ -3,8 +3,8 @@ package hep.dataforge.io import hep.dataforge.context.Context import hep.dataforge.context.Factory import hep.dataforge.context.Named +import hep.dataforge.io.IOFormat.Companion.NAME_KEY import hep.dataforge.io.IOFormatFactory.Companion.IO_FORMAT_TYPE -import hep.dataforge.io.IOPlugin.Companion.IO_FORMAT_NAME_KEY import hep.dataforge.meta.Meta import hep.dataforge.meta.MetaItem import hep.dataforge.meta.MetaRepr @@ -23,6 +23,11 @@ import kotlin.reflect.KClass interface IOFormat : MetaRepr { fun Output.writeObject(obj: T) fun Input.readObject(): T + + companion object{ + val NAME_KEY = "name".asName() + val META_KEY = "meta".asName() + } } fun Input.readWith(format: IOFormat): T = format.run { readObject() } @@ -54,7 +59,7 @@ class ListIOFormat(val format: IOFormat) : IOFormat> { } override fun toMeta(): Meta = Meta { - IO_FORMAT_NAME_KEY put "list" + NAME_KEY put "list" "contentFormat" put format.toMeta() } } @@ -79,7 +84,7 @@ interface IOFormatFactory : Factory>, Named, MetaRepr { val type: KClass override fun toMeta(): Meta = Meta { - IO_FORMAT_NAME_KEY put name.toString() + NAME_KEY put name.toString() } companion object { diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOPlugin.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOPlugin.kt index d31e4264..73fd26f1 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOPlugin.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOPlugin.kt @@ -2,11 +2,12 @@ package hep.dataforge.io import hep.dataforge.context.* import hep.dataforge.io.EnvelopeFormatFactory.Companion.ENVELOPE_FORMAT_TYPE +import hep.dataforge.io.IOFormat.Companion.META_KEY +import hep.dataforge.io.IOFormat.Companion.NAME_KEY import hep.dataforge.io.IOFormatFactory.Companion.IO_FORMAT_TYPE import hep.dataforge.io.MetaFormatFactory.Companion.META_FORMAT_TYPE import hep.dataforge.meta.* import hep.dataforge.names.Name -import hep.dataforge.names.asName import hep.dataforge.names.toName import kotlin.reflect.KClass @@ -18,12 +19,12 @@ class IOPlugin(meta: Meta) : AbstractPlugin(meta) { } fun resolveIOFormat(item: MetaItem<*>, type: KClass): IOFormat? { - val key = item.string ?: item.node[IO_FORMAT_NAME_KEY]?.string ?: error("Format name not defined") + val key = item.string ?: item.node[NAME_KEY]?.string ?: error("Format name not defined") val name = key.toName() return ioFormatFactories.find { it.name == name }?.let { @Suppress("UNCHECKED_CAST") if (it.type != type) error("Format type ${it.type} is not the same as requested type $type") - else it.invoke(item.node[IO_FORMAT_META_KEY].node ?: Meta.EMPTY, context) as IOFormat + else it.invoke(item.node[META_KEY].node ?: Meta.EMPTY, context) as IOFormat } } @@ -46,8 +47,8 @@ class IOPlugin(meta: Meta) : AbstractPlugin(meta) { envelopeFormatFactories.find { it.name == name }?.invoke(meta, context) fun resolveEnvelopeFormat(item: MetaItem<*>): EnvelopeFormat? { - val name = item.string ?: item.node[IO_FORMAT_NAME_KEY]?.string ?: error("Envelope format name not defined") - val meta = item.node[IO_FORMAT_META_KEY].node ?: Meta.EMPTY + val name = item.string ?: item.node[NAME_KEY]?.string ?: error("Envelope format name not defined") + val meta = item.node[META_KEY].node ?: Meta.EMPTY return resolveEnvelopeFormat(name.toName(), meta) } @@ -60,9 +61,6 @@ class IOPlugin(meta: Meta) : AbstractPlugin(meta) { } companion object : PluginFactory { - val IO_FORMAT_NAME_KEY = "name".asName() - val IO_FORMAT_META_KEY = "meta".asName() - val defaultMetaFormats: List = listOf(JsonMetaFormat, BinaryMetaFormat) val defaultEnvelopeFormats = listOf(TaggedEnvelopeFormat, TaglessEnvelopeFormat) diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/JsonMetaFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/JsonMetaFormat.kt index ac32d314..e19a9f7d 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/JsonMetaFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/JsonMetaFormat.kt @@ -4,6 +4,7 @@ package hep.dataforge.io import hep.dataforge.context.Context +import hep.dataforge.io.IOFormat.Companion.NAME_KEY import hep.dataforge.meta.Meta import hep.dataforge.meta.descriptors.NodeDescriptor import hep.dataforge.meta.node @@ -26,7 +27,7 @@ class JsonMetaFormat(private val json: Json = DEFAULT_JSON) : MetaFormat { } override fun toMeta(): Meta = Meta{ - IOPlugin.IO_FORMAT_NAME_KEY put name.toString() + NAME_KEY put name.toString() } override fun Input.readMeta(descriptor: NodeDescriptor?): Meta { diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt index 10bbdab9..17fbc589 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt @@ -1,6 +1,8 @@ package hep.dataforge.io import hep.dataforge.context.Context +import hep.dataforge.io.IOFormat.Companion.META_KEY +import hep.dataforge.io.IOFormat.Companion.NAME_KEY import hep.dataforge.meta.Meta import hep.dataforge.meta.enum import hep.dataforge.meta.get @@ -99,8 +101,8 @@ class TaggedEnvelopeFormat( } override fun toMeta(): Meta = Meta { - IOPlugin.IO_FORMAT_NAME_KEY put name.toString() - IOPlugin.IO_FORMAT_META_KEY put { + NAME_KEY put name.toString() + META_KEY put { "version" put version } } diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaglessEnvelopeFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaglessEnvelopeFormat.kt index 2d86f814..c800d878 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaglessEnvelopeFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaglessEnvelopeFormat.kt @@ -1,6 +1,8 @@ package hep.dataforge.io import hep.dataforge.context.Context +import hep.dataforge.io.IOFormat.Companion.META_KEY +import hep.dataforge.io.IOFormat.Companion.NAME_KEY import hep.dataforge.meta.Meta import hep.dataforge.meta.get import hep.dataforge.meta.isEmpty @@ -162,8 +164,8 @@ class TaglessEnvelopeFormat( } override fun toMeta(): Meta = Meta { - IOPlugin.IO_FORMAT_NAME_KEY put name.toString() - IOPlugin.IO_FORMAT_META_KEY put meta + NAME_KEY put name.toString() + META_KEY put meta } companion object : EnvelopeFormatFactory { diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/ioMisc.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/ioMisc.kt index eddb7b40..9b214c5c 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/ioMisc.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/ioMisc.kt @@ -19,9 +19,6 @@ inline fun buildByteArray(expectedSize: Int = 16, block: Output.() -> Unit): Byt inline fun Binary(expectedSize: Int = 16, block: Output.() -> Unit): Binary = buildByteArray(expectedSize, block).asBinary() -@Deprecated("To be replaced by Binary.EMPTY", level = DeprecationLevel.WARNING) -val EmptyBinary = ByteArrayBinary(ByteArray(0)) - /** * View section of a [Binary] as an independent binary */ diff --git a/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/MetaSerializerTest.kt b/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/MetaSerializerTest.kt index 86a569a6..dc8e5016 100644 --- a/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/MetaSerializerTest.kt +++ b/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/MetaSerializerTest.kt @@ -1,12 +1,9 @@ package hep.dataforge.io -import hep.dataforge.meta.Meta -import hep.dataforge.meta.MetaItem -import hep.dataforge.meta.MetaSerializer +import hep.dataforge.meta.* import hep.dataforge.names.Name import hep.dataforge.names.toName import kotlinx.serialization.cbor.Cbor -import kotlinx.serialization.json.Json import kotlin.test.Test import kotlin.test.assertEquals @@ -22,8 +19,8 @@ class MetaSerializerTest { @Test fun testMetaSerialization() { - val string = Json.indented.stringify(MetaSerializer, meta) - val restored = Json.plain.parse(MetaSerializer, string) + val string = JSON_PRETTY.stringify(MetaSerializer, meta) + val restored = JSON_PLAIN.parse(MetaSerializer, string) assertEquals(restored, meta) } @@ -38,8 +35,8 @@ class MetaSerializerTest { @Test fun testNameSerialization() { val name = "a.b.c".toName() - val string = Json.indented.stringify(Name.serializer(), name) - val restored = Json.plain.parse(Name.serializer(), string) + val string = JSON_PRETTY.stringify(Name.serializer(), name) + val restored = JSON_PLAIN.parse(Name.serializer(), string) assertEquals(restored, name) } diff --git a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/tcp/EnvelopeClient.kt b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/tcp/EnvelopeClient.kt index 9562d146..156acd89 100644 --- a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/tcp/EnvelopeClient.kt +++ b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/tcp/EnvelopeClient.kt @@ -3,7 +3,6 @@ package hep.dataforge.io.tcp import hep.dataforge.context.Context import hep.dataforge.context.ContextAware import hep.dataforge.io.* -import hep.dataforge.meta.EmptyMeta import hep.dataforge.meta.Meta import kotlinx.coroutines.asCoroutineDispatcher import kotlinx.coroutines.withContext @@ -17,7 +16,7 @@ class EnvelopeClient( val host: String, val port: Int, formatFactory: EnvelopeFormatFactory = TaggedEnvelopeFormat, - formatMeta: Meta = EmptyMeta + formatMeta: Meta = Meta.EMPTY ) : Responder, ContextAware { private val dispatcher = Executors.newSingleThreadExecutor().asCoroutineDispatcher() diff --git a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/tcp/EnvelopeServer.kt b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/tcp/EnvelopeServer.kt index 10c5b712..fc61ee49 100644 --- a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/tcp/EnvelopeServer.kt +++ b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/tcp/EnvelopeServer.kt @@ -6,7 +6,6 @@ import hep.dataforge.io.EnvelopeFormatFactory import hep.dataforge.io.Responder import hep.dataforge.io.TaggedEnvelopeFormat import hep.dataforge.io.type -import hep.dataforge.meta.EmptyMeta import hep.dataforge.meta.Meta import kotlinx.coroutines.* import java.net.ServerSocket @@ -19,7 +18,7 @@ class EnvelopeServer( val responder: Responder, val scope: CoroutineScope, formatFactory: EnvelopeFormatFactory = TaggedEnvelopeFormat, - formatMeta: Meta = EmptyMeta + formatMeta: Meta = Meta.EMPTY ) : ContextAware { private var job: Job? = null @@ -78,7 +77,7 @@ class EnvelopeServer( logger.debug { "Accepted request with type ${request.type} from ${socket.remoteSocketAddress}" } if (request.type == SHUTDOWN_ENVELOPE_TYPE) { //Echo shutdown command - outputStream.write{ + outputStream.write { writeObject(request) } logger.info { "Accepted graceful shutdown signal from ${socket.inetAddress}" } diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Meta.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Meta.kt index 75cae67d..1afbb183 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Meta.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Meta.kt @@ -188,7 +188,7 @@ abstract class MetaBase : Meta { override fun hashCode(): Int = items.hashCode() - override fun toString(): String = PRETTY_JSON.stringify(MetaSerializer, this) + override fun toString(): String = JSON_PRETTY.stringify(MetaSerializer, this) } /** diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/serializationUtils.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/serializationUtils.kt index 0ac825b9..41c33c7c 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/serializationUtils.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/serializationUtils.kt @@ -66,4 +66,5 @@ inline fun Encoder.encodeStructure( encoder.endStructure(desc) } -val PRETTY_JSON = Json(JsonConfiguration(prettyPrint = true, useArrayPolymorphism = true)) \ No newline at end of file +val JSON_PRETTY = Json(JsonConfiguration(prettyPrint = true, useArrayPolymorphism = true)) +val JSON_PLAIN = Json(JsonConfiguration(prettyPrint = true, useArrayPolymorphism = true)) \ No newline at end of file diff --git a/dataforge-meta/src/jsTest/kotlin/hep/dataforge/meta/DynamicMetaTest.kt b/dataforge-meta/src/jsTest/kotlin/hep/dataforge/meta/DynamicMetaTest.kt index 71eca4f5..b546ae6d 100644 --- a/dataforge-meta/src/jsTest/kotlin/hep/dataforge/meta/DynamicMetaTest.kt +++ b/dataforge-meta/src/jsTest/kotlin/hep/dataforge/meta/DynamicMetaTest.kt @@ -26,7 +26,7 @@ class DynamicMetaTest { @Test fun testMetaToDynamic(){ - val meta = buildMeta { + val meta = Meta { "a" put 22 "array" put listOf(1, 2, 3) "b" put "myString" diff --git a/dataforge-output/src/commonMain/kotlin/hep/dataforge/output/OutputManager.kt b/dataforge-output/src/commonMain/kotlin/hep/dataforge/output/OutputManager.kt index e88b29a5..a8b1697d 100644 --- a/dataforge-output/src/commonMain/kotlin/hep/dataforge/output/OutputManager.kt +++ b/dataforge-output/src/commonMain/kotlin/hep/dataforge/output/OutputManager.kt @@ -2,7 +2,6 @@ package hep.dataforge.output import hep.dataforge.context.* import hep.dataforge.context.PluginTag.Companion.DATAFORGE_GROUP -import hep.dataforge.meta.EmptyMeta import hep.dataforge.meta.Meta import hep.dataforge.names.Name import kotlinx.coroutines.CoroutineDispatcher @@ -24,7 +23,7 @@ interface OutputManager { type: KClass, name: Name, stage: Name = Name.EMPTY, - meta: Meta = EmptyMeta + meta: Meta = Meta.EMPTY ): Renderer } @@ -39,7 +38,7 @@ val Context.output: OutputManager get() = plugins.get() ?: ConsoleOutputManager( inline operator fun OutputManager.get( name: Name, stage: Name = Name.EMPTY, - meta: Meta = EmptyMeta + meta: Meta = Meta.EMPTY ): Renderer { return get(T::class, name, stage, meta) } @@ -47,7 +46,7 @@ inline operator fun OutputManager.get( /** * Directly render an object using the most suitable renderer */ -fun OutputManager.render(obj: Any, name: Name, stage: Name = Name.EMPTY, meta: Meta = EmptyMeta) = +fun OutputManager.render(obj: Any, name: Name, stage: Name = Name.EMPTY, meta: Meta = Meta.EMPTY) = get(obj::class, name, stage).render(obj, meta) /** diff --git a/dataforge-output/src/commonMain/kotlin/hep/dataforge/output/Renderer.kt b/dataforge-output/src/commonMain/kotlin/hep/dataforge/output/Renderer.kt index c1bcf6e5..9f8009d1 100644 --- a/dataforge-output/src/commonMain/kotlin/hep/dataforge/output/Renderer.kt +++ b/dataforge-output/src/commonMain/kotlin/hep/dataforge/output/Renderer.kt @@ -1,7 +1,6 @@ package hep.dataforge.output import hep.dataforge.context.ContextAware -import hep.dataforge.meta.EmptyMeta import hep.dataforge.meta.Meta /** @@ -18,5 +17,5 @@ interface Renderer : ContextAware { * By convention actual render is called in asynchronous mode, so this method should never * block execution */ - fun render(obj: T, meta: Meta = EmptyMeta) + fun render(obj: T, meta: Meta = Meta.EMPTY) } diff --git a/dataforge-tables/src/commonMain/kotlin/hep/dataforge/tables/io/textTableEnvelope.kt b/dataforge-tables/src/commonMain/kotlin/hep/dataforge/tables/io/textTableEnvelope.kt index 87f2e29d..e634346c 100644 --- a/dataforge-tables/src/commonMain/kotlin/hep/dataforge/tables/io/textTableEnvelope.kt +++ b/dataforge-tables/src/commonMain/kotlin/hep/dataforge/tables/io/textTableEnvelope.kt @@ -1,12 +1,11 @@ package hep.dataforge.tables.io -import hep.dataforge.io.Binary -import hep.dataforge.io.EmptyBinary import hep.dataforge.io.Envelope import hep.dataforge.meta.* import hep.dataforge.tables.SimpleColumnHeader import hep.dataforge.tables.Table import hep.dataforge.values.Value +import kotlinx.io.Binary import kotlinx.io.ByteArrayOutput import kotlinx.io.ExperimentalIoApi import kotlinx.io.asBinary @@ -39,5 +38,5 @@ fun TextRows.Companion.readEnvelope(envelope: Envelope): TextRows { .map { (_, item) -> SimpleColumnHeader(item.node["name"].string!!, Value::class, item.node["meta"].node ?: Meta.EMPTY) } - return TextRows(header, envelope.data ?: EmptyBinary) + return TextRows(header, envelope.data ?: Binary.EMPTY) } \ No newline at end of file diff --git a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskModel.kt b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskModel.kt index a811f428..2f7b6998 100644 --- a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskModel.kt +++ b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskModel.kt @@ -124,7 +124,7 @@ fun TaskDependencyContainer.allData(to: Name = Name.EMPTY) = AllDataDependency(t /** * A builder for [TaskModel] */ -class TaskModelBuilder(val name: Name, meta: Meta = EmptyMeta) : TaskDependencyContainer { +class TaskModelBuilder(val name: Name, meta: Meta = Meta.EMPTY) : TaskDependencyContainer { /** * Meta for current task. By default uses the whole input meta */ diff --git a/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/fileData.kt b/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/fileData.kt index 2b6d5454..6d16c37d 100644 --- a/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/fileData.kt +++ b/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/fileData.kt @@ -6,16 +6,19 @@ import hep.dataforge.io.* import hep.dataforge.meta.* import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.withContext +import kotlinx.io.asOutput import java.nio.file.FileSystem import java.nio.file.Files import java.nio.file.Path -import java.nio.file.StandardCopyOption +import java.nio.file.StandardOpenOption import java.nio.file.spi.FileSystemProvider +import java.util.zip.ZipEntry +import java.util.zip.ZipOutputStream import kotlin.reflect.KClass typealias FileFormatResolver = (Path, Meta) -> IOFormat -//private val zipFSProvider = ZipFileSystemProvider() + private fun newZFS(path: Path): FileSystem { val fsProvider = FileSystemProvider.installedProviders().find { it.scheme == "jar" } @@ -150,12 +153,43 @@ suspend fun IOPlugin.writeDataDirectory( } } + +private suspend fun ZipOutputStream.writeNode( + name: String, + item: DataItem, + dataFormat: IOFormat, + envelopeFormat: EnvelopeFormat = TaggedEnvelopeFormat +) { + withContext(Dispatchers.IO) { + when (item) { + is DataItem.Leaf -> { + //TODO add directory-based envelope writer + val envelope = item.data.toEnvelope(dataFormat) + val entry = ZipEntry(name) + putNextEntry(entry) + envelopeFormat.run { + asOutput().writeObject(envelope) + } + } + is DataItem.Node -> { + val entry = ZipEntry("$name/") + putNextEntry(entry) + closeEntry() + item.node.items.forEach { (token, item) -> + val childName = "$name/$token" + writeNode(childName, item, dataFormat, envelopeFormat) + } + } + } + } +} + +@DFExperimental suspend fun IOPlugin.writeZip( path: Path, node: DataNode, format: IOFormat, - envelopeFormat: EnvelopeFormat? = null, - metaFormat: MetaFormatFactory? = null + envelopeFormat: EnvelopeFormat = TaggedEnvelopeFormat ) { withContext(Dispatchers.IO) { val actualFile = if (path.toString().endsWith(".zip")) { @@ -163,21 +197,27 @@ suspend fun IOPlugin.writeZip( } else { path.resolveSibling(path.fileName.toString() + ".zip") } - if (Files.exists(actualFile) && Files.size(path) == 0.toLong()) { - Files.delete(path) - } - //Files.createFile(actualFile) - newZFS(actualFile).use { zipfs -> - val internalTargetPath = zipfs.getPath("/") - Files.createDirectories(internalTargetPath) - val tmp = Files.createTempDirectory("df_zip") - writeDataDirectory(tmp, node, format, envelopeFormat, metaFormat) - Files.list(tmp).forEach { sourcePath -> - val targetPath = sourcePath.fileName.toString() - val internalTargetPath = internalTargetPath.resolve(targetPath) - Files.copy(sourcePath, internalTargetPath, StandardCopyOption.REPLACE_EXISTING) - } + val fos = Files.newOutputStream(actualFile, StandardOpenOption.WRITE, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING) + val zos = ZipOutputStream(fos) + zos.use { + it.writeNode("", DataItem.Node(node), format, envelopeFormat) } + +// if (Files.exists(actualFile) && Files.size(path) == 0.toLong()) { +// Files.delete(path) +// } +// //Files.createFile(actualFile) +// newZFS(actualFile).use { zipfs -> +// val zipRootPath = zipfs.getPath("/") +// Files.createDirectories(zipRootPath) +// val tmp = Files.createTempDirectory("df_zip") +// writeDataDirectory(tmp, node, format, envelopeFormat, metaFormat) +// Files.list(tmp).forEach { sourcePath -> +// val targetPath = sourcePath.fileName.toString() +// val internalTargetPath = zipRootPath.resolve(targetPath) +// Files.copy(sourcePath, internalTargetPath, StandardCopyOption.REPLACE_EXISTING) +// } +// } } } diff --git a/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/FileDataTest.kt b/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/FileDataTest.kt index 4fc9e9a4..6c9537cc 100644 --- a/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/FileDataTest.kt +++ b/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/FileDataTest.kt @@ -5,16 +5,17 @@ import hep.dataforge.data.* import hep.dataforge.io.IOFormat import hep.dataforge.io.io import hep.dataforge.meta.DFExperimental +import hep.dataforge.meta.Meta import kotlinx.coroutines.runBlocking import kotlinx.io.Input import kotlinx.io.Output import kotlinx.io.text.readUtf8String import kotlinx.io.text.writeUtf8String import java.nio.file.Files -import kotlin.test.Ignore import kotlin.test.Test import kotlin.test.assertEquals + class FileDataTest { val dataNode = DataNode { node("dir") { @@ -37,6 +38,10 @@ class FileDataTest { return readUtf8String() } + override fun toMeta(): Meta = Meta { + IOFormat.NAME_KEY put "string" + } + } @Test @@ -56,7 +61,7 @@ class FileDataTest { @Test - @Ignore + @DFExperimental fun testZipWriteRead() { Global.io.run { val zip = Files.createTempFile("df_data_node", ".zip")