From 0fc219883249d689e149bcbc15a53efddaa977f3 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Mon, 16 May 2022 18:57:48 +0300 Subject: [PATCH] - Remove all unnecessary properties for `IOFormat` - Separate interfaces for `IOReader` and `IOWriter` --- CHANGELOG.md | 2 + build.gradle.kts | 8 ++ .../kscience/dataforge/provider/Provider.kt | 6 +- .../io/yaml/FrontMatterEnvelopeFormat.kt | 8 -- .../dataforge/io/yaml/YamlMetaFormat.kt | 8 -- .../kscience/dataforge/io/EnvelopeFormat.kt | 2 - .../kscience/dataforge/io/EnvelopeParts.kt | 21 +++- .../space/kscience/dataforge/io/IOFormat.kt | 107 +++++------------- .../space/kscience/dataforge/io/IOPlugin.kt | 10 +- .../kscience/dataforge/io/JsonMetaFormat.kt | 9 -- .../space/kscience/dataforge/io/MetaFormat.kt | 1 - .../dataforge/io/TaggedEnvelopeFormat.kt | 15 +-- .../dataforge/io/TaglessEnvelopeFormat.kt | 17 +-- .../space/kscience/dataforge/io/ioMisc.kt | 2 +- .../kscience/dataforge/io/MultipartTest.kt | 23 ++-- .../space/kscience/dataforge/io/fileIO.kt | 45 +++++--- .../dataforge/workspace/envelopeData.kt | 11 +- .../kscience/dataforge/workspace/fileData.kt | 85 ++++---------- .../dataforge/workspace/FileDataTest.kt | 21 +--- gradle.properties | 2 +- 20 files changed, 142 insertions(+), 261 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 17b972cc..95d5c0d5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,8 @@ - `ActiveDataSet` renamed to `DataSource` - `selectOne`->`getByType` - Data traversal in `DataSet` is done via iterator +- Remove all unnecessary properties for `IOFormat` +- Separate interfaces for `IOReader` and `IOWriter` ### Deprecated diff --git a/build.gradle.kts b/build.gradle.kts index a7560698..765e242f 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -1,3 +1,5 @@ +import org.jetbrains.kotlin.gradle.tasks.KotlinCompile + plugins { id("ru.mipt.npm.gradle.project") } @@ -5,6 +7,12 @@ plugins { allprojects { group = "space.kscience" version = "0.6.0-dev-7" + + tasks.withType{ + kotlinOptions{ + freeCompilerArgs = freeCompilerArgs + "-Xcontext-receivers" + } + } } subprojects { diff --git a/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/provider/Provider.kt b/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/provider/Provider.kt index e1940958..c91c2e4f 100644 --- a/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/provider/Provider.kt +++ b/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/provider/Provider.kt @@ -75,10 +75,8 @@ public inline fun Provider.provide(path: String, targetOverrid /** * Typed top level content */ -public fun Provider.top(target: String, type: KClass): Map { - return content(target).mapValues { - type.safeCast(it.value) ?: error("The type of element $it is ${it::class} but $type is expected") - } +public fun Provider.top(target: String, type: KClass): Map = content(target).mapValues { + type.safeCast(it.value) ?: error("The type of element ${it.value} is ${it.value::class} but $type is expected") } /** diff --git a/dataforge-io/dataforge-io-yaml/src/commonMain/kotlin/space/kscience/dataforge/io/yaml/FrontMatterEnvelopeFormat.kt b/dataforge-io/dataforge-io-yaml/src/commonMain/kotlin/space/kscience/dataforge/io/yaml/FrontMatterEnvelopeFormat.kt index 7f705a71..244284c8 100644 --- a/dataforge-io/dataforge-io-yaml/src/commonMain/kotlin/space/kscience/dataforge/io/yaml/FrontMatterEnvelopeFormat.kt +++ b/dataforge-io/dataforge-io-yaml/src/commonMain/kotlin/space/kscience/dataforge/io/yaml/FrontMatterEnvelopeFormat.kt @@ -7,10 +7,7 @@ import io.ktor.utils.io.core.readBytes import space.kscience.dataforge.context.Context import space.kscience.dataforge.context.Global import space.kscience.dataforge.io.* -import space.kscience.dataforge.io.IOFormat.Companion.META_KEY -import space.kscience.dataforge.io.IOFormat.Companion.NAME_KEY import space.kscience.dataforge.meta.Meta -import space.kscience.dataforge.misc.DFExperimental import space.kscience.dataforge.names.Name import space.kscience.dataforge.names.plus @@ -65,11 +62,6 @@ public class FrontMatterEnvelopeFormat( } } - override fun toMeta(): Meta = Meta { - NAME_KEY put name.toString() - META_KEY put meta - } - public companion object : EnvelopeFormatFactory { public const val SEPARATOR: String = "---" diff --git a/dataforge-io/dataforge-io-yaml/src/commonMain/kotlin/space/kscience/dataforge/io/yaml/YamlMetaFormat.kt b/dataforge-io/dataforge-io-yaml/src/commonMain/kotlin/space/kscience/dataforge/io/yaml/YamlMetaFormat.kt index b7c7c3e4..fffaa45b 100644 --- a/dataforge-io/dataforge-io-yaml/src/commonMain/kotlin/space/kscience/dataforge/io/yaml/YamlMetaFormat.kt +++ b/dataforge-io/dataforge-io-yaml/src/commonMain/kotlin/space/kscience/dataforge/io/yaml/YamlMetaFormat.kt @@ -4,8 +4,6 @@ import io.ktor.utils.io.core.Input import io.ktor.utils.io.core.Output import net.mamoe.yamlkt.* import space.kscience.dataforge.context.Context -import space.kscience.dataforge.io.IOFormat.Companion.META_KEY -import space.kscience.dataforge.io.IOFormat.Companion.NAME_KEY import space.kscience.dataforge.io.MetaFormat import space.kscience.dataforge.io.MetaFormatFactory import space.kscience.dataforge.io.readUtf8String @@ -14,7 +12,6 @@ import space.kscience.dataforge.meta.Meta import space.kscience.dataforge.meta.descriptors.MetaDescriptor import space.kscience.dataforge.meta.descriptors.get import space.kscience.dataforge.meta.isLeaf -import space.kscience.dataforge.misc.DFExperimental import space.kscience.dataforge.names.NameToken import space.kscience.dataforge.names.withIndex import space.kscience.dataforge.values.ListValue @@ -108,11 +105,6 @@ public class YamlMetaFormat(private val meta: Meta) : MetaFormat { return yaml.toMeta() } - override fun toMeta(): Meta = Meta { - NAME_KEY put FrontMatterEnvelopeFormat.name.toString() - META_KEY put meta - } - public companion object : MetaFormatFactory { override fun build(context: Context, meta: Meta): MetaFormat = YamlMetaFormat(meta) diff --git a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/EnvelopeFormat.kt b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/EnvelopeFormat.kt index 4dea572b..b705a3c0 100644 --- a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/EnvelopeFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/EnvelopeFormat.kt @@ -17,8 +17,6 @@ import kotlin.reflect.typeOf public data class PartialEnvelope(val meta: Meta, val dataOffset: Int, val dataSize: ULong?) public interface EnvelopeFormat : IOFormat { - override val type: KType get() = typeOf() - public val defaultMetaFormat: MetaFormatFactory get() = JsonMetaFormat public fun readPartial(input: Input): PartialEnvelope diff --git a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/EnvelopeParts.kt b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/EnvelopeParts.kt index 641247c3..72e9dfd4 100644 --- a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/EnvelopeParts.kt +++ b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/EnvelopeParts.kt @@ -1,5 +1,6 @@ package space.kscience.dataforge.io +import space.kscience.dataforge.context.invoke import space.kscience.dataforge.io.Envelope.Companion.ENVELOPE_NODE_KEY import space.kscience.dataforge.io.PartDescriptor.Companion.DEFAULT_MULTIPART_DATA_SEPARATOR import space.kscience.dataforge.io.PartDescriptor.Companion.MULTIPART_DATA_TYPE @@ -35,7 +36,7 @@ public typealias EnvelopeParts = List public fun EnvelopeBuilder.multipart( parts: EnvelopeParts, - separator: String = DEFAULT_MULTIPART_DATA_SEPARATOR + separator: String = DEFAULT_MULTIPART_DATA_SEPARATOR, ) { dataType = MULTIPART_DATA_TYPE @@ -67,17 +68,25 @@ public fun EnvelopeBuilder.multipart( } } +/** + * Put a list of envelopes as parts of given envelope + */ public fun EnvelopeBuilder.envelopes( envelopes: List, - format: EnvelopeFormat = TaggedEnvelopeFormat, - separator: String = DEFAULT_MULTIPART_DATA_SEPARATOR + formatFactory: EnvelopeFormatFactory = TaggedEnvelopeFormat, + formatMeta: Meta? = null, + separator: String = DEFAULT_MULTIPART_DATA_SEPARATOR, ) { val parts = envelopes.map { - val binary = format.toBinary(it) + val format = formatMeta?.let { formatFactory(formatMeta) } ?: formatFactory + val binary = Binary(it, format) EnvelopePart(binary, null) } - meta{ - set(MULTIPART_KEY + PART_FORMAT_KEY, format.toMeta()) + meta { + (MULTIPART_KEY + PART_FORMAT_KEY) put { + IOFormatFactory.NAME_KEY put formatFactory.name.toString() + formatMeta?.let { IOFormatFactory.META_KEY put formatMeta } + } } multipart(parts, separator) } diff --git a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/IOFormat.kt b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/IOFormat.kt index 7c3f8988..985a1e69 100644 --- a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/IOFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/IOFormat.kt @@ -1,12 +1,13 @@ package space.kscience.dataforge.io -import io.ktor.utils.io.core.* +import io.ktor.utils.io.core.Input +import io.ktor.utils.io.core.Output +import io.ktor.utils.io.core.readDouble +import io.ktor.utils.io.core.writeDouble import space.kscience.dataforge.context.Context import space.kscience.dataforge.context.Factory -import space.kscience.dataforge.io.IOFormat.Companion.NAME_KEY import space.kscience.dataforge.io.IOFormatFactory.Companion.IO_FORMAT_TYPE import space.kscience.dataforge.meta.Meta -import space.kscience.dataforge.meta.MetaRepr import space.kscience.dataforge.misc.Named import space.kscience.dataforge.misc.Type import space.kscience.dataforge.names.Name @@ -14,92 +15,53 @@ import space.kscience.dataforge.names.asName import kotlin.reflect.KType import kotlin.reflect.typeOf +public fun interface IOReader { + + public fun readObject(input: Input): T +} + +public fun interface IOWriter { + public fun writeObject(output: Output, obj: T) +} + + /** * And interface for reading and writing objects into with IO streams */ -public interface IOFormat : MetaRepr { - public val type: KType +public interface IOFormat : IOReader, IOWriter - public fun writeObject(output: Output, obj: T) - public fun readObject(input: Input): T +public fun Input.readObject(format: IOReader): T = format.readObject(this@readObject) - public companion object { - public val NAME_KEY: Name = "name".asName() - public val META_KEY: Name = "meta".asName() - } -} - -public fun Input.readWith(format: IOFormat): T = format.readObject(this@readWith) - -public fun IOFormat.readObject(binary: Binary): T = binary.read { +public fun IOFormat.readObjectFrom(binary: Binary): T = binary.read { readObject(this) } /** * Read given binary as object using given format */ -public fun Binary.readWith(format: IOFormat): T = read { - readWith(format) +public fun Binary.readWith(format: IOReader): T = read { + readObject(format) } -public fun Output.writeWith(format: IOFormat, obj: T): Unit = - format.run { writeObject(this@writeWith, obj) } +public fun Output.writeObject(format: IOWriter, obj: T): Unit = + format.run { writeObject(this@writeObject, obj) } -public inline fun IOFormat.Companion.listOf( - format: IOFormat, -): IOFormat> = object : IOFormat> { - override val type: KType = typeOf>() - - override fun writeObject(output: Output, obj: List) { - output.writeInt(obj.size) - format.run { - obj.forEach { - writeObject(output, it) - } - } - } - - override fun readObject(input: Input): List { - val size = input.readInt() - return format.run { - List(size) { readObject(input) } - } - } - - override fun toMeta(): Meta = Meta { - NAME_KEY put "list" - "contentFormat" put format.toMeta() - } - -} - -//public fun ObjectPool.fill(block: Buffer.() -> Unit): Buffer { -// val buffer = borrow() -// return try { -// buffer.apply(block) -// } catch (ex: Exception) { -// //recycle(buffer) -// throw ex -// } -//} @Type(IO_FORMAT_TYPE) -public interface IOFormatFactory : Factory>, Named, MetaRepr { +public interface IOFormatFactory : Factory>, Named { /** * Explicit type for dynamic type checks */ public val type: KType - override fun toMeta(): Meta = Meta { - NAME_KEY put name.toString() - } - public companion object { public const val IO_FORMAT_TYPE: String = "io.format" + public val NAME_KEY: Name = "name".asName() + public val META_KEY: Name = "meta".asName() } } -public fun IOFormat.toBinary(obj: T): Binary = Binary { writeObject(this, obj) } +public fun Binary(obj: T, format: IOWriter): Binary = Binary { format.writeObject(this, obj) } public object DoubleIOFormat : IOFormat, IOFormatFactory { override fun build(context: Context, meta: Meta): IOFormat = this @@ -113,21 +75,4 @@ public object DoubleIOFormat : IOFormat, IOFormatFactory { } override fun readObject(input: Input): Double = input.readDouble() -} - -//public object ValueIOFormat : IOFormat, IOFormatFactory { -// override fun invoke(meta: Meta, context: Context): IOFormat = this -// -// override val name: Name = "value".asName() -// -// override val type: KType get() = typeOf() -// -// override fun writeObject(output: Output, obj: Value) { -// BinaryMetaFormat.run { output.writeValue(obj) } -// } -// -// override fun readObject(input: Input): Value { -// return (BinaryMetaFormat.run { input.readMetaItem() } as? MetaItemValue)?.value -// ?: error("The item is not a value") -// } -//} \ No newline at end of file +} \ No newline at end of file diff --git a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/IOPlugin.kt b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/IOPlugin.kt index 9e1f0dd1..dc6541f2 100644 --- a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/IOPlugin.kt +++ b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/IOPlugin.kt @@ -2,8 +2,6 @@ package space.kscience.dataforge.io import space.kscience.dataforge.context.* import space.kscience.dataforge.io.EnvelopeFormatFactory.Companion.ENVELOPE_FORMAT_TYPE -import space.kscience.dataforge.io.IOFormat.Companion.META_KEY -import space.kscience.dataforge.io.IOFormat.Companion.NAME_KEY import space.kscience.dataforge.io.IOFormatFactory.Companion.IO_FORMAT_TYPE import space.kscience.dataforge.io.MetaFormatFactory.Companion.META_FORMAT_TYPE import space.kscience.dataforge.meta.Meta @@ -20,12 +18,12 @@ public class IOPlugin(meta: Meta) : AbstractPlugin(meta) { } public fun resolveIOFormat(item: Meta, type: KClass): IOFormat? { - val key = item.string ?: item[NAME_KEY]?.string ?: error("Format name not defined") + val key = item.string ?: item[IOFormatFactory.NAME_KEY]?.string ?: error("Format name not defined") val name = Name.parse(key) return ioFormatFactories.find { it.name == name }?.let { @Suppress("UNCHECKED_CAST") if (it.type != type) error("Format type ${it.type} is not the same as requested type $type") - else it.build(context, item[META_KEY] ?: Meta.EMPTY) as IOFormat + else it.build(context, item[IOFormatFactory.META_KEY] ?: Meta.EMPTY) as IOFormat } } @@ -47,8 +45,8 @@ public class IOPlugin(meta: Meta) : AbstractPlugin(meta) { envelopeFormatFactories.find { it.name == name }?.build(context, meta) public fun resolveEnvelopeFormat(item: Meta): EnvelopeFormat? { - val name = item.string ?: item[NAME_KEY]?.string ?: error("Envelope format name not defined") - val meta = item[META_KEY] ?: Meta.EMPTY + val name = item.string ?: item[IOFormatFactory.NAME_KEY]?.string ?: error("Envelope format name not defined") + val meta = item[IOFormatFactory.META_KEY] ?: Meta.EMPTY return resolveEnvelopeFormat(Name.parse(name), meta) } diff --git a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/JsonMetaFormat.kt b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/JsonMetaFormat.kt index d5beee22..4bb3c82d 100644 --- a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/JsonMetaFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/JsonMetaFormat.kt @@ -8,30 +8,21 @@ import io.ktor.utils.io.core.Output import kotlinx.serialization.json.Json import kotlinx.serialization.json.JsonObject import space.kscience.dataforge.context.Context -import space.kscience.dataforge.io.IOFormat.Companion.NAME_KEY import space.kscience.dataforge.meta.Meta import space.kscience.dataforge.meta.descriptors.MetaDescriptor import space.kscience.dataforge.meta.toJson import space.kscience.dataforge.meta.toMeta -import kotlin.reflect.KType -import kotlin.reflect.typeOf /** * A Json format for Meta representation */ public class JsonMetaFormat(private val json: Json = DEFAULT_JSON) : MetaFormat { - override val type: KType get() = typeOf() - override fun writeMeta(output: Output, meta: Meta, descriptor: MetaDescriptor?) { val jsonObject = meta.toJson(descriptor) output.writeUtf8String(json.encodeToString(JsonObject.serializer(), jsonObject)) } - override fun toMeta(): Meta = Meta { - NAME_KEY put name.toString() - } - override fun readMeta(input: Input, descriptor: MetaDescriptor?): Meta { val str = input.readUtf8String()//readByteArray().decodeToString() val jsonElement = json.parseToJsonElement(str) diff --git a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/MetaFormat.kt b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/MetaFormat.kt index 46d7b540..5e009a12 100644 --- a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/MetaFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/MetaFormat.kt @@ -20,7 +20,6 @@ import kotlin.reflect.typeOf * A format for meta serialization */ public interface MetaFormat : IOFormat { - override val type: KType get() = typeOf() override fun writeObject(output: Output, obj: Meta) { writeMeta(output, obj, null) diff --git a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/TaggedEnvelopeFormat.kt b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/TaggedEnvelopeFormat.kt index 933dc343..2e5d3cc6 100644 --- a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/TaggedEnvelopeFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/TaggedEnvelopeFormat.kt @@ -3,8 +3,6 @@ package space.kscience.dataforge.io import io.ktor.utils.io.core.* import space.kscience.dataforge.context.Context import space.kscience.dataforge.context.Global -import space.kscience.dataforge.io.IOFormat.Companion.META_KEY -import space.kscience.dataforge.io.IOFormat.Companion.NAME_KEY import space.kscience.dataforge.meta.Meta import space.kscience.dataforge.meta.enum import space.kscience.dataforge.meta.get @@ -49,7 +47,7 @@ public class TaggedEnvelopeFormat( formatMeta: Meta, ) { val metaFormat = metaFormatFactory.build(this@TaggedEnvelopeFormat.io.context, formatMeta) - val metaBytes = metaFormat.toBinary(envelope.meta) + val metaBytes = Binary(envelope.meta,metaFormat) val actualSize: ULong = (envelope.data?.size ?: 0).toULong() val tag = Tag(metaFormatFactory.key, metaBytes.size.toUInt() + 2u, actualSize) output.writeBinary(tag.toBinary()) @@ -74,7 +72,7 @@ public class TaggedEnvelopeFormat( val metaBinary = input.readBinary(tag.metaSize.toInt()) - val meta: Meta = metaFormat.readObject(metaBinary) + val meta: Meta = metaFormat.readObjectFrom(metaBinary) val data = input.readBinary(tag.dataSize.toInt()) @@ -89,7 +87,7 @@ public class TaggedEnvelopeFormat( val metaBinary = input.readBinary(tag.metaSize.toInt()) - val meta: Meta = metaFormat.readObject(metaBinary) + val meta: Meta = metaFormat.readObjectFrom(metaBinary) return PartialEnvelope(meta, (version.tagSize + tag.metaSize).toInt(), tag.dataSize) @@ -106,13 +104,6 @@ public class TaggedEnvelopeFormat( DF03(24u) } - override fun toMeta(): Meta = Meta { - NAME_KEY put name.toString() - META_KEY put { - "version" put version - } - } - public companion object : EnvelopeFormatFactory { private const val START_SEQUENCE = "#~" private const val END_SEQUENCE = "~#\r\n" diff --git a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/TaglessEnvelopeFormat.kt b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/TaglessEnvelopeFormat.kt index 9a0f4a98..4e2bacfc 100644 --- a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/TaglessEnvelopeFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/TaglessEnvelopeFormat.kt @@ -3,8 +3,6 @@ package space.kscience.dataforge.io import io.ktor.utils.io.core.* import space.kscience.dataforge.context.Context import space.kscience.dataforge.context.Global -import space.kscience.dataforge.io.IOFormat.Companion.META_KEY -import space.kscience.dataforge.io.IOFormat.Companion.NAME_KEY import space.kscience.dataforge.meta.Meta import space.kscience.dataforge.meta.get import space.kscience.dataforge.meta.isEmpty @@ -50,11 +48,11 @@ public class TaglessEnvelopeFormat( //Printing meta if (!envelope.meta.isEmpty()) { - val metaBytes = metaFormat.toBinary(envelope.meta) + val metaBinary = Binary(envelope.meta, metaFormat) output.writeProperty(META_LENGTH_PROPERTY, - metaBytes.size + 2) + metaBinary.size + 2) output.writeUtf8String(this.metaStart + "\r\n") - output.writeBinary(metaBytes) + output.writeBinary(metaBinary) output.writeRawString("\r\n") } @@ -102,7 +100,7 @@ public class TaglessEnvelopeFormat( val metaFormat = properties[META_TYPE_PROPERTY]?.let { io.resolveMetaFormat(it) } ?: JsonMetaFormat val metaSize = properties[META_LENGTH_PROPERTY]?.toInt() meta = if (metaSize != null) { - metaFormat.readObject(input.readBinary(metaSize)) + metaFormat.readObjectFrom(input.readBinary(metaSize)) } else { error("Can't partially read an envelope with undefined meta size") } @@ -170,7 +168,7 @@ public class TaglessEnvelopeFormat( val metaSize = properties[META_LENGTH_PROPERTY]?.toInt() meta = if (metaSize != null) { offset += metaSize - metaFormat.readObject(input.readBinary(metaSize)) + metaFormat.readObjectFrom(input.readBinary(metaSize)) } else { error("Can't partially read an envelope with undefined meta size") } @@ -187,11 +185,6 @@ public class TaglessEnvelopeFormat( return PartialEnvelope(meta, offset, dataSize) } - override fun toMeta(): Meta = Meta { - NAME_KEY put name.toString() - META_KEY put meta - } - public companion object : EnvelopeFormatFactory { private val propertyPattern = "#\\?\\s*([\\w.]*)\\s*:\\s*([^;]*);?".toRegex() diff --git a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/ioMisc.kt b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/ioMisc.kt index e72c0eed..5c17e9ac 100644 --- a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/ioMisc.kt +++ b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/ioMisc.kt @@ -183,7 +183,7 @@ public fun Input.discardWithSeparator( atMost: Int = Int.MAX_VALUE, skipUntilEndOfLine: Boolean = false, ): Int { - val dummy: Output = object :Output(ChunkBuffer.Pool){ + val dummy: Output = object : Output(ChunkBuffer.Pool) { override fun closeDestination() { // Do nothing } diff --git a/dataforge-io/src/commonTest/kotlin/space/kscience/dataforge/io/MultipartTest.kt b/dataforge-io/src/commonTest/kotlin/space/kscience/dataforge/io/MultipartTest.kt index 8ffaef14..6315d744 100644 --- a/dataforge-io/src/commonTest/kotlin/space/kscience/dataforge/io/MultipartTest.kt +++ b/dataforge-io/src/commonTest/kotlin/space/kscience/dataforge/io/MultipartTest.kt @@ -32,18 +32,17 @@ class MultipartTest { @Test fun testParts() { - TaglessEnvelopeFormat.run { - val singleEnvelopeData = toBinary(envelopes[0]) - val singleEnvelopeSize = singleEnvelopeData.size - val bytes = toBinary(partsEnvelope) - assertTrue(envelopes.size * singleEnvelopeSize < bytes.size) - val reconstructed = bytes.readWith(this) - println(reconstructed.meta) - val parts = reconstructed.parts() - val envelope = parts[2].envelope(io) - assertEquals(2, envelope.meta["value"].int) - println(reconstructed.data!!.size) - } + val format = TaglessEnvelopeFormat + val singleEnvelopeData = Binary(envelopes[0], format) + val singleEnvelopeSize = singleEnvelopeData.size + val bytes = Binary(partsEnvelope, format) + assertTrue(envelopes.size * singleEnvelopeSize < bytes.size) + val reconstructed = bytes.readWith(format) + println(reconstructed.meta) + val parts = reconstructed.parts() + val envelope = parts[2].envelope(io) + assertEquals(2, envelope.meta["value"].int) + println(reconstructed.data!!.size) } } \ No newline at end of file diff --git a/dataforge-io/src/jvmMain/kotlin/space/kscience/dataforge/io/fileIO.kt b/dataforge-io/src/jvmMain/kotlin/space/kscience/dataforge/io/fileIO.kt index 1f2d3041..64f4907e 100644 --- a/dataforge-io/src/jvmMain/kotlin/space/kscience/dataforge/io/fileIO.kt +++ b/dataforge-io/src/jvmMain/kotlin/space/kscience/dataforge/io/fileIO.kt @@ -97,7 +97,36 @@ public val IOPlugin.Companion.DATA_FILE_NAME: String get() = "@data" /** * Read file containing meta using given [formatOverride] or file extension to infer meta type. - * If [path] is a directory search for file starting with `meta` in it + * If [path] is a directory search for file starting with `meta` in it. + * + * Returns null if meta could not be resolved + */ +public fun IOPlugin.readMetaFileOrNull( + path: Path, + formatOverride: MetaFormat? = null, + descriptor: MetaDescriptor? = null, +): Meta? { + if (!Files.exists(path)) return null + + val actualPath: Path = if (Files.isDirectory(path)) { + Files.list(path).asSequence().find { it.fileName.startsWith(IOPlugin.META_FILE_NAME) } + ?: return null + } else { + path + } + val extension = actualPath.fileName.toString().substringAfterLast('.') + + val metaFormat = formatOverride ?: resolveMetaFormat(extension) ?: return null + return actualPath.read { + metaFormat.readMeta(this, descriptor) + } +} + +/** + * Read file containing meta using given [formatOverride] or file extension to infer meta type. + * If [path] is a directory search for file starting with `meta` in it. + * + * Fails if nothing works. */ public fun IOPlugin.readMetaFile( path: Path, @@ -120,6 +149,7 @@ public fun IOPlugin.readMetaFile( } } + /** * Write meta to file using [metaFormat]. If [path] is a directory, write a file with name equals name of [metaFormat]. * Like "meta.json" @@ -196,22 +226,11 @@ public fun IOPlugin.readEnvelopeFile( return SimpleEnvelope(meta, data) } - return formatPicker(path)?.let { format -> - format.readFile(path) - } ?: if (readNonEnvelopes) { // if no format accepts file, read it as binary + return formatPicker(path)?.readFile(path) ?: if (readNonEnvelopes) { // if no format accepts file, read it as binary SimpleEnvelope(Meta.EMPTY, path.asBinary()) } else error("Can't infer format for file $path") } -/** - * Write a binary into file. Throws an error if file already exists - */ -public fun IOFormat.writeToFile(path: Path, obj: T) { - path.write { - writeObject(this, obj) - } -} - /** * Write envelope file to given [path] using [envelopeFormat] and optional [metaFormat] */ diff --git a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/envelopeData.kt b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/envelopeData.kt index 55c98352..ce5be133 100644 --- a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/envelopeData.kt +++ b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/envelopeData.kt @@ -4,19 +4,18 @@ import space.kscience.dataforge.data.Data import space.kscience.dataforge.data.await import space.kscience.dataforge.io.* import space.kscience.dataforge.misc.DFInternal +import kotlin.reflect.typeOf /** * Convert an [Envelope] to a data via given format. The actual parsing is done lazily. */ @OptIn(DFInternal::class) -public fun Envelope.toData(format: IOFormat): Data { - return Data(format.type, meta) { - data?.readWith(format) ?: error("Can't convert envelope without data to Data") - } +public inline fun Envelope.toData(format: IOReader): Data = Data(typeOf(), meta) { + data?.readWith(format) ?: error("Can't convert envelope without data to Data") } -public suspend fun Data.toEnvelope(format: IOFormat): Envelope { +public suspend fun Data.toEnvelope(format: IOWriter): Envelope { val obj = await() - val binary = format.toBinary(obj) + val binary = Binary(obj, format) return SimpleEnvelope(meta, binary) } \ No newline at end of file diff --git a/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/fileData.kt b/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/fileData.kt index d758c45c..aa3ddede 100644 --- a/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/fileData.kt +++ b/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/fileData.kt @@ -2,11 +2,12 @@ package space.kscience.dataforge.workspace import io.ktor.utils.io.streams.asOutput import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.runBlocking import kotlinx.coroutines.withContext import space.kscience.dataforge.data.* import space.kscience.dataforge.io.* -import space.kscience.dataforge.meta.* +import space.kscience.dataforge.meta.Meta +import space.kscience.dataforge.meta.get +import space.kscience.dataforge.meta.string import space.kscience.dataforge.misc.DFExperimental import java.nio.file.FileSystem import java.nio.file.Files @@ -15,28 +16,13 @@ import java.nio.file.StandardOpenOption import java.nio.file.spi.FileSystemProvider import java.util.zip.ZipEntry import java.util.zip.ZipOutputStream -import kotlin.reflect.KType -import kotlin.reflect.typeOf import kotlin.streams.toList //public typealias FileFormatResolver = (Path, Meta) -> IOFormat -public interface FileFormatResolver { - public val type: KType - public operator fun invoke(path: Path, meta: Meta): IOFormat -} +public typealias FileFormatResolver = (path: Path, meta: Meta) -> IOReader -@PublishedApi -internal inline fun IOPlugin.formatResolver(): FileFormatResolver = - object : FileFormatResolver { - override val type: KType = typeOf() - - @OptIn(DFExperimental::class) - override fun invoke(path: Path, meta: Meta): IOFormat = - resolveIOFormat() ?: error("Can't resolve IO format for ${T::class}") - } - private fun newZFS(path: Path): FileSystem { val fsProvider = FileSystemProvider.installedProviders().find { it.scheme == "jar" } ?: error("Zip file system provider not found") @@ -46,14 +32,9 @@ private fun newZFS(path: Path): FileSystem { /** * Read data with supported envelope format and binary format. If envelope format is null, then read binary directly from file. * The operation is blocking since it must read meta header. The reading of envelope body is lazy - * @param type explicit type of data read - * @param dataFormat binary format - * @param envelopeFormat the format of envelope. If null, file is read directly - * @param metaFile the relative file for optional meta override - * @param metaFileFormat the meta format for override */ @DFExperimental -public fun IOPlugin.readDataFile( +public inline fun IOPlugin.readDataFile( path: Path, formatResolver: FileFormatResolver, ): Data { @@ -62,34 +43,26 @@ public fun IOPlugin.readDataFile( return envelope.toData(format) } -@DFExperimental -public inline fun IOPlugin.readDataFile(path: Path): Data = readDataFile(path, formatResolver()) - /** * Add file/directory-based data tree item */ -@DFExperimental -public suspend fun DataSetBuilder.file( - plugin: IOPlugin, +context(IOPlugin) @DFExperimental +public fun DataSetBuilder.file( path: Path, - formatResolver: FileFormatResolver, + formatResolver: FileFormatResolver, ) { //If path is a single file or a special directory, read it as single datum if (!Files.isDirectory(path) || Files.list(path).allMatch { it.fileName.toString().startsWith("@") }) { - plugin.run { - val data = readDataFile(path, formatResolver) - val name = data.meta[Envelope.ENVELOPE_NAME_KEY].string - ?: path.fileName.toString().replace(".df", "") - data(name, data) - } + val data = readDataFile(path, formatResolver) + val name = data.meta[Envelope.ENVELOPE_NAME_KEY].string + ?: path.fileName.toString().replace(".df", "") + data(name, data) } else { //otherwise, read as directory - plugin.run { - val data = readDataDirectory(path, formatResolver) - val name = data.meta[Envelope.ENVELOPE_NAME_KEY].string - ?: path.fileName.toString().replace(".df", "") - node(name, data) - } + val data = readDataDirectory(path, formatResolver) + val name = data.meta[Envelope.ENVELOPE_NAME_KEY].string + ?: path.fileName.toString().replace(".df", "") + node(name, data) } } @@ -97,10 +70,10 @@ public suspend fun DataSetBuilder.file( * Read the directory as a data node. If [path] is a zip archive, read it as directory */ @DFExperimental -public suspend fun IOPlugin.readDataDirectory( +public fun IOPlugin.readDataDirectory( path: Path, - formatResolver: FileFormatResolver, -): DataTree { + formatResolver: FileFormatResolver, +): DataTree { //read zipped data node if (path.fileName != null && path.fileName.toString().endsWith(".zip")) { //Using explicit Zip file system to avoid bizarre compatibility bugs @@ -108,24 +81,18 @@ public suspend fun IOPlugin.readDataDirectory( return readDataDirectory(fs.rootDirectories.first(), formatResolver) } if (!Files.isDirectory(path)) error("Provided path $path is not a directory") - return DataTree(formatResolver.type) { + return DataTree { Files.list(path).toList().forEach { path -> val fileName = path.fileName.toString() if (fileName.startsWith(IOPlugin.META_FILE_NAME)) { meta(readMetaFile(path)) } else if (!fileName.startsWith("@")) { - runBlocking { - file(this@readDataDirectory, path, formatResolver) - } + file(path, formatResolver) } } } } -@DFExperimental -public suspend inline fun IOPlugin.readDataDirectory(path: Path): DataTree = - readDataDirectory(path, formatResolver()) - /** * Write data tree to existing directory or create a new one using default [java.nio.file.FileSystem] provider */ @@ -133,7 +100,7 @@ public suspend inline fun IOPlugin.readDataDirectory(path: Pat public suspend fun IOPlugin.writeDataDirectory( path: Path, tree: DataTree, - format: IOFormat, + format: IOWriter, envelopeFormat: EnvelopeFormat? = null, metaFormat: MetaFormatFactory? = null, ) { @@ -179,11 +146,9 @@ private suspend fun ZipOutputStream.writeNode( val envelope = treeItem.data.toEnvelope(dataFormat) val entry = ZipEntry(name) putNextEntry(entry) - envelopeFormat.run { - asOutput().run { - writeEnvelope(this, envelope) - flush() - } + asOutput().run { + envelopeFormat.writeEnvelope(this, envelope) + flush() } } is DataTreeItem.Node -> { diff --git a/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/FileDataTest.kt b/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/FileDataTest.kt index d1bbb606..a47a648d 100644 --- a/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/FileDataTest.kt +++ b/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/FileDataTest.kt @@ -9,12 +9,8 @@ import space.kscience.dataforge.io.IOFormat import space.kscience.dataforge.io.io import space.kscience.dataforge.io.readUtf8String import space.kscience.dataforge.io.writeUtf8String -import space.kscience.dataforge.meta.Meta import space.kscience.dataforge.misc.DFExperimental import java.nio.file.Files -import java.nio.file.Path -import kotlin.reflect.KType -import kotlin.reflect.typeOf import kotlin.test.Test import kotlin.test.assertEquals @@ -35,8 +31,6 @@ class FileDataTest { object StringIOFormat : IOFormat { - override val type: KType = typeOf() - override fun writeObject(output: Output, obj: String) { output.writeUtf8String(obj) } @@ -45,17 +39,6 @@ class FileDataTest { return input.readUtf8String() } - override fun toMeta(): Meta = Meta { - IOFormat.NAME_KEY put "string" - } - - } - - object StringFormatResolver : FileFormatResolver { - override val type: KType = typeOf() - - override fun invoke(path: Path, meta: Meta): IOFormat = StringIOFormat - } @Test @@ -66,7 +49,7 @@ class FileDataTest { runBlocking { writeDataDirectory(dir, dataNode, StringIOFormat) println(dir.toUri().toString()) - val reconstructed = readDataDirectory(dir, StringFormatResolver) + val reconstructed = readDataDirectory(dir) { _, _ -> StringIOFormat } assertEquals(dataNode["dir.a"]?.meta, reconstructed["dir.a"]?.meta) assertEquals(dataNode["b"]?.await(), reconstructed["b"]?.await()) } @@ -82,7 +65,7 @@ class FileDataTest { runBlocking { writeZip(zip, dataNode, StringIOFormat) println(zip.toUri().toString()) - val reconstructed = readDataDirectory(zip, StringFormatResolver) + val reconstructed = readDataDirectory(zip) { _, _ -> StringIOFormat } assertEquals(dataNode["dir.a"]?.meta, reconstructed["dir.a"]?.meta) assertEquals(dataNode["b"]?.await(), reconstructed["b"]?.await()) } diff --git a/gradle.properties b/gradle.properties index 0de6ab36..1aeadb9b 100644 --- a/gradle.properties +++ b/gradle.properties @@ -4,4 +4,4 @@ kotlin.code.style=official kotlin.mpp.stability.nowarn=true -toolsVersion=0.11.4-kotlin-1.6.20 +toolsVersion=0.11.5-kotlin-1.6.21