diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/EnvelopeParts.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/EnvelopeParts.kt index eb1dd696..d1b86195 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/EnvelopeParts.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/EnvelopeParts.kt @@ -3,6 +3,7 @@ package hep.dataforge.io import hep.dataforge.context.Global import hep.dataforge.io.EnvelopeParts.FORMAT_META_KEY import hep.dataforge.io.EnvelopeParts.FORMAT_NAME_KEY +import hep.dataforge.io.EnvelopeParts.INDEX_KEY import hep.dataforge.io.EnvelopeParts.MULTIPART_DATA_TYPE import hep.dataforge.io.EnvelopeParts.SIZE_KEY import hep.dataforge.meta.* @@ -13,6 +14,7 @@ import hep.dataforge.names.toName object EnvelopeParts { val MULTIPART_KEY = "multipart".asName() val SIZE_KEY = Envelope.ENVELOPE_NODE_KEY + MULTIPART_KEY + "size" + val INDEX_KEY = Envelope.ENVELOPE_NODE_KEY + MULTIPART_KEY + "index" val FORMAT_NAME_KEY = Envelope.ENVELOPE_NODE_KEY + MULTIPART_KEY + "format" val FORMAT_META_KEY = Envelope.ENVELOPE_NODE_KEY + MULTIPART_KEY + "meta" @@ -37,8 +39,37 @@ fun EnvelopeBuilder.multipart(format: EnvelopeFormatFactory, envelopes: Collecti } } -fun EnvelopeBuilder.multipart(formatFactory: EnvelopeFormatFactory, builder: suspend SequenceScope.() -> Unit) = - multipart(formatFactory, sequence(builder).toList()) +/** + * Create a multipart partition in the envelope adding additional name-index mapping in meta + */ +@DFExperimental +fun EnvelopeBuilder.multipart(format: EnvelopeFormatFactory, envelopes: Map) { + dataType = MULTIPART_DATA_TYPE + meta { + SIZE_KEY put envelopes.size + FORMAT_NAME_KEY put format.name.toString() + } + data { + format.run { + var counter = 0 + envelopes.forEach {(key, envelope)-> + writeObject(envelope) + meta{ + append(INDEX_KEY, buildMeta { + "key" put key + "index" put counter + }) + } + counter++ + } + } + } +} + +fun EnvelopeBuilder.multipart( + formatFactory: EnvelopeFormatFactory, + builder: suspend SequenceScope.() -> Unit +) = multipart(formatFactory, sequence(builder).toList()) /** * If given envelope supports multipart data, return a sequence of those parts (could be empty). Otherwise return null. diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOPlugin.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOPlugin.kt index ae88e4f4..a463a053 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOPlugin.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOPlugin.kt @@ -52,7 +52,7 @@ class IOPlugin(meta: Meta) : AbstractPlugin(meta) { companion object : PluginFactory { val defaultMetaFormats: List = listOf(JsonMetaFormat, BinaryMetaFormat) - val defaultEnvelopeFormats = listOf(TaggedEnvelopeFormat) + val defaultEnvelopeFormats = listOf(TaggedEnvelopeFormat, TaglessEnvelopeFormat) override val tag: PluginTag = PluginTag("io", group = PluginTag.DATAFORGE_GROUP) diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaglessEnvelopeFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaglessEnvelopeFormat.kt index 5c4b7dae..1cc62a2b 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaglessEnvelopeFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaglessEnvelopeFormat.kt @@ -49,6 +49,7 @@ class TaglessEnvelopeFormat( writeText(dataStart + "\r\n") writeFully(data.toBytes()) } + flush() } override fun Input.readObject(): Envelope { @@ -191,7 +192,7 @@ class TaglessEnvelopeFormat( return try { val buffer = ByteArray(TAGLESS_ENVELOPE_HEADER.length) input.readFully(buffer) - return if (buffer.toString() == TAGLESS_ENVELOPE_HEADER) { + return if (String(buffer) == TAGLESS_ENVELOPE_HEADER) { TaglessEnvelopeFormat(io) } else { null diff --git a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/fileIO.kt b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/fileIO.kt index 14e4c077..eff9e705 100644 --- a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/fileIO.kt +++ b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/fileIO.kt @@ -58,6 +58,24 @@ fun IOPlugin.writeMetaFile( } } +/** + * Return inferred [EnvelopeFormat] if only one format could read given file. If no format accepts file, return null. If + * multiple formats accepts file, throw an error. + */ +fun IOPlugin.peekBinaryFormat(binary: Binary): EnvelopeFormat? { + val formats = envelopeFormatFactories.mapNotNull { factory -> + binary.read { + factory.peekFormat(this@peekBinaryFormat, this@read) + } + } + + return when (formats.size) { + 0 -> null + 1 -> formats.first() + else -> error("Envelope format binary recognition clash") + } +} + /** * Read and envelope from file if the file exists, return null if file does not exist. * @@ -72,7 +90,11 @@ fun IOPlugin.writeMetaFile( * Return null otherwise. */ @DFExperimental -fun IOPlugin.readEnvelopeFile(path: Path, readNonEnvelopes: Boolean = false): Envelope? { +fun IOPlugin.readEnvelopeFile( + path: Path, + readNonEnvelopes: Boolean = false, + formatPeeker: IOPlugin.(Binary) -> EnvelopeFormat? = IOPlugin::peekBinaryFormat +): Envelope? { if (!Files.exists(path)) return null //read two-files directory @@ -99,24 +121,13 @@ fun IOPlugin.readEnvelopeFile(path: Path, readNonEnvelopes: Boolean = false): En val binary = path.asBinary() - val formats = envelopeFormatFactories.mapNotNull { factory -> + return formatPeeker(binary)?.run { binary.read { - factory.peekFormat(this@readEnvelopeFile, this@read) + readObject() } - } - return when (formats.size) { - 0 -> if (readNonEnvelopes) { - SimpleEnvelope(Meta.empty, binary) - } else { - null - } - 1 -> formats.first().run { - binary.read { - readObject() - } - } - else -> error("Envelope format file recognition clash") - } + } ?: if (readNonEnvelopes) { // if no format accepts file, read it as binary + SimpleEnvelope(Meta.empty, binary) + } else null } fun IOPlugin.writeEnvelopeFile( diff --git a/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/FileEnvelopeTest.kt b/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/FileEnvelopeTest.kt index f4847cfd..b8bfafaa 100644 --- a/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/FileEnvelopeTest.kt +++ b/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/FileEnvelopeTest.kt @@ -22,10 +22,23 @@ class FileEnvelopeTest { @Test fun testFileWriteRead() { - val tmpPath = Files.createTempFile("dataforge_test", ".df") - Global.io.writeEnvelopeFile(tmpPath,envelope) - println(tmpPath.toUri()) - val restored: Envelope = Global.io.readEnvelopeFile(tmpPath)!! - assertTrue { envelope.contentEquals(restored) } + Global.io.run { + val tmpPath = Files.createTempFile("dataforge_test", ".df") + writeEnvelopeFile(tmpPath, envelope) + println(tmpPath.toUri()) + val restored: Envelope = readEnvelopeFile(tmpPath)!! + assertTrue { envelope.contentEquals(restored) } + } + } + + @Test + fun testFileWriteReadTagless() { + Global.io.run { + val tmpPath = Files.createTempFile("dataforge_test_tagless", ".df") + writeEnvelopeFile(tmpPath, envelope, format = TaglessEnvelopeFormat) + println(tmpPath.toUri()) + val restored: Envelope = readEnvelopeFile(tmpPath)!! + assertTrue { envelope.contentEquals(restored) } + } } } \ No newline at end of file