kscience/dataforge-core
7
0
Fork 0
Code Issues Pull Requests 1 Packages Projects Releases 12 Wiki Activity

Fixed TaglessEnvelopeFormat resolution

Browse Source
This commit is contained in:
Alexander Nozik 2019-11-13 21:12:19 +03:00
parent d10bd40763
commit 4370a66164
5 changed files with 82 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
dataforge-io/src/commonMain/kotlin/hep/dataforge/io/EnvelopeParts.kt
View File

@ -3,6 +3,7 @@ package hep.dataforge.io
import hep.dataforge.context.Global
import hep.dataforge.io.EnvelopeParts.FORMAT_META_KEY
import hep.dataforge.io.EnvelopeParts.FORMAT_NAME_KEY
import hep.dataforge.io.EnvelopeParts.INDEX_KEY
import hep.dataforge.io.EnvelopeParts.MULTIPART_DATA_TYPE
import hep.dataforge.io.EnvelopeParts.SIZE_KEY
import hep.dataforge.meta.*
@ -13,6 +14,7 @@ import hep.dataforge.names.toName
object EnvelopeParts {
val MULTIPART_KEY = "multipart".asName()
val SIZE_KEY = Envelope.ENVELOPE_NODE_KEY + MULTIPART_KEY + "size"
val INDEX_KEY = Envelope.ENVELOPE_NODE_KEY + MULTIPART_KEY + "index"
val FORMAT_NAME_KEY = Envelope.ENVELOPE_NODE_KEY + MULTIPART_KEY + "format"
val FORMAT_META_KEY = Envelope.ENVELOPE_NODE_KEY + MULTIPART_KEY + "meta"
@ -37,8 +39,37 @@ fun EnvelopeBuilder.multipart(format: EnvelopeFormatFactory, envelopes: Collecti
}
}
fun EnvelopeBuilder.multipart(formatFactory: EnvelopeFormatFactory, builder: suspend SequenceScope<Envelope>.() -> Unit) =
multipart(formatFactory, sequence(builder).toList())
/**
* Create a multipart partition in the envelope adding additional name-index mapping in meta
*/
@DFExperimental
fun EnvelopeBuilder.multipart(format: EnvelopeFormatFactory, envelopes: Map<String, Envelope>) {
dataType = MULTIPART_DATA_TYPE
meta {
SIZE_KEY put envelopes.size
FORMAT_NAME_KEY put format.name.toString()
}
data {
format.run {
var counter = 0
envelopes.forEach {(key, envelope)->
writeObject(envelope)
meta{
append(INDEX_KEY, buildMeta {
"key" put key
"index" put counter
})
}
counter++
}
}
}
}
fun EnvelopeBuilder.multipart(
formatFactory: EnvelopeFormatFactory,
builder: suspend SequenceScope<Envelope>.() -> Unit
) = multipart(formatFactory, sequence(builder).toList())
/**
* If given envelope supports multipart data, return a sequence of those parts (could be empty). Otherwise return null.

2
dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOPlugin.kt
View File

@ -52,7 +52,7 @@ class IOPlugin(meta: Meta) : AbstractPlugin(meta) {
companion object : PluginFactory<IOPlugin> {
val defaultMetaFormats: List<MetaFormatFactory> = listOf(JsonMetaFormat, BinaryMetaFormat)
val defaultEnvelopeFormats = listOf(TaggedEnvelopeFormat)
val defaultEnvelopeFormats = listOf(TaggedEnvelopeFormat, TaglessEnvelopeFormat)
override val tag: PluginTag = PluginTag("io", group = PluginTag.DATAFORGE_GROUP)

3
dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaglessEnvelopeFormat.kt
View File

@ -49,6 +49,7 @@ class TaglessEnvelopeFormat(
writeText(dataStart + "\r\n")
writeFully(data.toBytes())
}
flush()
}
override fun Input.readObject(): Envelope {
@ -191,7 +192,7 @@ class TaglessEnvelopeFormat(
return try {
val buffer = ByteArray(TAGLESS_ENVELOPE_HEADER.length)
input.readFully(buffer)
return if (buffer.toString() == TAGLESS_ENVELOPE_HEADER) {
return if (String(buffer) == TAGLESS_ENVELOPE_HEADER) {
TaglessEnvelopeFormat(io)
} else {
null

45
dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/fileIO.kt
View File

@ -58,6 +58,24 @@ fun IOPlugin.writeMetaFile(
}
}
/**
* Return inferred [EnvelopeFormat] if only one format could read given file. If no format accepts file, return null. If
* multiple formats accepts file, throw an error.
*/
fun IOPlugin.peekBinaryFormat(binary: Binary): EnvelopeFormat? {
val formats = envelopeFormatFactories.mapNotNull { factory ->
binary.read {
factory.peekFormat(this@peekBinaryFormat, this@read)
}
}
return when (formats.size) {
0 -> null
1 -> formats.first()
else -> error("Envelope format binary recognition clash")
}
}
/**
* Read and envelope from file if the file exists, return null if file does not exist.
*
@ -72,7 +90,11 @@ fun IOPlugin.writeMetaFile(
* Return null otherwise.
*/
@DFExperimental
fun IOPlugin.readEnvelopeFile(path: Path, readNonEnvelopes: Boolean = false): Envelope? {
fun IOPlugin.readEnvelopeFile(
path: Path,
readNonEnvelopes: Boolean = false,
formatPeeker: IOPlugin.(Binary) -> EnvelopeFormat? = IOPlugin::peekBinaryFormat
): Envelope? {
if (!Files.exists(path)) return null
//read two-files directory
@ -99,24 +121,13 @@ fun IOPlugin.readEnvelopeFile(path: Path, readNonEnvelopes: Boolean = false): En
val binary = path.asBinary()
val formats = envelopeFormatFactories.mapNotNull { factory ->
return formatPeeker(binary)?.run {
binary.read {
factory.peekFormat(this@readEnvelopeFile, this@read)
readObject()
}
}
return when (formats.size) {
0 -> if (readNonEnvelopes) {
SimpleEnvelope(Meta.empty, binary)
} else {
null
}
1 -> formats.first().run {
binary.read {
readObject()
}
}
else -> error("Envelope format file recognition clash")
}
} ?: if (readNonEnvelopes) { // if no format accepts file, read it as binary
SimpleEnvelope(Meta.empty, binary)
} else null
}
fun IOPlugin.writeEnvelopeFile(

23
dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/FileEnvelopeTest.kt
View File

@ -22,10 +22,23 @@ class FileEnvelopeTest {
@Test
fun testFileWriteRead() {
val tmpPath = Files.createTempFile("dataforge_test", ".df")
Global.io.writeEnvelopeFile(tmpPath,envelope)
println(tmpPath.toUri())
val restored: Envelope = Global.io.readEnvelopeFile(tmpPath)!!
assertTrue { envelope.contentEquals(restored) }
Global.io.run {
val tmpPath = Files.createTempFile("dataforge_test", ".df")
writeEnvelopeFile(tmpPath, envelope)
println(tmpPath.toUri())
val restored: Envelope = readEnvelopeFile(tmpPath)!!
assertTrue { envelope.contentEquals(restored) }
}
}
@Test
fun testFileWriteReadTagless() {
Global.io.run {
val tmpPath = Files.createTempFile("dataforge_test_tagless", ".df")
writeEnvelopeFile(tmpPath, envelope, format = TaglessEnvelopeFormat)
println(tmpPath.toUri())
val restored: Envelope = readEnvelopeFile(tmpPath)!!
assertTrue { envelope.contentEquals(restored) }
}
}
}