kscience/dataforge-core
8
0
Fork 0
Code Issues Pull Requests 1 Packages Projects Releases 12 Wiki Activity

Fixed leaked byte packets in io

Browse Source
This commit is contained in:
Alexander Nozik 2019-10-30 19:43:26 +03:00
parent ade4eec96e
commit 33c8e7088e
7 changed files with 79 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Binary.kt
View File

@ -1,9 +1,6 @@
package hep.dataforge.io
import kotlinx.io.core.ByteReadPacket
import kotlinx.io.core.Input
import kotlinx.io.core.buildPacket
import kotlinx.io.core.readBytes
import kotlinx.io.core.*
import kotlin.math.min
/**
@ -45,7 +42,7 @@ fun Binary.toBytes(): ByteArray = read {
@ExperimentalUnsignedTypes
fun RandomAccessBinary.readPacket(from: UInt, size: UInt): ByteReadPacket = read(from, size) {
ByteReadPacket(this.readBytes())
buildPacket { copyTo(this) }
}
@ExperimentalUnsignedTypes
@ -65,7 +62,9 @@ inline class ArrayBinary(val array: ByteArray) : RandomAccessBinary {
override fun <R> read(from: UInt, size: UInt, block: Input.() -> R): R {
val theSize = min(size, array.size.toUInt() - from)
return ByteReadPacket(array, from.toInt(), theSize.toInt()).block()
return buildPacket {
writeFully(array, from.toInt(), theSize.toInt())
}.block()
}
}

2
dataforge-io/src/commonMain/kotlin/hep/dataforge/io/MetaFormat.kt
View File

@ -52,7 +52,7 @@ fun Meta.toBytes(format: MetaFormat = JsonMetaFormat.default): ByteReadPacket =
}
fun MetaFormat.parse(str: String): Meta {
return ByteReadPacket(str.toByteArray()).readObject()
return buildPacket { writeText(str) }.readObject()
}
fun MetaFormatFactory.parse(str: String): Meta = invoke().parse(str)

11
dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt
View File

@ -7,6 +7,7 @@ import hep.dataforge.meta.string
import hep.dataforge.names.Name
import hep.dataforge.names.plus
import hep.dataforge.names.toName
import kotlinx.io.charsets.Charsets
import kotlinx.io.core.*
@ExperimentalUnsignedTypes
@ -30,10 +31,12 @@ class TaggedEnvelopeFormat(
override fun Output.writeObject(obj: Envelope) {
val metaBytes = metaFormat.writeBytes(obj.meta)
val tag = Tag(metaFormatKey, metaBytes.size.toUInt(), obj.data?.size ?: 0.toULong())
val tag = Tag(metaFormatKey, metaBytes.size.toUInt() + 2u, obj.data?.size ?: 0.toULong())
writePacket(tag.toBytes())
writeFully(metaBytes)
writeText("\r\n")
obj.data?.read { copyTo(this@writeObject) }
flush()
}
/**
@ -92,14 +95,14 @@ class TaggedEnvelopeFormat(
}
private fun Input.readTag(): Tag {
val start = readTextExactBytes(2)
val start = readTextExactBytes(2, charset = Charsets.ISO_8859_1)
if (start != START_SEQUENCE) error("The input is not an envelope")
val version = readTextExactBytes(4)
val version = readTextExactBytes(4, charset = Charsets.ISO_8859_1)
if (version != VERSION) error("Wrong version of DataForge: expected $VERSION but found $version")
val metaFormatKey = readShort()
val metaLength = readUInt()
val dataLength = readULong()
val end = readTextExactBytes(4)
val end = readTextExactBytes(4, charset = Charsets.ISO_8859_1)
if (end != END_SEQUENCE) error("The input is not an envelope")
return Tag(metaFormatKey, metaLength, dataLength)
}

10
dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaglessEnvelopeFormat.kt
View File

@ -72,7 +72,9 @@ class TaglessEnvelopeFormat(
val metaFormat = properties[META_TYPE_PROPERTY]?.let { io.metaFormat(it) } ?: JsonMetaFormat.default
val metaSize = properties.get(META_LENGTH_PROPERTY)?.toInt()
meta = if (metaSize != null) {
val metaPacket = ByteReadPacket(readBytes(metaSize))
val metaPacket = buildPacket {
writeFully(readBytes(metaSize))
}
metaFormat.run { metaPacket.readObject() }
} else {
metaFormat.run {
@ -126,11 +128,13 @@ class TaglessEnvelopeFormat(
val metaSize = properties.get(META_LENGTH_PROPERTY)?.toInt()
meta = if (metaSize != null) {
val metaPacket = ByteReadPacket(readBytes(metaSize))
val metaPacket = buildPacket {
writeFully(readBytes(metaSize))
}
offset += metaSize.toUInt()
metaFormat.run { metaPacket.readObject() }
} else {
error("Can't partially read an envelope with undefined meta size")
error("Can't partially read an envelope with undefined meta size")
}
}

4
dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileBinary.kt
View File

@ -1,7 +1,7 @@
package hep.dataforge.io
import kotlinx.io.core.ByteReadPacket
import kotlinx.io.core.Input
import kotlinx.io.core.buildPacket
import java.nio.channels.FileChannel
import java.nio.file.Files
import java.nio.file.Path
@ -17,7 +17,7 @@ class FileBinary(val path: Path, private val offset: UInt = 0u, size: ULong? = n
FileChannel.open(path, StandardOpenOption.READ).use {
val theSize: UInt = min(size, Files.size(path).toUInt() - offset)
val buffer = it.map(FileChannel.MapMode.READ_ONLY, (from + offset).toLong(), theSize.toLong())
return ByteReadPacket(buffer).block()
return buildPacket { writeFully(buffer) }.block()
}
}
}

2
dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileEnvelope.kt
View File

@ -15,7 +15,7 @@ class FileEnvelope internal constructor(val path: Path, val format: EnvelopeForm
init {
val input = Files.newByteChannel(path, StandardOpenOption.READ).asInput()
partialEnvelope = format.run { input.readPartial() }
partialEnvelope = format.run { input.use { it.readPartial()} }
}
override val meta: Meta get() = partialEnvelope.meta

56
dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/FileBinaryTest.kt Normal file
View File

@ -0,0 +1,56 @@
package hep.dataforge.io
import hep.dataforge.context.Global
import java.nio.file.Files
import kotlin.test.Test
import kotlin.test.assertEquals
class FileBinaryTest {
val envelope = Envelope {
meta {
"a" put "AAA"
"b" put 22.2
}
dataType = "hep.dataforge.test"
dataID = "myData" // добавил только что
data {
writeDouble(16.7)
}
}
@Test
fun testSize() {
val binary = envelope.data
assertEquals(binary?.size?.toInt(), binary?.toBytes()?.size)
}
@Test
fun testFileData(){
val dataFile = Files.createTempFile("dataforge_test_bin", ".bin")
dataFile.toFile().writeText("This is my binary")
val envelopeFromFile = Envelope {
meta {
"a" put "AAA"
"b" put 22.2
}
dataType = "hep.dataforge.satellite"
dataID = "cellDepositTest" // добавил только что
data = dataFile.asBinary()
}
val binary = envelopeFromFile.data!!
println(binary.toBytes().size)
assertEquals(binary.size.toInt(), binary.toBytes().size)
}
@Test
fun testFileDataSizeRewriting() {
println(System.getProperty("user.dir"))
val tmpPath = Files.createTempFile("dataforge_test", ".df")
Global.io.writeEnvelopeFile(tmpPath, envelope)
val binary = Global.io.readEnvelopeFile(tmpPath).data!!
assertEquals(binary.size.toInt(), binary.toBytes().size)
}
}