From 33c8e7088e3e0b22e56e1ac7cbca5ab1b4ac401d Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Wed, 30 Oct 2019 19:43:26 +0300 Subject: [PATCH] Fixed leaked byte packets in io --- .../kotlin/hep/dataforge/io/Binary.kt | 11 ++-- .../kotlin/hep/dataforge/io/MetaFormat.kt | 2 +- .../hep/dataforge/io/TaggedEnvelopeFormat.kt | 11 ++-- .../hep/dataforge/io/TaglessEnvelopeFormat.kt | 10 +++- .../kotlin/hep/dataforge/io/FileBinary.kt | 4 +- .../kotlin/hep/dataforge/io/FileEnvelope.kt | 2 +- .../kotlin/hep/dataforge/io/FileBinaryTest.kt | 56 +++++++++++++++++++ 7 files changed, 79 insertions(+), 17 deletions(-) create mode 100644 dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/FileBinaryTest.kt diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Binary.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Binary.kt index 252de7bc..ca05de4d 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Binary.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Binary.kt @@ -1,9 +1,6 @@ package hep.dataforge.io -import kotlinx.io.core.ByteReadPacket -import kotlinx.io.core.Input -import kotlinx.io.core.buildPacket -import kotlinx.io.core.readBytes +import kotlinx.io.core.* import kotlin.math.min /** @@ -45,7 +42,7 @@ fun Binary.toBytes(): ByteArray = read { @ExperimentalUnsignedTypes fun RandomAccessBinary.readPacket(from: UInt, size: UInt): ByteReadPacket = read(from, size) { - ByteReadPacket(this.readBytes()) + buildPacket { copyTo(this) } } @ExperimentalUnsignedTypes @@ -65,7 +62,9 @@ inline class ArrayBinary(val array: ByteArray) : RandomAccessBinary { override fun read(from: UInt, size: UInt, block: Input.() -> R): R { val theSize = min(size, array.size.toUInt() - from) - return ByteReadPacket(array, from.toInt(), theSize.toInt()).block() + return buildPacket { + writeFully(array, from.toInt(), theSize.toInt()) + }.block() } } diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/MetaFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/MetaFormat.kt index 313d04ae..ca9a53a2 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/MetaFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/MetaFormat.kt @@ -52,7 +52,7 @@ fun Meta.toBytes(format: MetaFormat = JsonMetaFormat.default): ByteReadPacket = } fun MetaFormat.parse(str: String): Meta { - return ByteReadPacket(str.toByteArray()).readObject() + return buildPacket { writeText(str) }.readObject() } fun MetaFormatFactory.parse(str: String): Meta = invoke().parse(str) diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt index b69a5f19..c51cb36d 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt @@ -7,6 +7,7 @@ import hep.dataforge.meta.string import hep.dataforge.names.Name import hep.dataforge.names.plus import hep.dataforge.names.toName +import kotlinx.io.charsets.Charsets import kotlinx.io.core.* @ExperimentalUnsignedTypes @@ -30,10 +31,12 @@ class TaggedEnvelopeFormat( override fun Output.writeObject(obj: Envelope) { val metaBytes = metaFormat.writeBytes(obj.meta) - val tag = Tag(metaFormatKey, metaBytes.size.toUInt(), obj.data?.size ?: 0.toULong()) + val tag = Tag(metaFormatKey, metaBytes.size.toUInt() + 2u, obj.data?.size ?: 0.toULong()) writePacket(tag.toBytes()) writeFully(metaBytes) + writeText("\r\n") obj.data?.read { copyTo(this@writeObject) } + flush() } /** @@ -92,14 +95,14 @@ class TaggedEnvelopeFormat( } private fun Input.readTag(): Tag { - val start = readTextExactBytes(2) + val start = readTextExactBytes(2, charset = Charsets.ISO_8859_1) if (start != START_SEQUENCE) error("The input is not an envelope") - val version = readTextExactBytes(4) + val version = readTextExactBytes(4, charset = Charsets.ISO_8859_1) if (version != VERSION) error("Wrong version of DataForge: expected $VERSION but found $version") val metaFormatKey = readShort() val metaLength = readUInt() val dataLength = readULong() - val end = readTextExactBytes(4) + val end = readTextExactBytes(4, charset = Charsets.ISO_8859_1) if (end != END_SEQUENCE) error("The input is not an envelope") return Tag(metaFormatKey, metaLength, dataLength) } diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaglessEnvelopeFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaglessEnvelopeFormat.kt index 83fbe46b..8e7c9e2b 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaglessEnvelopeFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaglessEnvelopeFormat.kt @@ -72,7 +72,9 @@ class TaglessEnvelopeFormat( val metaFormat = properties[META_TYPE_PROPERTY]?.let { io.metaFormat(it) } ?: JsonMetaFormat.default val metaSize = properties.get(META_LENGTH_PROPERTY)?.toInt() meta = if (metaSize != null) { - val metaPacket = ByteReadPacket(readBytes(metaSize)) + val metaPacket = buildPacket { + writeFully(readBytes(metaSize)) + } metaFormat.run { metaPacket.readObject() } } else { metaFormat.run { @@ -126,11 +128,13 @@ class TaglessEnvelopeFormat( val metaSize = properties.get(META_LENGTH_PROPERTY)?.toInt() meta = if (metaSize != null) { - val metaPacket = ByteReadPacket(readBytes(metaSize)) + val metaPacket = buildPacket { + writeFully(readBytes(metaSize)) + } offset += metaSize.toUInt() metaFormat.run { metaPacket.readObject() } } else { - error("Can't partially read an envelope with undefined meta size") + error("Can't partially read an envelope with undefined meta size") } } diff --git a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileBinary.kt b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileBinary.kt index 2c357a47..b935a103 100644 --- a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileBinary.kt +++ b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileBinary.kt @@ -1,7 +1,7 @@ package hep.dataforge.io -import kotlinx.io.core.ByteReadPacket import kotlinx.io.core.Input +import kotlinx.io.core.buildPacket import java.nio.channels.FileChannel import java.nio.file.Files import java.nio.file.Path @@ -17,7 +17,7 @@ class FileBinary(val path: Path, private val offset: UInt = 0u, size: ULong? = n FileChannel.open(path, StandardOpenOption.READ).use { val theSize: UInt = min(size, Files.size(path).toUInt() - offset) val buffer = it.map(FileChannel.MapMode.READ_ONLY, (from + offset).toLong(), theSize.toLong()) - return ByteReadPacket(buffer).block() + return buildPacket { writeFully(buffer) }.block() } } } diff --git a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileEnvelope.kt b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileEnvelope.kt index dcc8685d..3187cd54 100644 --- a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileEnvelope.kt +++ b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileEnvelope.kt @@ -15,7 +15,7 @@ class FileEnvelope internal constructor(val path: Path, val format: EnvelopeForm init { val input = Files.newByteChannel(path, StandardOpenOption.READ).asInput() - partialEnvelope = format.run { input.readPartial() } + partialEnvelope = format.run { input.use { it.readPartial()} } } override val meta: Meta get() = partialEnvelope.meta diff --git a/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/FileBinaryTest.kt b/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/FileBinaryTest.kt new file mode 100644 index 00000000..94403dcd --- /dev/null +++ b/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/FileBinaryTest.kt @@ -0,0 +1,56 @@ +package hep.dataforge.io + +import hep.dataforge.context.Global +import java.nio.file.Files +import kotlin.test.Test +import kotlin.test.assertEquals + +class FileBinaryTest { + val envelope = Envelope { + meta { + "a" put "AAA" + "b" put 22.2 + } + dataType = "hep.dataforge.test" + dataID = "myData" // добавил только что + data { + writeDouble(16.7) + } + } + + @Test + fun testSize() { + val binary = envelope.data + assertEquals(binary?.size?.toInt(), binary?.toBytes()?.size) + } + + @Test + fun testFileData(){ + val dataFile = Files.createTempFile("dataforge_test_bin", ".bin") + dataFile.toFile().writeText("This is my binary") + val envelopeFromFile = Envelope { + meta { + "a" put "AAA" + "b" put 22.2 + } + dataType = "hep.dataforge.satellite" + dataID = "cellDepositTest" // добавил только что + data = dataFile.asBinary() + } + val binary = envelopeFromFile.data!! + println(binary.toBytes().size) + assertEquals(binary.size.toInt(), binary.toBytes().size) + + } + + @Test + fun testFileDataSizeRewriting() { + println(System.getProperty("user.dir")) + val tmpPath = Files.createTempFile("dataforge_test", ".df") + Global.io.writeEnvelopeFile(tmpPath, envelope) + + val binary = Global.io.readEnvelopeFile(tmpPath).data!! + assertEquals(binary.size.toInt(), binary.toBytes().size) + } + +} \ No newline at end of file