From 2aba1b48dce011906231ba5ab67353f9901cadfa Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Mon, 10 Jul 2023 10:06:39 +0300 Subject: [PATCH 1/3] [WIP] Refactoring to kotlinx-io --- CHANGELOG.md | 1 + dataforge-io/build.gradle.kts | 14 ++-- .../space/kscience/dataforge/io/Binary.kt | 58 +++++++++----- .../kscience/dataforge/io/EnvelopeBuilder.kt | 4 +- .../kscience/dataforge/io/EnvelopeFormat.kt | 4 +- .../kscience/dataforge/io/EnvelopeParts.kt | 15 ++-- .../space/kscience/dataforge/io/IOFormat.kt | 26 ++++--- .../space/kscience/dataforge/io/ioMisc.kt | 75 +++++++++---------- 8 files changed, 113 insertions(+), 84 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ac4a5574..ac3db029 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ ### Added ### Changed +- Kotlin 1.9 ### Deprecated diff --git a/dataforge-io/build.gradle.kts b/dataforge-io/build.gradle.kts index 91319d4d..94aefa51 100644 --- a/dataforge-io/build.gradle.kts +++ b/dataforge-io/build.gradle.kts @@ -1,22 +1,24 @@ -import space.kscience.gradle.KScienceVersions - plugins { id("space.kscience.gradle.mpp") } description = "IO module" +val ioVersion = "0.2.0" + kscience { jvm() js() native() - useSerialization("1.4.1") - useSerialization("1.4.1", sourceSet = space.kscience.gradle.DependencySourceSet.TEST) { + useSerialization() + useSerialization(sourceSet = space.kscience.gradle.DependencySourceSet.TEST) { cbor() } dependencies { - api(project(":dataforge-context")) - api("io.ktor:ktor-io:${KScienceVersions.ktorVersion}") + api(projects.dataforgeContext) + api("org.jetbrains.kotlinx:kotlinx-io-core:$ioVersion") + api("org.jetbrains.kotlinx:kotlinx-io-bytestring:$ioVersion") + //api("io.ktor:ktor-io:${KScienceVersions.ktorVersion}") } } diff --git a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/Binary.kt b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/Binary.kt index bcc20064..c4e3bfbe 100644 --- a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/Binary.kt +++ b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/Binary.kt @@ -1,6 +1,6 @@ package space.kscience.dataforge.io -import io.ktor.utils.io.core.* +import kotlinx.io.* import kotlin.math.min /** @@ -17,13 +17,13 @@ public interface Binary { * Read maximum of [atMost] bytes as input from the binary, starting at [offset]. The generated input is always closed * when leaving scope, so it could not be leaked outside of scope of [block]. */ - public fun read(offset: Int = 0, atMost: Int = size - offset, block: Input.() -> R): R + public fun read(offset: Int = 0, atMost: Int = size - offset, block: Source.() -> R): R - public suspend fun readSuspend(offset: Int = 0, atMost: Int = size - offset, block: suspend Input.() -> R): R + public suspend fun readSuspend(offset: Int = 0, atMost: Int = size - offset, block: suspend Source.() -> R): R /** * Read a binary with given [offset] relative to this binary and given [binarySize]. - * In general, resulting binary is of the same type as this one, but it is not guaranteed. + * In general, the resulting binary is of the same type as this one, but it is not guaranteed. */ public fun view(offset: Int, binarySize: Int = size - offset): Binary @@ -32,31 +32,55 @@ public interface Binary { } } +public class ByteArraySource( + public val byteArray: ByteArray, + public val offset: Int, + public val size: Int, +) : RawSource { + override fun close() { + // Do nothing + } + + override fun readAtMostTo(sink: Buffer, byteCount: Long): Long { + val byteRead = min(byteCount, size.toLong()) + sink.write(byteArray, offset, offset + byteRead.toInt()) + return byteRead + } +} + internal class ByteArrayBinary( internal val array: ByteArray, internal val start: Int = 0, override val size: Int = array.size - start, ) : Binary { - override fun read(offset: Int, atMost: Int, block: Input.() -> R): R { + override fun read(offset: Int, atMost: Int, block: Source.() -> R): R { require(offset >= 0) { "Offset must be positive" } require(offset < array.size) { "Offset $offset is larger than array size" } - val input = ByteReadPacket( + + val input = ByteArraySource( array, offset + start, min(atMost, size - offset) - ) - return input.use(block) + ).buffered() + + return try { + block(input) + } finally { + input.close() + } } - override suspend fun readSuspend(offset: Int, atMost: Int, block: suspend Input.() -> R): R { + override suspend fun readSuspend(offset: Int, atMost: Int, block: suspend Source.() -> R): R { require(offset >= 0) { "Offset must be positive" } require(offset < array.size) { "Offset $offset is larger than array size" } - val input = ByteReadPacket( + + val input = ByteArraySource( array, offset + start, min(atMost, size - offset) - ) + ).buffered() + return try { block(input) } finally { @@ -77,26 +101,26 @@ public fun Binary.toByteArray(): ByteArray = if (this is ByteArrayBinary) { array.copyOfRange(start, start + size) // TODO do we need to ensure data safety here? } else { read { - readBytes() + readByteArray() } } //TODO optimize for file-based Inputs -public fun Input.readBinary(size: Int? = null): Binary { - val array = if (size == null) readBytes() else readBytes(size) +public fun Source.readBinary(size: Int? = null): Binary { + val array = if (size == null) readByteArray() else readByteArray(size) return ByteArrayBinary(array) } /** * Direct write of binary to the output. Returns the number of bytes written */ -public fun Output.writeBinary(binary: Binary): Int { +public fun Sink.writeBinary(binary: Binary): Int { return if (binary is ByteArrayBinary) { - writeFully(binary.array, binary.start, binary.start + binary.size) + write(binary.array, binary.start, binary.start + binary.size) binary.size } else { binary.read { - copyTo(this@writeBinary).toInt() + transferTo(this@writeBinary).toInt() } } } \ No newline at end of file diff --git a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/EnvelopeBuilder.kt b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/EnvelopeBuilder.kt index eedd2075..e38df146 100644 --- a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/EnvelopeBuilder.kt +++ b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/EnvelopeBuilder.kt @@ -1,6 +1,6 @@ package space.kscience.dataforge.io -import io.ktor.utils.io.core.Output +import kotlinx.io.Sink import space.kscience.dataforge.meta.* public class EnvelopeBuilder : Envelope { @@ -33,7 +33,7 @@ public class EnvelopeBuilder : Envelope { /** * Construct a data binary from given builder */ - public inline fun data(block: Output.() -> Unit) { + public inline fun data(block: Sink.() -> Unit) { data = ByteArray { block() }.asBinary() } diff --git a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/EnvelopeFormat.kt b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/EnvelopeFormat.kt index 28b59abb..d748d9f8 100644 --- a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/EnvelopeFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/EnvelopeFormat.kt @@ -1,6 +1,6 @@ package space.kscience.dataforge.io -import io.ktor.utils.io.core.Input +import kotlinx.io.Source import space.kscience.dataforge.context.Context import space.kscience.dataforge.io.EnvelopeFormatFactory.Companion.ENVELOPE_FORMAT_TYPE import space.kscience.dataforge.meta.Meta @@ -15,7 +15,7 @@ public interface EnvelopeFormat : IOFormat { override val type: KType get() = typeOf() } -public fun EnvelopeFormat.read(input: Input): Envelope = readObject(input) +public fun EnvelopeFormat.read(input: Source): Envelope = readObject(input) @Type(ENVELOPE_FORMAT_TYPE) public interface EnvelopeFormatFactory : IOFormatFactory, EnvelopeFormat { diff --git a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/EnvelopeParts.kt b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/EnvelopeParts.kt index 248ecf19..0d3017d6 100644 --- a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/EnvelopeParts.kt +++ b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/EnvelopeParts.kt @@ -1,5 +1,8 @@ package space.kscience.dataforge.io +import kotlinx.io.bytestring.ByteString +import kotlinx.io.bytestring.decodeToString +import kotlinx.io.write import space.kscience.dataforge.io.Envelope.Companion.ENVELOPE_NODE_KEY import space.kscience.dataforge.io.PartDescriptor.Companion.DEFAULT_MULTIPART_DATA_SEPARATOR import space.kscience.dataforge.io.PartDescriptor.Companion.MULTIPART_DATA_TYPE @@ -20,7 +23,7 @@ private class PartDescriptor : Scheme() { val PARTS_KEY = MULTIPART_KEY + "parts" val SEPARATOR_KEY = MULTIPART_KEY + "separator" - const val DEFAULT_MULTIPART_DATA_SEPARATOR = "\r\n#~PART~#\r\n" + val DEFAULT_MULTIPART_DATA_SEPARATOR = "\r\n#~PART~#\r\n".toACIIByteString() const val MULTIPART_DATA_TYPE = "envelope.multipart" } @@ -32,12 +35,12 @@ public typealias EnvelopeParts = List public fun EnvelopeBuilder.multipart( parts: EnvelopeParts, - separator: String = DEFAULT_MULTIPART_DATA_SEPARATOR, + separator: ByteString = DEFAULT_MULTIPART_DATA_SEPARATOR, ) { dataType = MULTIPART_DATA_TYPE var offsetCounter = 0 - val separatorSize = separator.length + val separatorSize = separator.size val partDescriptors = parts.map { (binary, description) -> offsetCounter += separatorSize PartDescriptor { @@ -51,14 +54,14 @@ public fun EnvelopeBuilder.multipart( meta { if (separator != DEFAULT_MULTIPART_DATA_SEPARATOR) { - SEPARATOR_KEY put separator + SEPARATOR_KEY put separator.decodeToString() } setIndexed(PARTS_KEY, partDescriptors.map { it.toMeta() }) } data { parts.forEach { - writeRawString(separator) + write(separator) writeBinary(it.binary) } } @@ -69,7 +72,7 @@ public fun EnvelopeBuilder.multipart( */ public fun EnvelopeBuilder.envelopes( envelopes: List, - separator: String = DEFAULT_MULTIPART_DATA_SEPARATOR, + separator: ByteString = DEFAULT_MULTIPART_DATA_SEPARATOR, ) { val parts = envelopes.map { val binary = Binary(it, TaggedEnvelopeFormat) diff --git a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/IOFormat.kt b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/IOFormat.kt index 12ea587b..623260e9 100644 --- a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/IOFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/IOFormat.kt @@ -1,6 +1,8 @@ package space.kscience.dataforge.io -import io.ktor.utils.io.core.* +import kotlinx.io.Sink +import kotlinx.io.Source +import kotlinx.io.readByteArray import space.kscience.dataforge.context.Context import space.kscience.dataforge.context.Factory import space.kscience.dataforge.io.IOFormatFactory.Companion.IO_FORMAT_TYPE @@ -21,7 +23,7 @@ public interface IOReader { */ public val type: KType - public fun readObject(input: Input): T + public fun readObject(input: Source): T public fun readObject(binary: Binary): T = binary.read { readObject(this) } @@ -32,21 +34,21 @@ public interface IOReader { public val binary: IOReader = object : IOReader { override val type: KType = typeOf() - override fun readObject(input: Input): Binary = input.readBytes().asBinary() + override fun readObject(input: Source): Binary = input.readByteArray().asBinary() override fun readObject(binary: Binary): Binary = binary } } } -public inline fun IOReader(crossinline read: Input.() -> T): IOReader = object : IOReader { +public inline fun IOReader(crossinline read: Source.() -> T): IOReader = object : IOReader { override val type: KType = typeOf() - override fun readObject(input: Input): T = input.read() + override fun readObject(input: Source): T = input.read() } public fun interface IOWriter { - public fun writeObject(output: Output, obj: T) + public fun writeObject(output: Sink, obj: T) } /** @@ -54,20 +56,20 @@ public fun interface IOWriter { */ public interface IOFormat : IOReader, IOWriter -public fun Input.readObject(format: IOReader): T = format.readObject(this@readObject) +public fun Source.readObject(format: IOReader): T = format.readObject(this@readObject) public fun IOFormat.readObjectFrom(binary: Binary): T = binary.read { readObject(this) } /** - * Read given binary as object using given format + * Read given binary as an object using given format */ public fun Binary.readWith(format: IOReader): T = read { readObject(format) } -public fun Output.writeObject(format: IOWriter, obj: T): Unit = +public fun Sink.writeObject(format: IOWriter, obj: T): Unit = format.writeObject(this@writeObject, obj) @@ -94,9 +96,9 @@ public object DoubleIOFormat : IOFormat, IOFormatFactory { override val type: KType get() = typeOf() - override fun writeObject(output: Output, obj: Double) { - output.writeDouble(obj) + override fun writeObject(output: Sink, obj: Double) { + output.writeLong(obj.toBits()) } - override fun readObject(input: Input): Double = input.readDouble() + override fun readObject(input: Source): Double = Double.fromBits(input.readLong()) } \ No newline at end of file diff --git a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/ioMisc.kt b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/ioMisc.kt index ad03ba6e..c0132895 100644 --- a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/ioMisc.kt +++ b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/ioMisc.kt @@ -1,40 +1,38 @@ package space.kscience.dataforge.io -import io.ktor.utils.io.bits.Memory -import io.ktor.utils.io.charsets.Charsets -import io.ktor.utils.io.charsets.decodeExactBytes -import io.ktor.utils.io.core.* -import io.ktor.utils.io.core.internal.ChunkBuffer +import kotlinx.io.* +import kotlinx.io.bytestring.ByteString import space.kscience.dataforge.meta.Meta import space.kscience.dataforge.misc.DFExperimental -public fun Output.writeRawString(str: String) { - writeFully(str.toByteArray(Charsets.ISO_8859_1)) +/** + * Convert a string literal, containing only ASCII characters to a [ByteString]. + * Throws an error if there are non-ASCII characters. + */ +public fun String.toACIIByteString(): ByteString { + val bytes = ByteArray(length) { + val char = get(it) + val code = char.code + if (code > Byte.MAX_VALUE) error("Symbol $char is not ASCII symbol") else code.toByte() + } + return ByteString(bytes) } -public fun Output.writeUtf8String(str: String) { - writeFully(str.encodeToByteArray()) -} +public inline fun Buffer(block: Sink.() -> Unit): Buffer = Buffer().apply(block) -public fun Input.readRawString(size: Int): String { - return Charsets.ISO_8859_1.newDecoder().decodeExactBytes(this, size) -} +//public fun Source.readSafeUtf8Line(): String = readUTF8Line() ?: error("Line not found") -public fun Input.readUtf8String(): String = readBytes().decodeToString() +public inline fun ByteArray(block: Sink.() -> Unit): ByteArray = + Buffer(block).readByteArray() -public fun Input.readSafeUtf8Line(): String = readUTF8Line() ?: error("Line not found") - -public inline fun ByteArray(block: Output.() -> Unit): ByteArray = - buildPacket(block).readBytes() - -public inline fun Binary(block: Output.() -> Unit): Binary = +public inline fun Binary(block: Sink.() -> Unit): Binary = ByteArray(block).asBinary() public operator fun Binary.get(range: IntRange): Binary = view(range.first, range.last - range.first) /** * Return inferred [EnvelopeFormat] if only one format could read given file. If no format accepts the binary, return null. If - * multiple formats accepts binary, throw an error. + * multiple formats accept binary, throw an error. */ public fun IOPlugin.peekBinaryEnvelopeFormat(binary: Binary): EnvelopeFormat? { val formats = envelopeFormatFactories.mapNotNull { factory -> @@ -111,32 +109,31 @@ private fun RingByteArray.toArray(): ByteArray = ByteArray(size) { get(it) } * * @return bytes actually being read, including separator */ -public fun Input.readWithSeparatorTo( - output: Output, +public fun Source.readWithSeparatorTo( + output: Sink, separator: ByteArray, atMost: Int = Int.MAX_VALUE, errorOnEof: Boolean = false, ): Int { var counter = 0 val rb = RingByteArray(ByteArray(separator.size)) - takeWhile { buffer -> - while (buffer.canRead()) { - val byte = buffer.readByte() - counter++ - if (counter >= atMost) error("Maximum number of bytes to be read $atMost reached.") - rb.push(byte) - if (rb.contentEquals(separator)) { - return counter - } else if (rb.isFull()) { - output.writeByte(rb[0]) - } + + while (!exhausted()) { + val byte = readByte() + counter++ + if (counter >= atMost) error("Maximum number of bytes to be read $atMost reached.") + rb.push(byte) + if (rb.contentEquals(separator)) { + return counter + } else if (rb.isFull()) { + output.writeByte(rb[0]) } - !endOfInput } + if (errorOnEof) { error("Read to the end of input without encountering ${separator.decodeToString()}") } else { - for(i in 1 until rb.size){ + for (i in 1 until rb.size) { output.writeByte(rb[i]) } counter += (rb.size - 1) @@ -144,18 +141,18 @@ public fun Input.readWithSeparatorTo( } } -public fun Input.discardLine(): Int { +public fun Source.discardLine(): Int { return discardUntilDelimiter('\n'.code.toByte()).also { discard(1) }.toInt() + 1 } -public fun Input.discardWithSeparator( +public fun Source.discardWithSeparator( separator: ByteArray, atMost: Int = Int.MAX_VALUE, errorOnEof: Boolean = false, ): Int { - val dummy: Output = object : Output(ChunkBuffer.Pool) { + val dummy: Sink = object : Sink(ChunkBuffer.Pool) { override fun closeDestination() { // Do nothing } From a699c36f8e7a695930cb540be818f7e4385862d7 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Tue, 11 Jul 2023 23:03:52 +0300 Subject: [PATCH 2/3] [WIP] Refactoring to kotlinx-io --- dataforge-io/build.gradle.kts | 2 +- .../io/yaml/FrontMatterEnvelopeFormat.kt | 38 +++--- .../dataforge/io/yaml/YamlMetaFormat.kt | 24 ++-- .../space/kscience/dataforge/io/Binary.kt | 31 +---- .../kscience/dataforge/io/EnvelopeParts.kt | 2 +- .../space/kscience/dataforge/io/IOFormat.kt | 14 +-- .../kscience/dataforge/io/JsonMetaFormat.kt | 24 ++-- .../space/kscience/dataforge/io/MetaFormat.kt | 22 ++-- .../dataforge/io/TaggedEnvelopeFormat.kt | 79 ++++++------ .../dataforge/io/TaglessEnvelopeFormat.kt | 56 ++++----- .../space/kscience/dataforge/io/ioMisc.kt | 115 ++++++++++++++---- .../space/kscience/dataforge/io/BinaryTest.kt | 1 - .../dataforge/io/EnvelopeFormatTest.kt | 17 +-- .../space/kscience/dataforge/io/IOTest.kt | 21 ++-- .../kscience/dataforge/io/MultipartTest.kt | 5 +- .../kscience/dataforge/io/ioTestUtils.kt | 5 +- .../space/kscience/dataforge/io/fileIO.kt | 29 +++-- .../space/kscience/dataforge/io/resourceIO.kt | 12 +- .../kscience/dataforge/io/FileBinaryTest.kt | 1 - .../kscience/dataforge/io/FileEnvelopeTest.kt | 1 - .../dataforge/workspace/FileWorkspaceCache.kt | 4 - 21 files changed, 275 insertions(+), 228 deletions(-) diff --git a/dataforge-io/build.gradle.kts b/dataforge-io/build.gradle.kts index 94aefa51..b4ad8ade 100644 --- a/dataforge-io/build.gradle.kts +++ b/dataforge-io/build.gradle.kts @@ -23,5 +23,5 @@ kscience { } readme{ - maturity = space.kscience.gradle.Maturity.PROTOTYPE + maturity = space.kscience.gradle.Maturity.EXPERIMENTAL } \ No newline at end of file diff --git a/dataforge-io/dataforge-io-yaml/src/commonMain/kotlin/space/kscience/dataforge/io/yaml/FrontMatterEnvelopeFormat.kt b/dataforge-io/dataforge-io-yaml/src/commonMain/kotlin/space/kscience/dataforge/io/yaml/FrontMatterEnvelopeFormat.kt index 046f08fa..736f3306 100644 --- a/dataforge-io/dataforge-io-yaml/src/commonMain/kotlin/space/kscience/dataforge/io/yaml/FrontMatterEnvelopeFormat.kt +++ b/dataforge-io/dataforge-io-yaml/src/commonMain/kotlin/space/kscience/dataforge/io/yaml/FrontMatterEnvelopeFormat.kt @@ -1,7 +1,11 @@ package space.kscience.dataforge.io.yaml -import io.ktor.utils.io.core.Input -import io.ktor.utils.io.core.Output +import kotlinx.io.Sink +import kotlinx.io.Source +import kotlinx.io.bytestring.ByteString +import kotlinx.io.bytestring.encodeToByteString +import kotlinx.io.readByteString +import kotlinx.io.writeString import space.kscience.dataforge.context.Context import space.kscience.dataforge.context.Global import space.kscience.dataforge.io.* @@ -19,18 +23,18 @@ public class FrontMatterEnvelopeFormat( var offset = 0 offset += discardWithSeparator( - SEPARATOR.encodeToByteArray(), + SEPARATOR, atMost = 1024, ) val line = ByteArray { - offset += readWithSeparatorTo(this, "\n".encodeToByteArray()) + offset += readWithSeparatorTo(this, "\n".encodeToByteString()) }.decodeToString() val readMetaFormat = line.trim().takeIf { it.isNotBlank() }?.let { io.resolveMetaFormat(it) } ?: YamlMetaFormat val packet = ByteArray { - offset += readWithSeparatorTo(this, SEPARATOR.encodeToByteArray()) + offset += readWithSeparatorTo(this, SEPARATOR) } offset += discardLine() @@ -39,25 +43,25 @@ public class FrontMatterEnvelopeFormat( Envelope(meta, binary.view(offset)) } - override fun readObject(input: Input): Envelope = readObject(input.readBinary()) + override fun readObject(source: Source): Envelope = readObject(source.readBinary()) override fun writeObject( - output: Output, + sink: Sink, obj: Envelope, ) { val metaFormat = metaFormatFactory.build(io.context, meta) val formatSuffix = if (metaFormat is YamlMetaFormat) "" else metaFormatFactory.shortName - output.writeRawString("$SEPARATOR${formatSuffix}\r\n") - metaFormat.run { metaFormat.writeObject(output, obj.meta) } - output.writeRawString("$SEPARATOR\r\n") + sink.writeString("$SEPARATOR${formatSuffix}\r\n") + metaFormat.run { metaFormat.writeObject(sink, obj.meta) } + sink.writeString("$SEPARATOR\r\n") //Printing data obj.data?.let { data -> - output.writeBinary(data) + sink.writeBinary(data) } } public companion object : EnvelopeFormatFactory { - public const val SEPARATOR: String = "---" + public val SEPARATOR: ByteString = "---".encodeToByteString() private val metaTypeRegex = "---(\\w*)\\s*".toRegex() @@ -69,8 +73,8 @@ public class FrontMatterEnvelopeFormat( override fun peekFormat(io: IOPlugin, binary: Binary): EnvelopeFormat? = binary.read { //read raw string to avoid UTF issues - val line = readRawString(3) - return@read if (line == "---") { + val line = readByteString(3) + return@read if (line == "---".encodeToByteString()) { default } else { null @@ -82,12 +86,12 @@ public class FrontMatterEnvelopeFormat( override fun readObject(binary: Binary): Envelope = default.readObject(binary) override fun writeObject( - output: Output, + sink: Sink, obj: Envelope, - ): Unit = default.writeObject(output, obj) + ): Unit = default.writeObject(sink, obj) - override fun readObject(input: Input): Envelope = default.readObject(input) + override fun readObject(source: Source): Envelope = default.readObject(source) } } \ No newline at end of file diff --git a/dataforge-io/dataforge-io-yaml/src/commonMain/kotlin/space/kscience/dataforge/io/yaml/YamlMetaFormat.kt b/dataforge-io/dataforge-io-yaml/src/commonMain/kotlin/space/kscience/dataforge/io/yaml/YamlMetaFormat.kt index 51c143a6..589b7e66 100644 --- a/dataforge-io/dataforge-io-yaml/src/commonMain/kotlin/space/kscience/dataforge/io/yaml/YamlMetaFormat.kt +++ b/dataforge-io/dataforge-io-yaml/src/commonMain/kotlin/space/kscience/dataforge/io/yaml/YamlMetaFormat.kt @@ -1,13 +1,13 @@ package space.kscience.dataforge.io.yaml -import io.ktor.utils.io.core.Input -import io.ktor.utils.io.core.Output +import kotlinx.io.Sink +import kotlinx.io.Source +import kotlinx.io.readString +import kotlinx.io.writeString import net.mamoe.yamlkt.* import space.kscience.dataforge.context.Context import space.kscience.dataforge.io.MetaFormat import space.kscience.dataforge.io.MetaFormatFactory -import space.kscience.dataforge.io.readUtf8String -import space.kscience.dataforge.io.writeUtf8String import space.kscience.dataforge.meta.* import space.kscience.dataforge.meta.descriptors.MetaDescriptor import space.kscience.dataforge.meta.descriptors.get @@ -89,14 +89,14 @@ public fun YamlMap.toMeta(): Meta = YamlMeta(this) */ public class YamlMetaFormat(private val meta: Meta) : MetaFormat { - override fun writeMeta(output: Output, meta: Meta, descriptor: MetaDescriptor?) { + override fun writeMeta(sink: Sink, meta: Meta, descriptor: MetaDescriptor?) { val yaml: YamlMap = meta.toYaml() val string = Yaml.encodeToString(YamlMap.serializer(), yaml) - output.writeUtf8String(string) + sink.writeString(string) } - override fun readMeta(input: Input, descriptor: MetaDescriptor?): Meta { - val yaml = Yaml.decodeYamlMapFromString(input.readUtf8String()) + override fun readMeta(source: Source, descriptor: MetaDescriptor?): Meta { + val yaml = Yaml.decodeYamlMapFromString(source.readString()) return yaml.toMeta() } @@ -109,10 +109,10 @@ public class YamlMetaFormat(private val meta: Meta) : MetaFormat { private val default = YamlMetaFormat(Meta.EMPTY) - override fun writeMeta(output: Output, meta: Meta, descriptor: MetaDescriptor?): Unit = - default.writeMeta(output, meta, descriptor) + override fun writeMeta(sink: Sink, meta: Meta, descriptor: MetaDescriptor?): Unit = + default.writeMeta(sink, meta, descriptor) - override fun readMeta(input: Input, descriptor: MetaDescriptor?): Meta = - default.readMeta(input, descriptor) + override fun readMeta(source: Source, descriptor: MetaDescriptor?): Meta = + default.readMeta(source, descriptor) } } \ No newline at end of file diff --git a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/Binary.kt b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/Binary.kt index c4e3bfbe..f85dd98e 100644 --- a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/Binary.kt +++ b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/Binary.kt @@ -1,6 +1,9 @@ package space.kscience.dataforge.io -import kotlinx.io.* +import kotlinx.io.Sink +import kotlinx.io.Source +import kotlinx.io.buffered +import kotlinx.io.readByteArray import kotlin.math.min /** @@ -32,22 +35,6 @@ public interface Binary { } } -public class ByteArraySource( - public val byteArray: ByteArray, - public val offset: Int, - public val size: Int, -) : RawSource { - override fun close() { - // Do nothing - } - - override fun readAtMostTo(sink: Buffer, byteCount: Long): Long { - val byteRead = min(byteCount, size.toLong()) - sink.write(byteArray, offset, offset + byteRead.toInt()) - return byteRead - } -} - internal class ByteArrayBinary( internal val array: ByteArray, internal val start: Int = 0, @@ -58,17 +45,11 @@ internal class ByteArrayBinary( require(offset >= 0) { "Offset must be positive" } require(offset < array.size) { "Offset $offset is larger than array size" } - val input = ByteArraySource( + return ByteArraySource( array, offset + start, min(atMost, size - offset) - ).buffered() - - return try { - block(input) - } finally { - input.close() - } + ).buffered().use(block) } override suspend fun readSuspend(offset: Int, atMost: Int, block: suspend Source.() -> R): R { diff --git a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/EnvelopeParts.kt b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/EnvelopeParts.kt index 0d3017d6..183e7b03 100644 --- a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/EnvelopeParts.kt +++ b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/EnvelopeParts.kt @@ -23,7 +23,7 @@ private class PartDescriptor : Scheme() { val PARTS_KEY = MULTIPART_KEY + "parts" val SEPARATOR_KEY = MULTIPART_KEY + "separator" - val DEFAULT_MULTIPART_DATA_SEPARATOR = "\r\n#~PART~#\r\n".toACIIByteString() + val DEFAULT_MULTIPART_DATA_SEPARATOR = "\r\n#~PART~#\r\n".toAsciiByteString() const val MULTIPART_DATA_TYPE = "envelope.multipart" } diff --git a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/IOFormat.kt b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/IOFormat.kt index 623260e9..d0d00a9a 100644 --- a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/IOFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/IOFormat.kt @@ -23,7 +23,7 @@ public interface IOReader { */ public val type: KType - public fun readObject(input: Source): T + public fun readObject(source: Source): T public fun readObject(binary: Binary): T = binary.read { readObject(this) } @@ -34,7 +34,7 @@ public interface IOReader { public val binary: IOReader = object : IOReader { override val type: KType = typeOf() - override fun readObject(input: Source): Binary = input.readByteArray().asBinary() + override fun readObject(source: Source): Binary = source.readByteArray().asBinary() override fun readObject(binary: Binary): Binary = binary } @@ -44,11 +44,11 @@ public interface IOReader { public inline fun IOReader(crossinline read: Source.() -> T): IOReader = object : IOReader { override val type: KType = typeOf() - override fun readObject(input: Source): T = input.read() + override fun readObject(source: Source): T = source.read() } public fun interface IOWriter { - public fun writeObject(output: Sink, obj: T) + public fun writeObject(sink: Sink, obj: T) } /** @@ -96,9 +96,9 @@ public object DoubleIOFormat : IOFormat, IOFormatFactory { override val type: KType get() = typeOf() - override fun writeObject(output: Sink, obj: Double) { - output.writeLong(obj.toBits()) + override fun writeObject(sink: Sink, obj: Double) { + sink.writeLong(obj.toBits()) } - override fun readObject(input: Source): Double = Double.fromBits(input.readLong()) + override fun readObject(source: Source): Double = Double.fromBits(source.readLong()) } \ No newline at end of file diff --git a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/JsonMetaFormat.kt b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/JsonMetaFormat.kt index 4bb3c82d..91dbe75f 100644 --- a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/JsonMetaFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/JsonMetaFormat.kt @@ -1,10 +1,10 @@ -@file:Suppress("UNUSED_PARAMETER") - package space.kscience.dataforge.io -import io.ktor.utils.io.core.Input -import io.ktor.utils.io.core.Output +import kotlinx.io.Sink +import kotlinx.io.Source +import kotlinx.io.readString +import kotlinx.io.writeString import kotlinx.serialization.json.Json import kotlinx.serialization.json.JsonObject import space.kscience.dataforge.context.Context @@ -18,13 +18,13 @@ import space.kscience.dataforge.meta.toMeta */ public class JsonMetaFormat(private val json: Json = DEFAULT_JSON) : MetaFormat { - override fun writeMeta(output: Output, meta: Meta, descriptor: MetaDescriptor?) { + override fun writeMeta(sink: Sink, meta: Meta, descriptor: MetaDescriptor?) { val jsonObject = meta.toJson(descriptor) - output.writeUtf8String(json.encodeToString(JsonObject.serializer(), jsonObject)) + sink.writeString(json.encodeToString(JsonObject.serializer(), jsonObject)) } - override fun readMeta(input: Input, descriptor: MetaDescriptor?): Meta { - val str = input.readUtf8String()//readByteArray().decodeToString() + override fun readMeta(source: Source, descriptor: MetaDescriptor?): Meta { + val str = source.readString() val jsonElement = json.parseToJsonElement(str) return jsonElement.toMeta(descriptor) } @@ -39,10 +39,10 @@ public class JsonMetaFormat(private val json: Json = DEFAULT_JSON) : MetaFormat private val default = JsonMetaFormat() - override fun writeMeta(output: Output, meta: Meta, descriptor: MetaDescriptor?): Unit = - default.run { writeMeta(output, meta, descriptor) } + override fun writeMeta(sink: Sink, meta: Meta, descriptor: MetaDescriptor?): Unit = + default.run { writeMeta(sink, meta, descriptor) } - override fun readMeta(input: Input, descriptor: MetaDescriptor?): Meta = - default.run { readMeta(input, descriptor) } + override fun readMeta(source: Source, descriptor: MetaDescriptor?): Meta = + default.run { readMeta(source, descriptor) } } } diff --git a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/MetaFormat.kt b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/MetaFormat.kt index c902c2e6..25eb8f2f 100644 --- a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/MetaFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/MetaFormat.kt @@ -1,9 +1,9 @@ package space.kscience.dataforge.io -import io.ktor.utils.io.core.ByteReadPacket -import io.ktor.utils.io.core.Input -import io.ktor.utils.io.core.Output -import io.ktor.utils.io.core.use + +import kotlinx.io.Sink +import kotlinx.io.Source +import kotlinx.io.buffered import space.kscience.dataforge.context.Context import space.kscience.dataforge.context.Global import space.kscience.dataforge.io.MetaFormatFactory.Companion.META_FORMAT_TYPE @@ -23,19 +23,19 @@ public interface MetaFormat : IOFormat { override val type: KType get() = typeOf() - override fun writeObject(output: Output, obj: Meta) { - writeMeta(output, obj, null) + override fun writeObject(sink: Sink, obj: Meta) { + writeMeta(sink, obj, null) } - override fun readObject(input: Input): Meta = readMeta(input) + override fun readObject(source: Source): Meta = readMeta(source) public fun writeMeta( - output: Output, + sink: Sink, meta: Meta, descriptor: MetaDescriptor? = null, ) - public fun readMeta(input: Input, descriptor: MetaDescriptor? = null): Meta + public fun readMeta(source: Source, descriptor: MetaDescriptor? = null): Meta } @Type(META_FORMAT_TYPE) @@ -63,9 +63,7 @@ public fun Meta.toString(format: MetaFormat): String = ByteArray { public fun Meta.toString(formatFactory: MetaFormatFactory): String = toString(formatFactory.build(Global, Meta.EMPTY)) -public fun MetaFormat.parse(str: String): Meta { - return ByteReadPacket(str.encodeToByteArray()).use { readObject(it) } -} +public fun MetaFormat.parse(str: String): Meta = readObject(StringSource(str).buffered()) public fun MetaFormatFactory.parse(str: String, formatMeta: Meta): Meta = build(Global, formatMeta).parse(str) diff --git a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/TaggedEnvelopeFormat.kt b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/TaggedEnvelopeFormat.kt index a320837d..763c90b1 100644 --- a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/TaggedEnvelopeFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/TaggedEnvelopeFormat.kt @@ -1,6 +1,7 @@ package space.kscience.dataforge.io -import io.ktor.utils.io.core.* +import kotlinx.io.* +import kotlinx.io.bytestring.decodeToString import space.kscience.dataforge.context.Context import space.kscience.dataforge.context.Global import space.kscience.dataforge.meta.Meta @@ -18,7 +19,7 @@ import space.kscience.dataforge.names.plus public class TaggedEnvelopeFormat( public val io: IOPlugin, public val version: VERSION = VERSION.DF02, - public val metaFormatFactory: MetaFormatFactory = JsonMetaFormat + public val metaFormatFactory: MetaFormatFactory = JsonMetaFormat, ) : EnvelopeFormat { // private val metaFormat = io.metaFormat(metaFormatKey) @@ -26,59 +27,60 @@ public class TaggedEnvelopeFormat( private fun Tag.toBinary() = Binary { - writeRawString(START_SEQUENCE) - writeRawString(version.name) + write(START_SEQUENCE) + writeString(version.name) writeShort(metaFormatKey) writeUInt(metaSize) when (version) { VERSION.DF02 -> { writeUInt(dataSize.toUInt()) } + VERSION.DF03 -> { writeULong(dataSize) } } - writeRawString(END_SEQUENCE) + write(END_SEQUENCE) } override fun writeObject( - output: Output, + sink: Sink, obj: Envelope, ) { val metaFormat = metaFormatFactory.build(io.context, Meta.EMPTY) - val metaBytes = Binary(obj.meta,metaFormat) + val metaBytes = Binary(obj.meta, metaFormat) val actualSize: ULong = (obj.data?.size ?: 0).toULong() val tag = Tag(metaFormatFactory.key, metaBytes.size.toUInt() + 2u, actualSize) - output.writeBinary(tag.toBinary()) - output.writeBinary(metaBytes) - output.writeRawString("\r\n") + sink.writeBinary(tag.toBinary()) + sink.writeBinary(metaBytes) + sink.writeString("\r\n") obj.data?.let { - output.writeBinary(it) + sink.writeBinary(it) } } /** * Read an envelope from input into memory * - * @param input an input to read from + * @param source an input to read from * @param formats a collection of meta formats to resolve */ - override fun readObject(input: Input): Envelope { - val tag = input.readTag(this.version) + override fun readObject(source: Source): Envelope { + val tag = source.readTag(this.version) val metaFormat = io.resolveMetaFormat(tag.metaFormatKey) ?: error("Meta format with key ${tag.metaFormatKey} not found") - val metaBinary = input.readBinary(tag.metaSize.toInt()) + val metaBinary = source.readBinary(tag.metaSize.toInt()) val meta: Meta = metaFormat.readObjectFrom(metaBinary) - val data = input.readBinary(tag.dataSize.toInt()) + val data = source.readBinary(tag.dataSize.toInt()) return SimpleEnvelope(meta, data) } - override fun readObject(binary: Binary): Envelope = binary.read{ + override fun readObject(binary: Binary): Envelope = binary.read { val tag = readTag(version) val metaFormat = io.resolveMetaFormat(tag.metaFormatKey) @@ -104,8 +106,8 @@ public class TaggedEnvelopeFormat( } public companion object : EnvelopeFormatFactory { - private const val START_SEQUENCE = "#~" - private const val END_SEQUENCE = "~#\r\n" + private val START_SEQUENCE = "#~".toAsciiByteString() + private val END_SEQUENCE = "~#\r\n".toAsciiByteString() override val name: Name = EnvelopeFormatFactory.ENVELOPE_FACTORY_NAME + "tagged" @@ -121,35 +123,33 @@ public class TaggedEnvelopeFormat( return TaggedEnvelopeFormat(io, version) } - private fun Input.readTag(version: VERSION): Tag { - val start = readRawString(2) + private fun Source.readTag(version: VERSION): Tag { + val start = readByteString(2) if (start != START_SEQUENCE) error("The input is not an envelope") - val versionString = readRawString(4) - if (version.name != versionString) error("Wrong version of DataForge: expected $version but found $versionString") + val versionString = readByteString(4) + if (version.name.toAsciiByteString() != versionString) error("Wrong version of DataForge: expected $version but found $versionString") val metaFormatKey = readShort() val metaLength = readUInt() val dataLength: ULong = when (version) { VERSION.DF02 -> readUInt().toULong() VERSION.DF03 -> readULong() } - val end = readRawString(4) + val end = readByteString(4) if (end != END_SEQUENCE) error("The input is not an envelope") return Tag(metaFormatKey, metaLength, dataLength) } - override fun peekFormat(io: IOPlugin, binary: Binary): EnvelopeFormat? { - return try { - binary.read { - val header = readRawString(6) - return@read when (header.substring(2..5)) { - VERSION.DF02.name -> TaggedEnvelopeFormat(io, VERSION.DF02) - VERSION.DF03.name -> TaggedEnvelopeFormat(io, VERSION.DF03) - else -> null - } + override fun peekFormat(io: IOPlugin, binary: Binary): EnvelopeFormat? = try { + binary.read { + val header = readByteString(6) + when (header.substring(2, 6).decodeToString()) { + VERSION.DF02.name -> TaggedEnvelopeFormat(io, VERSION.DF02) + VERSION.DF03.name -> TaggedEnvelopeFormat(io, VERSION.DF03) + else -> null } - } catch (ex: Exception) { - null } + } catch (ex: Exception) { + null } private val default by lazy { build(Global, Meta.EMPTY) } @@ -158,16 +158,13 @@ public class TaggedEnvelopeFormat( default.run { readObject(binary) } override fun writeObject( - output: Output, + sink: Sink, obj: Envelope, ): Unit = default.run { - writeObject( - output, - obj, - ) + writeObject(sink, obj) } - override fun readObject(input: Input): Envelope = default.readObject(input) + override fun readObject(source: Source): Envelope = default.readObject(source) } } \ No newline at end of file diff --git a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/TaglessEnvelopeFormat.kt b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/TaglessEnvelopeFormat.kt index e6dc054f..2d49a3c0 100644 --- a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/TaglessEnvelopeFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/TaglessEnvelopeFormat.kt @@ -1,9 +1,9 @@ package space.kscience.dataforge.io -import io.ktor.utils.io.core.ByteReadPacket -import io.ktor.utils.io.core.Input -import io.ktor.utils.io.core.Output -import io.ktor.utils.io.core.readUTF8UntilDelimiterTo + +import kotlinx.io.* +import kotlinx.io.bytestring.ByteString +import kotlinx.io.bytestring.encodeToByteString import space.kscience.dataforge.context.Context import space.kscience.dataforge.context.Global import space.kscience.dataforge.meta.Meta @@ -29,34 +29,35 @@ public class TaglessEnvelopeFormat( // } override fun writeObject( - output: Output, + sink: Sink, obj: Envelope, ) { val metaFormat = metaFormatFactory.build(this.io.context, meta) //printing header - output.writeRawString(TAGLESS_ENVELOPE_HEADER + "\r\n") + sink.write(TAGLESS_ENVELOPE_HEADER) + sink.writeString("\r\n") //Printing meta if (!obj.meta.isEmpty()) { val metaBinary = Binary(obj.meta, metaFormat) - output.writeUtf8String(META_START + "-${metaFormatFactory.shortName}\r\n") - output.writeBinary(metaBinary) - output.writeRawString("\r\n") + sink.writeString(META_START + "-${metaFormatFactory.shortName}\r\n") + sink.writeBinary(metaBinary) + sink.writeString("\r\n") } //Printing data obj.data?.let { data -> //val actualSize: Int = envelope.data?.size ?: 0 - output.writeUtf8String(DATA_START + "\r\n") - output.writeBinary(data) + sink.writeString(DATA_START + "\r\n") + sink.writeBinary(data) } } - override fun readObject(input: Input): Envelope { + override fun readObject(source: Source): Envelope { //read preamble - input.discardWithSeparator( - TAGLESS_ENVELOPE_HEADER.encodeToByteArray(), + source.discardWithSeparator( + TAGLESS_ENVELOPE_HEADER, atMost = 1024, ) @@ -64,22 +65,22 @@ public class TaglessEnvelopeFormat( var data: Binary? = null - input.discardWithSeparator( + source.discardWithSeparator( SEPARATOR_PREFIX, atMost = 1024, ) var header: String = ByteArray { - input.readUTF8UntilDelimiterTo(this, "\n") + source.readWithSeparatorTo(this, "\n".encodeToByteString()) }.decodeToString() - while (!input.endOfInput) { + while (!source.exhausted()) { val block = ByteArray { - input.readWithSeparatorTo(this, SEPARATOR_PREFIX) + source.readWithSeparatorTo(this, SEPARATOR_PREFIX) } val nextHeader = ByteArray { - input.readWithSeparatorTo(this, "\n".encodeToByteArray()) + source.readWithSeparatorTo(this, "\n".encodeToByteString()) }.decodeToString() //terminate on end @@ -89,7 +90,7 @@ public class TaglessEnvelopeFormat( if (header.startsWith("META")) { //TODO check format val metaFormat: MetaFormatFactory = JsonMetaFormat - meta = metaFormat.readMeta(ByteReadPacket(block)) + meta = metaFormat.readMeta(ByteArraySource(block).buffered()) } if (header.startsWith("DATA")) { @@ -111,9 +112,9 @@ public class TaglessEnvelopeFormat( public const val TAGLESS_ENVELOPE_TYPE: String = "tagless" - public val SEPARATOR_PREFIX: ByteArray = "\n#~".encodeToByteArray() + public val SEPARATOR_PREFIX: ByteString = "\n#~".encodeToByteString() - public const val TAGLESS_ENVELOPE_HEADER: String = "#~DFTL" + public val TAGLESS_ENVELOPE_HEADER: ByteString = "#~DFTL".encodeToByteString() // public const val META_START_PROPERTY: String = "metaSeparator" public const val META_START: String = "#~META" @@ -134,21 +135,18 @@ public class TaglessEnvelopeFormat( override fun readObject(binary: Binary): Envelope = default.run { readObject(binary) } override fun writeObject( - output: Output, + sink: Sink, obj: Envelope, ): Unit = default.run { - writeObject( - output, - obj, - ) + writeObject(sink, obj) } - override fun readObject(input: Input): Envelope = default.readObject(input) + override fun readObject(source: Source): Envelope = default.readObject(source) override fun peekFormat(io: IOPlugin, binary: Binary): EnvelopeFormat? { return try { binary.read { - val string = readRawString(TAGLESS_ENVELOPE_HEADER.length) + val string = readByteString(TAGLESS_ENVELOPE_HEADER.size) return@read if (string == TAGLESS_ENVELOPE_HEADER) { TaglessEnvelopeFormat(io) } else { diff --git a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/ioMisc.kt b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/ioMisc.kt index c0132895..ff7f985d 100644 --- a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/ioMisc.kt +++ b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/ioMisc.kt @@ -2,14 +2,17 @@ package space.kscience.dataforge.io import kotlinx.io.* import kotlinx.io.bytestring.ByteString +import kotlinx.io.bytestring.decodeToString +import kotlinx.io.bytestring.encodeToByteString import space.kscience.dataforge.meta.Meta import space.kscience.dataforge.misc.DFExperimental +import kotlin.math.min /** * Convert a string literal, containing only ASCII characters to a [ByteString]. * Throws an error if there are non-ASCII characters. */ -public fun String.toACIIByteString(): ByteString { +public fun String.toAsciiByteString(): ByteString { val bytes = ByteArray(length) { val char = get(it) val code = char.code @@ -94,24 +97,30 @@ private class RingByteArray( else -> inputArray.indices.all { inputArray[it] == get(it) } } + fun contentEquals(byteString: ByteString): Boolean = when { + byteString.size != buffer.size -> false + size < buffer.size -> false + else -> (0 until byteString.size).all { byteString[it] == get(it) } + } + } private fun RingByteArray.toArray(): ByteArray = ByteArray(size) { get(it) } /** - * Read [Input] into [output] until designated multibyte [separator] and optionally continues until + * Read [Source] into [output] until designated multibyte [separator] and optionally continues until * the end of the line after it. Throw error if [separator] not found and [atMost] bytes are read. * Also fails if [separator] not found until the end of input. * - * Separator itself is not read into Output. + * The Separator itself is not read into [Sink]. * * @param errorOnEof if true error is thrown if separator is never encountered * * @return bytes actually being read, including separator */ public fun Source.readWithSeparatorTo( - output: Sink, - separator: ByteArray, + output: Sink?, + separator: ByteString, atMost: Int = Int.MAX_VALUE, errorOnEof: Boolean = false, ): Int { @@ -126,7 +135,7 @@ public fun Source.readWithSeparatorTo( if (rb.contentEquals(separator)) { return counter } else if (rb.isFull()) { - output.writeByte(rb[0]) + output?.writeByte(rb[0]) } } @@ -134,33 +143,93 @@ public fun Source.readWithSeparatorTo( error("Read to the end of input without encountering ${separator.decodeToString()}") } else { for (i in 1 until rb.size) { - output.writeByte(rb[i]) + output?.writeByte(rb[i]) } counter += (rb.size - 1) return counter } } -public fun Source.discardLine(): Int { - return discardUntilDelimiter('\n'.code.toByte()).also { - discard(1) - }.toInt() + 1 -} - +/** + * Discard all bytes until [separator] is encountered. Separator is discarded sa well. + * Return the total number of bytes read. + */ public fun Source.discardWithSeparator( - separator: ByteArray, + separator: ByteString, atMost: Int = Int.MAX_VALUE, errorOnEof: Boolean = false, -): Int { - val dummy: Sink = object : Sink(ChunkBuffer.Pool) { - override fun closeDestination() { - // Do nothing - } +): Int = readWithSeparatorTo(null, separator, atMost, errorOnEof) - override fun flush(source: Memory, offset: Int, length: Int) { - // Do nothing - } +/** + * Discard all symbol until newline is discovered. Carriage return is not discarded. + */ +public fun Source.discardLine( + atMost: Int = Int.MAX_VALUE, + errorOnEof: Boolean = false, +): Int = discardWithSeparator("\n".encodeToByteString(), atMost, errorOnEof) + + +/** + * A [Source] based on [ByteArray] + */ +public class ByteArraySource( + private val byteArray: ByteArray, + private val offset: Int = 0, + private val size: Int = byteArray.size - offset, +) : RawSource { + + init { + require(offset >= 0) { "Offset must be positive" } + require(offset + size <= byteArray.size) { "End index is ${offset + size}, but the array size is ${byteArray.size}" } } - return readWithSeparatorTo(dummy, separator, atMost, errorOnEof) + private var pointer = offset + + override fun close() { + // Do nothing + } + + override fun readAtMostTo(sink: Buffer, byteCount: Long): Long { + if (pointer == offset + size) return -1 + val byteRead = min(byteCount.toInt(), (size + offset - pointer)) + sink.write(byteArray, pointer, pointer + byteRead) + pointer += byteRead + return byteRead.toLong() + } } + +/** + * A [Source] based on [String] + */ +public class StringSource( + public val string: String, + public val offset: Int = 0, + public val size: Int = string.length - offset, +) : RawSource { + + private var pointer = offset + + override fun close() { + // Do nothing + } + + override fun readAtMostTo(sink: Buffer, byteCount: Long): Long { + if (pointer == offset + size) return -1 + val byteRead = min(byteCount.toInt(), (size + offset - pointer)) + sink.writeString(string, pointer, pointer + byteRead) + pointer += byteRead + return byteRead.toLong() + } +} + +public fun Sink.writeDouble(value: Double) { + writeLong(value.toBits()) +} + +public fun Source.readDouble(): Double = Double.fromBits(readLong()) + +public fun Sink.writeFloat(value: Float) { + writeInt(value.toBits()) +} + +public fun Source.readFloat(): Float = Float.fromBits(readInt()) \ No newline at end of file diff --git a/dataforge-io/src/commonTest/kotlin/space/kscience/dataforge/io/BinaryTest.kt b/dataforge-io/src/commonTest/kotlin/space/kscience/dataforge/io/BinaryTest.kt index 530b1a4c..250b8426 100644 --- a/dataforge-io/src/commonTest/kotlin/space/kscience/dataforge/io/BinaryTest.kt +++ b/dataforge-io/src/commonTest/kotlin/space/kscience/dataforge/io/BinaryTest.kt @@ -1,6 +1,5 @@ package space.kscience.dataforge.io -import io.ktor.utils.io.core.readInt import kotlin.test.Test import kotlin.test.assertEquals diff --git a/dataforge-io/src/commonTest/kotlin/space/kscience/dataforge/io/EnvelopeFormatTest.kt b/dataforge-io/src/commonTest/kotlin/space/kscience/dataforge/io/EnvelopeFormatTest.kt index 999f3175..3200e24b 100644 --- a/dataforge-io/src/commonTest/kotlin/space/kscience/dataforge/io/EnvelopeFormatTest.kt +++ b/dataforge-io/src/commonTest/kotlin/space/kscience/dataforge/io/EnvelopeFormatTest.kt @@ -1,6 +1,7 @@ package space.kscience.dataforge.io -import io.ktor.utils.io.core.readBytes +import kotlinx.io.readByteArray +import kotlinx.io.writeString import kotlin.test.Test import kotlin.test.assertEquals @@ -12,7 +13,7 @@ class EnvelopeFormatTest { "d" put 22.2 } data { - writeUtf8String("12345678") + writeString("12345678") } } @@ -22,7 +23,7 @@ class EnvelopeFormatTest { val res = readFromByteArray(byteArray) assertEquals(envelope.meta, res.meta) val bytes = res.data?.read { - readBytes() + readByteArray() } assertEquals("12345678", bytes?.decodeToString()) } @@ -34,13 +35,13 @@ class EnvelopeFormatTest { val res = readFromByteArray(byteArray) assertEquals(envelope.meta, res.meta) val bytes = res.data?.read { - readBytes() + readByteArray() } - assertEquals("12345678", bytes?.decodeToString()) + assertEquals("12345678", bytes?.decodeToString()) } @Test - fun testManualDftl(){ + fun testManualDftl() { val envelopeString = """ #~DFTL #~META @@ -56,8 +57,8 @@ class EnvelopeFormatTest { val res = TaglessEnvelopeFormat.readFromByteArray(envelopeString.encodeToByteArray()) assertEquals(envelope.meta, res.meta) val bytes = res.data?.read { - readBytes() + readByteArray() } - assertEquals("12345678", bytes?.decodeToString()) + assertEquals("12345678", bytes?.decodeToString()) } } \ No newline at end of file diff --git a/dataforge-io/src/commonTest/kotlin/space/kscience/dataforge/io/IOTest.kt b/dataforge-io/src/commonTest/kotlin/space/kscience/dataforge/io/IOTest.kt index f02066da..683df330 100644 --- a/dataforge-io/src/commonTest/kotlin/space/kscience/dataforge/io/IOTest.kt +++ b/dataforge-io/src/commonTest/kotlin/space/kscience/dataforge/io/IOTest.kt @@ -1,8 +1,9 @@ package space.kscience.dataforge.io -import io.ktor.utils.io.core.ByteReadPacket -import io.ktor.utils.io.core.readBytes -import io.ktor.utils.io.core.readUTF8Line +import kotlinx.io.buffered +import kotlinx.io.bytestring.encodeToByteString +import kotlinx.io.readByteArray +import kotlinx.io.readLine import kotlin.test.Test import kotlin.test.assertEquals import kotlin.test.assertFails @@ -11,9 +12,9 @@ class IOTest { @Test fun readBytes() { val bytes = ByteArray(8) { it.toByte() } - val input = ByteReadPacket(bytes) - @Suppress("UNUSED_VARIABLE") val first = input.readBytes(4) - val second = input.readBytes(4) + val input = ByteArraySource(bytes).buffered() + @Suppress("UNUSED_VARIABLE") val first = input.readByteArray(4) + val second = input.readByteArray(4) assertEquals(4.toByte(), second[0]) } @@ -31,25 +32,25 @@ class IOTest { binary.read { val array = ByteArray { - val read = readWithSeparatorTo(this, "---".encodeToByteArray()) + discardLine() + val read = readWithSeparatorTo(this, "---".encodeToByteString()) + discardLine() assertEquals(12, read) } assertEquals(""" aaa bbb """.trimIndent(),array.decodeToString().trim()) - assertEquals("ccc", readUTF8Line()?.trim()) + assertEquals("ccc", readLine()?.trim()) } assertFails { binary.read { - discardWithSeparator("---".encodeToByteArray(), atMost = 3 ) + discardWithSeparator("---".encodeToByteString(), atMost = 3 ) } } assertFails { binary.read{ - discardWithSeparator("-+-".encodeToByteArray(), errorOnEof = true) + discardWithSeparator("-+-".encodeToByteString(), errorOnEof = true) } } diff --git a/dataforge-io/src/commonTest/kotlin/space/kscience/dataforge/io/MultipartTest.kt b/dataforge-io/src/commonTest/kotlin/space/kscience/dataforge/io/MultipartTest.kt index 2412b969..6ad5d8e3 100644 --- a/dataforge-io/src/commonTest/kotlin/space/kscience/dataforge/io/MultipartTest.kt +++ b/dataforge-io/src/commonTest/kotlin/space/kscience/dataforge/io/MultipartTest.kt @@ -1,5 +1,6 @@ package space.kscience.dataforge.io +import kotlinx.io.writeString import space.kscience.dataforge.context.Global import space.kscience.dataforge.meta.get import space.kscience.dataforge.meta.int @@ -18,9 +19,9 @@ class MultipartTest { "value" put it } data { - writeUtf8String("Hello World $it") + writeString("Hello World $it") repeat(300) { - writeRawString("$it ") + writeString("$it ") } } } diff --git a/dataforge-io/src/commonTest/kotlin/space/kscience/dataforge/io/ioTestUtils.kt b/dataforge-io/src/commonTest/kotlin/space/kscience/dataforge/io/ioTestUtils.kt index 8eeb526b..4754f682 100644 --- a/dataforge-io/src/commonTest/kotlin/space/kscience/dataforge/io/ioTestUtils.kt +++ b/dataforge-io/src/commonTest/kotlin/space/kscience/dataforge/io/ioTestUtils.kt @@ -1,12 +1,11 @@ package space.kscience.dataforge.io -import io.ktor.utils.io.core.ByteReadPacket -import io.ktor.utils.io.core.use +import kotlinx.io.buffered fun IOFormat.writeToByteArray(obj: T): ByteArray = ByteArray { writeObject(this, obj) } -fun IOFormat.readFromByteArray(array: ByteArray): T = ByteReadPacket(array).use { +fun IOFormat.readFromByteArray(array: ByteArray): T = ByteArraySource(array).buffered().use { readObject(it) } \ No newline at end of file diff --git a/dataforge-io/src/jvmMain/kotlin/space/kscience/dataforge/io/fileIO.kt b/dataforge-io/src/jvmMain/kotlin/space/kscience/dataforge/io/fileIO.kt index cf88d3ae..3cc81b32 100644 --- a/dataforge-io/src/jvmMain/kotlin/space/kscience/dataforge/io/fileIO.kt +++ b/dataforge-io/src/jvmMain/kotlin/space/kscience/dataforge/io/fileIO.kt @@ -1,8 +1,11 @@ package space.kscience.dataforge.io -import io.ktor.utils.io.core.* -import io.ktor.utils.io.streams.asOutput + import kotlinx.coroutines.runBlocking +import kotlinx.io.Sink +import kotlinx.io.Source +import kotlinx.io.asSink +import kotlinx.io.buffered import space.kscience.dataforge.meta.Meta import space.kscience.dataforge.meta.descriptors.MetaDescriptor import space.kscience.dataforge.meta.isEmpty @@ -23,18 +26,18 @@ internal class PathBinary( override val size: Int = Files.size(path).toInt() - fileOffset, ) : Binary { - override fun read(offset: Int, atMost: Int, block: Input.() -> R): R = runBlocking { + override fun read(offset: Int, atMost: Int, block: Source.() -> R): R = runBlocking { readSuspend(offset, atMost, block) } - override suspend fun readSuspend(offset: Int, atMost: Int, block: suspend Input.() -> R): R { + override suspend fun readSuspend(offset: Int, atMost: Int, block: suspend Source.() -> R): R { val actualOffset = offset + fileOffset val actualSize = min(atMost, size - offset) val array = path.inputStream().use { it.skip(actualOffset.toLong()) it.readNBytes(actualSize) } - return ByteReadPacket(array).block() + return ByteArraySource(array).buffered().use { it.block() } } override fun view(offset: Int, binarySize: Int) = PathBinary(path, fileOffset + offset, binarySize) @@ -42,36 +45,36 @@ internal class PathBinary( public fun Path.asBinary(): Binary = PathBinary(this) -public fun Path.read(block: Input.() -> R): R = asBinary().read(block = block) +public fun Path.read(block: Source.() -> R): R = asBinary().read(block = block) /** * Write a live output to a newly created file. If file does not exist, throws error */ -public fun Path.write(block: Output.() -> Unit): Unit { +public fun Path.write(block: Sink.() -> Unit): Unit { val stream = Files.newOutputStream(this, StandardOpenOption.WRITE, StandardOpenOption.CREATE_NEW) - stream.asOutput().use(block) + stream.asSink().buffered().use(block) } /** * Create a new file or append to exiting one with given output [block] */ -public fun Path.append(block: Output.() -> Unit): Unit { +public fun Path.append(block: Sink.() -> Unit): Unit { val stream = Files.newOutputStream( this, StandardOpenOption.WRITE, StandardOpenOption.APPEND, StandardOpenOption.CREATE ) - stream.asOutput().use(block) + stream.asSink().buffered().use(block) } /** * Create a new file or replace existing one using given output [block] */ -public fun Path.rewrite(block: Output.() -> Unit): Unit { +public fun Path.rewrite(block: Sink.() -> Unit): Unit { val stream = Files.newOutputStream( this, StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.CREATE ) - stream.asOutput().use(block) + stream.asSink().buffered().use(block) } @DFExperimental @@ -260,7 +263,7 @@ public fun IOPlugin.writeEnvelopeDirectory( val dataFile = path.resolve(IOPlugin.DATA_FILE_NAME) dataFile.write { envelope.data?.read { - val copied = copyTo(this@write) + val copied = transferTo(this@write) if (copied != envelope.data?.size?.toLong()) { error("The number of copied bytes does not equal data size") } diff --git a/dataforge-io/src/jvmMain/kotlin/space/kscience/dataforge/io/resourceIO.kt b/dataforge-io/src/jvmMain/kotlin/space/kscience/dataforge/io/resourceIO.kt index f1997d21..2e12c3e1 100644 --- a/dataforge-io/src/jvmMain/kotlin/space/kscience/dataforge/io/resourceIO.kt +++ b/dataforge-io/src/jvmMain/kotlin/space/kscience/dataforge/io/resourceIO.kt @@ -1,9 +1,11 @@ package space.kscience.dataforge.io -import io.ktor.utils.io.core.Input -import io.ktor.utils.io.streams.asInput +import kotlinx.io.Source +import kotlinx.io.asSource +import kotlinx.io.buffered -public fun IOPlugin.resource(name: String): Binary? = context.javaClass.getResource(name)?.readBytes()?.asBinary() -public inline fun IOPlugin.readResource(name: String, block: Input.() -> R): R = - context.javaClass.getResource(name)?.openStream()?.asInput()?.block() ?: error("Can't read resource $name") \ No newline at end of file +public fun IOPlugin.resource(name: String): Binary? = { }.javaClass.getResource(name)?.readBytes()?.asBinary() + +public inline fun IOPlugin.readResource(name: String, block: Source.() -> R): R = + { }.javaClass.getResource(name)?.openStream()?.asSource()?.buffered()?.block() ?: error("Can't read resource $name") \ No newline at end of file diff --git a/dataforge-io/src/jvmTest/kotlin/space/kscience/dataforge/io/FileBinaryTest.kt b/dataforge-io/src/jvmTest/kotlin/space/kscience/dataforge/io/FileBinaryTest.kt index d03c1c1c..c50eee37 100644 --- a/dataforge-io/src/jvmTest/kotlin/space/kscience/dataforge/io/FileBinaryTest.kt +++ b/dataforge-io/src/jvmTest/kotlin/space/kscience/dataforge/io/FileBinaryTest.kt @@ -1,6 +1,5 @@ package space.kscience.dataforge.io -import io.ktor.utils.io.core.writeDouble import space.kscience.dataforge.context.Global import space.kscience.dataforge.misc.DFExperimental import java.nio.file.Files diff --git a/dataforge-io/src/jvmTest/kotlin/space/kscience/dataforge/io/FileEnvelopeTest.kt b/dataforge-io/src/jvmTest/kotlin/space/kscience/dataforge/io/FileEnvelopeTest.kt index ee5ea41f..d9e6d000 100644 --- a/dataforge-io/src/jvmTest/kotlin/space/kscience/dataforge/io/FileEnvelopeTest.kt +++ b/dataforge-io/src/jvmTest/kotlin/space/kscience/dataforge/io/FileEnvelopeTest.kt @@ -1,6 +1,5 @@ package space.kscience.dataforge.io -import io.ktor.utils.io.core.writeDouble import space.kscience.dataforge.context.Global import space.kscience.dataforge.misc.DFExperimental import java.nio.file.Files diff --git a/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/FileWorkspaceCache.kt b/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/FileWorkspaceCache.kt index fdcb49a6..1f725a60 100644 --- a/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/FileWorkspaceCache.kt +++ b/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/FileWorkspaceCache.kt @@ -1,9 +1,5 @@ package space.kscience.dataforge.workspace -import io.ktor.utils.io.core.Input -import io.ktor.utils.io.core.Output -import io.ktor.utils.io.core.readBytes -import io.ktor.utils.io.core.writeFully import kotlinx.serialization.ExperimentalSerializationApi import kotlinx.serialization.KSerializer import kotlinx.serialization.json.Json From a136db16ff9fb0de938d124ac0fe9f3408ccc030 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Sat, 15 Jul 2023 10:49:42 +0300 Subject: [PATCH 3/3] Refactoring to kotlinx-io complete --- dataforge-workspace/build.gradle.kts | 36 ++++++++++++++----- .../dataforge/workspace/FileWorkspaceCache.kt | 13 +++---- .../kscience/dataforge/workspace/zipData.kt | 15 ++++---- .../dataforge/workspace/FileDataTest.kt | 19 +++++----- gradle.properties | 1 + gradle/wrapper/gradle-wrapper.properties | 2 +- 6 files changed, 55 insertions(+), 31 deletions(-) diff --git a/dataforge-workspace/build.gradle.kts b/dataforge-workspace/build.gradle.kts index 28a633c8..9f10df98 100644 --- a/dataforge-workspace/build.gradle.kts +++ b/dataforge-workspace/build.gradle.kts @@ -10,17 +10,35 @@ kscience{ useSerialization{ protobuf() } - dependencies { - api(projects.dataforgeContext) - api(projects.dataforgeData) - api(projects.dataforgeIo) - } - dependencies(jvmTest){ - implementation(spclibs.logback.classic) - implementation(projects.dataforgeIo.dataforgeIoYaml) - } +// dependencies { +// api(projects.dataforgeContext) +// api(projects.dataforgeData) +// api(projects.dataforgeIo) +// } +// dependencies(jvmTest){ +// implementation(spclibs.logback.classic) +// implementation(projects.dataforgeIo.dataforgeIoYaml) +// } } readme{ maturity = space.kscience.gradle.Maturity.EXPERIMENTAL +} + +kotlin{ + sourceSets{ + commonMain{ + dependencies { + api(projects.dataforgeContext) + api(projects.dataforgeData) + api(projects.dataforgeIo) + } + } + getByName("jvmTest"){ + dependencies { + implementation(spclibs.logback.classic) + implementation(projects.dataforgeIo.dataforgeIoYaml) + } + } + } } \ No newline at end of file diff --git a/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/FileWorkspaceCache.kt b/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/FileWorkspaceCache.kt index 1f725a60..a637ad97 100644 --- a/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/FileWorkspaceCache.kt +++ b/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/FileWorkspaceCache.kt @@ -1,5 +1,6 @@ package space.kscience.dataforge.workspace +import kotlinx.io.* import kotlinx.serialization.ExperimentalSerializationApi import kotlinx.serialization.KSerializer import kotlinx.serialization.json.Json @@ -26,10 +27,10 @@ public class JsonIOFormat(override val type: KType) : IOFormat { @Suppress("UNCHECKED_CAST") private val serializer: KSerializer = serializer(type) as KSerializer - override fun readObject(input: Input): T = Json.decodeFromString(serializer, input.readUtf8String()) + override fun readObject(input: Source): T = Json.decodeFromString(serializer, input.readString()) - override fun writeObject(output: Output, obj: T) { - output.writeUtf8String(Json.encodeToString(serializer, obj)) + override fun writeObject(output: Sink, obj: T) { + output.writeString(Json.encodeToString(serializer, obj)) } } @@ -39,10 +40,10 @@ public class ProtobufIOFormat(override val type: KType) : IOFormat { @Suppress("UNCHECKED_CAST") private val serializer: KSerializer = serializer(type) as KSerializer - override fun readObject(input: Input): T = ProtoBuf.decodeFromByteArray(serializer, input.readBytes()) + override fun readObject(input: Source): T = ProtoBuf.decodeFromByteArray(serializer, input.readByteArray()) - override fun writeObject(output: Output, obj: T) { - output.writeFully(ProtoBuf.encodeToByteArray(serializer, obj)) + override fun writeObject(output: Sink, obj: T) { + output.write(ProtoBuf.encodeToByteArray(serializer, obj)) } } diff --git a/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/zipData.kt b/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/zipData.kt index 706a2d0b..d23753a2 100644 --- a/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/zipData.kt +++ b/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/zipData.kt @@ -1,13 +1,10 @@ package space.kscience.dataforge.workspace -import io.ktor.utils.io.streams.asOutput import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.withContext import space.kscience.dataforge.data.DataTree import space.kscience.dataforge.data.DataTreeItem -import space.kscience.dataforge.io.EnvelopeFormat -import space.kscience.dataforge.io.IOFormat -import space.kscience.dataforge.io.TaggedEnvelopeFormat +import space.kscience.dataforge.io.* import space.kscience.dataforge.misc.DFExperimental import java.nio.file.Files import java.nio.file.Path @@ -28,11 +25,15 @@ private suspend fun ZipOutputStream.writeNode( val envelope = treeItem.data.toEnvelope(dataFormat) val entry = ZipEntry(name) putNextEntry(entry) - asOutput().run { - envelopeFormat.writeObject(this, envelope) - flush() + + //TODO remove additional copy + val bytes = ByteArray { + writeObject(envelopeFormat, envelope) } + write(bytes) + } + is DataTreeItem.Node -> { val entry = ZipEntry("$name/") putNextEntry(entry) diff --git a/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/FileDataTest.kt b/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/FileDataTest.kt index 6ec31e0c..313ea82e 100644 --- a/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/FileDataTest.kt +++ b/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/FileDataTest.kt @@ -1,8 +1,11 @@ package space.kscience.dataforge.workspace -import io.ktor.utils.io.core.Input -import io.ktor.utils.io.core.Output import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.test.runTest +import kotlinx.io.Sink +import kotlinx.io.Source +import kotlinx.io.readString +import kotlinx.io.writeString import space.kscience.dataforge.context.Context import space.kscience.dataforge.context.Global import space.kscience.dataforge.data.* @@ -36,11 +39,11 @@ class FileDataTest { object StringIOFormat : IOFormat { override val type: KType get() = typeOf() - override fun writeObject(output: Output, obj: String) { - output.writeUtf8String(obj) + override fun writeObject(output: Sink, obj: String) { + output.writeString(obj) } - override fun readObject(input: Input): String = input.readUtf8String() + override fun readObject(input: Source): String = input.readString() } @Test @@ -59,9 +62,9 @@ class FileDataTest { @Test @DFExperimental - fun testZipWriteRead() = with(Global.io) { - val zip = Files.createTempFile("df_data_node", ".zip") - runBlocking { + fun testZipWriteRead() = runTest { + with(Global.io) { + val zip = Files.createTempFile("df_data_node", ".zip") dataNode.writeZip(zip, StringIOFormat) println(zip.toUri().toString()) val reconstructed = readDataDirectory(zip) { _, _ -> StringIOFormat } diff --git a/gradle.properties b/gradle.properties index df992b2c..d26973e8 100644 --- a/gradle.properties +++ b/gradle.properties @@ -7,3 +7,4 @@ kotlin.incremental.js.ir=true kotlin.native.ignoreDisabledTargets=true toolsVersion=0.14.9-kotlin-1.9.0 +kotlin.experimental.tryK2=true \ No newline at end of file diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 15de9024..84a0b92f 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-8.2-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.2.1-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists