Compare commits
No commits in common. "259b882e63764c6e1737f16f2a64fb2b69745b8b" and "3806f97c77e4c41ffe11f0228015388cb87ba21f" have entirely different histories.
259b882e63
...
3806f97c77
@ -5,8 +5,6 @@
|
||||
### Added
|
||||
|
||||
### Changed
|
||||
- Kotlin 1.9.20.
|
||||
- Migrated from ktor-io to kotlinx-io.
|
||||
|
||||
### Deprecated
|
||||
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
|
||||
import space.kscience.gradle.isInDevelopment
|
||||
import space.kscience.gradle.useApache2Licence
|
||||
import space.kscience.gradle.useSPCTeam
|
||||
|
||||
@ -8,7 +9,7 @@ plugins {
|
||||
|
||||
allprojects {
|
||||
group = "space.kscience"
|
||||
version = "0.6.3-dev-1"
|
||||
version = "0.6.2"
|
||||
}
|
||||
|
||||
subprojects {
|
||||
@ -30,7 +31,14 @@ ksciencePublish {
|
||||
useApache2Licence()
|
||||
useSPCTeam()
|
||||
}
|
||||
repository("spc","https://maven.sciprog.center/kscience")
|
||||
github("dataforge-core", "SciProgCentre")
|
||||
space(
|
||||
if (isInDevelopment) {
|
||||
"https://maven.pkg.jetbrains.space/spc/p/sci/dev"
|
||||
} else {
|
||||
"https://maven.pkg.jetbrains.space/spc/p/sci/maven"
|
||||
}
|
||||
)
|
||||
sonatype()
|
||||
}
|
||||
|
||||
|
||||
@ -1,11 +1,11 @@
|
||||
import space.kscience.gradle.KScienceVersions
|
||||
|
||||
plugins {
|
||||
id("space.kscience.gradle.mpp")
|
||||
}
|
||||
|
||||
description = "IO module"
|
||||
|
||||
val ioVersion = "0.2.1"
|
||||
|
||||
kscience {
|
||||
jvm()
|
||||
js()
|
||||
@ -15,12 +15,11 @@ kscience {
|
||||
cbor()
|
||||
}
|
||||
dependencies {
|
||||
api(projects.dataforgeContext)
|
||||
api("org.jetbrains.kotlinx:kotlinx-io-core:$ioVersion")
|
||||
api("org.jetbrains.kotlinx:kotlinx-io-bytestring:$ioVersion")
|
||||
api(project(":dataforge-context"))
|
||||
api("io.ktor:ktor-io:${KScienceVersions.ktorVersion}")
|
||||
}
|
||||
}
|
||||
|
||||
readme{
|
||||
maturity = space.kscience.gradle.Maturity.EXPERIMENTAL
|
||||
readme {
|
||||
maturity = space.kscience.gradle.Maturity.PROTOTYPE
|
||||
}
|
||||
@ -1,11 +1,7 @@
|
||||
package space.kscience.dataforge.io.yaml
|
||||
|
||||
import kotlinx.io.Sink
|
||||
import kotlinx.io.Source
|
||||
import kotlinx.io.bytestring.ByteString
|
||||
import kotlinx.io.bytestring.encodeToByteString
|
||||
import kotlinx.io.readByteString
|
||||
import kotlinx.io.writeString
|
||||
import io.ktor.utils.io.core.Input
|
||||
import io.ktor.utils.io.core.Output
|
||||
import space.kscience.dataforge.context.Context
|
||||
import space.kscience.dataforge.context.Global
|
||||
import space.kscience.dataforge.io.*
|
||||
@ -23,18 +19,18 @@ public class FrontMatterEnvelopeFormat(
|
||||
var offset = 0
|
||||
|
||||
offset += discardWithSeparator(
|
||||
SEPARATOR,
|
||||
SEPARATOR.encodeToByteArray(),
|
||||
atMost = 1024,
|
||||
)
|
||||
|
||||
val line = ByteArray {
|
||||
offset += readWithSeparatorTo(this, "\n".encodeToByteString())
|
||||
offset += readWithSeparatorTo(this, "\n".encodeToByteArray())
|
||||
}.decodeToString()
|
||||
|
||||
val readMetaFormat = line.trim().takeIf { it.isNotBlank() }?.let { io.resolveMetaFormat(it) } ?: YamlMetaFormat
|
||||
|
||||
val packet = ByteArray {
|
||||
offset += readWithSeparatorTo(this, SEPARATOR)
|
||||
offset += readWithSeparatorTo(this, SEPARATOR.encodeToByteArray())
|
||||
}
|
||||
|
||||
offset += discardLine()
|
||||
@ -43,25 +39,25 @@ public class FrontMatterEnvelopeFormat(
|
||||
Envelope(meta, binary.view(offset))
|
||||
}
|
||||
|
||||
override fun readObject(source: Source): Envelope = readObject(source.readBinary())
|
||||
override fun readObject(input: Input): Envelope = readObject(input.readBinary())
|
||||
|
||||
override fun writeObject(
|
||||
sink: Sink,
|
||||
output: Output,
|
||||
obj: Envelope,
|
||||
) {
|
||||
val metaFormat = metaFormatFactory.build(io.context, meta)
|
||||
val formatSuffix = if (metaFormat is YamlMetaFormat) "" else metaFormatFactory.shortName
|
||||
sink.writeString("$SEPARATOR${formatSuffix}\r\n")
|
||||
metaFormat.run { metaFormat.writeObject(sink, obj.meta) }
|
||||
sink.writeString("$SEPARATOR\r\n")
|
||||
output.writeRawString("$SEPARATOR${formatSuffix}\r\n")
|
||||
metaFormat.run { metaFormat.writeObject(output, obj.meta) }
|
||||
output.writeRawString("$SEPARATOR\r\n")
|
||||
//Printing data
|
||||
obj.data?.let { data ->
|
||||
sink.writeBinary(data)
|
||||
output.writeBinary(data)
|
||||
}
|
||||
}
|
||||
|
||||
public companion object : EnvelopeFormatFactory {
|
||||
public val SEPARATOR: ByteString = "---".encodeToByteString()
|
||||
public const val SEPARATOR: String = "---"
|
||||
|
||||
private val metaTypeRegex = "---(\\w*)\\s*".toRegex()
|
||||
|
||||
@ -73,8 +69,8 @@ public class FrontMatterEnvelopeFormat(
|
||||
|
||||
override fun peekFormat(io: IOPlugin, binary: Binary): EnvelopeFormat? = binary.read {
|
||||
//read raw string to avoid UTF issues
|
||||
val line = readByteString(3)
|
||||
return@read if (line == "---".encodeToByteString()) {
|
||||
val line = readRawString(3)
|
||||
return@read if (line == "---") {
|
||||
default
|
||||
} else {
|
||||
null
|
||||
@ -86,12 +82,12 @@ public class FrontMatterEnvelopeFormat(
|
||||
override fun readObject(binary: Binary): Envelope = default.readObject(binary)
|
||||
|
||||
override fun writeObject(
|
||||
sink: Sink,
|
||||
output: Output,
|
||||
obj: Envelope,
|
||||
): Unit = default.writeObject(sink, obj)
|
||||
): Unit = default.writeObject(output, obj)
|
||||
|
||||
|
||||
override fun readObject(source: Source): Envelope = default.readObject(source)
|
||||
override fun readObject(input: Input): Envelope = default.readObject(input)
|
||||
|
||||
}
|
||||
}
|
||||
@ -1,13 +1,13 @@
|
||||
package space.kscience.dataforge.io.yaml
|
||||
|
||||
import kotlinx.io.Sink
|
||||
import kotlinx.io.Source
|
||||
import kotlinx.io.readString
|
||||
import kotlinx.io.writeString
|
||||
import io.ktor.utils.io.core.Input
|
||||
import io.ktor.utils.io.core.Output
|
||||
import net.mamoe.yamlkt.*
|
||||
import space.kscience.dataforge.context.Context
|
||||
import space.kscience.dataforge.io.MetaFormat
|
||||
import space.kscience.dataforge.io.MetaFormatFactory
|
||||
import space.kscience.dataforge.io.readUtf8String
|
||||
import space.kscience.dataforge.io.writeUtf8String
|
||||
import space.kscience.dataforge.meta.*
|
||||
import space.kscience.dataforge.meta.descriptors.MetaDescriptor
|
||||
import space.kscience.dataforge.meta.descriptors.get
|
||||
@ -89,14 +89,14 @@ public fun YamlMap.toMeta(): Meta = YamlMeta(this)
|
||||
*/
|
||||
public class YamlMetaFormat(private val meta: Meta) : MetaFormat {
|
||||
|
||||
override fun writeMeta(sink: Sink, meta: Meta, descriptor: MetaDescriptor?) {
|
||||
override fun writeMeta(output: Output, meta: Meta, descriptor: MetaDescriptor?) {
|
||||
val yaml: YamlMap = meta.toYaml()
|
||||
val string = Yaml.encodeToString(YamlMap.serializer(), yaml)
|
||||
sink.writeString(string)
|
||||
output.writeUtf8String(string)
|
||||
}
|
||||
|
||||
override fun readMeta(source: Source, descriptor: MetaDescriptor?): Meta {
|
||||
val yaml = Yaml.decodeYamlMapFromString(source.readString())
|
||||
override fun readMeta(input: Input, descriptor: MetaDescriptor?): Meta {
|
||||
val yaml = Yaml.decodeYamlMapFromString(input.readUtf8String())
|
||||
return yaml.toMeta()
|
||||
}
|
||||
|
||||
@ -109,10 +109,10 @@ public class YamlMetaFormat(private val meta: Meta) : MetaFormat {
|
||||
|
||||
private val default = YamlMetaFormat(Meta.EMPTY)
|
||||
|
||||
override fun writeMeta(sink: Sink, meta: Meta, descriptor: MetaDescriptor?): Unit =
|
||||
default.writeMeta(sink, meta, descriptor)
|
||||
override fun writeMeta(output: Output, meta: Meta, descriptor: MetaDescriptor?): Unit =
|
||||
default.writeMeta(output, meta, descriptor)
|
||||
|
||||
override fun readMeta(source: Source, descriptor: MetaDescriptor?): Meta =
|
||||
default.readMeta(source, descriptor)
|
||||
override fun readMeta(input: Input, descriptor: MetaDescriptor?): Meta =
|
||||
default.readMeta(input, descriptor)
|
||||
}
|
||||
}
|
||||
@ -1,9 +1,6 @@
|
||||
package space.kscience.dataforge.io
|
||||
|
||||
import kotlinx.io.Sink
|
||||
import kotlinx.io.Source
|
||||
import kotlinx.io.buffered
|
||||
import kotlinx.io.readByteArray
|
||||
import io.ktor.utils.io.core.*
|
||||
import kotlin.math.min
|
||||
|
||||
/**
|
||||
@ -20,13 +17,13 @@ public interface Binary {
|
||||
* Read maximum of [atMost] bytes as input from the binary, starting at [offset]. The generated input is always closed
|
||||
* when leaving scope, so it could not be leaked outside of scope of [block].
|
||||
*/
|
||||
public fun <R> read(offset: Int = 0, atMost: Int = size - offset, block: Source.() -> R): R
|
||||
public fun <R> read(offset: Int = 0, atMost: Int = size - offset, block: Input.() -> R): R
|
||||
|
||||
public suspend fun <R> readSuspend(offset: Int = 0, atMost: Int = size - offset, block: suspend Source.() -> R): R
|
||||
public suspend fun <R> readSuspend(offset: Int = 0, atMost: Int = size - offset, block: suspend Input.() -> R): R
|
||||
|
||||
/**
|
||||
* Read a binary with given [offset] relative to this binary and given [binarySize].
|
||||
* In general, the resulting binary is of the same type as this one, but it is not guaranteed.
|
||||
* In general, resulting binary is of the same type as this one, but it is not guaranteed.
|
||||
*/
|
||||
public fun view(offset: Int, binarySize: Int = size - offset): Binary
|
||||
|
||||
@ -41,27 +38,25 @@ internal class ByteArrayBinary(
|
||||
override val size: Int = array.size - start,
|
||||
) : Binary {
|
||||
|
||||
override fun <R> read(offset: Int, atMost: Int, block: Source.() -> R): R {
|
||||
override fun <R> read(offset: Int, atMost: Int, block: Input.() -> R): R {
|
||||
require(offset >= 0) { "Offset must be positive" }
|
||||
require(offset < array.size) { "Offset $offset is larger than array size" }
|
||||
|
||||
return ByteArraySource(
|
||||
val input = ByteReadPacket(
|
||||
array,
|
||||
offset + start,
|
||||
min(atMost, size - offset)
|
||||
).buffered().use(block)
|
||||
)
|
||||
return input.use(block)
|
||||
}
|
||||
|
||||
override suspend fun <R> readSuspend(offset: Int, atMost: Int, block: suspend Source.() -> R): R {
|
||||
override suspend fun <R> readSuspend(offset: Int, atMost: Int, block: suspend Input.() -> R): R {
|
||||
require(offset >= 0) { "Offset must be positive" }
|
||||
require(offset < array.size) { "Offset $offset is larger than array size" }
|
||||
|
||||
val input = ByteArraySource(
|
||||
val input = ByteReadPacket(
|
||||
array,
|
||||
offset + start,
|
||||
min(atMost, size - offset)
|
||||
).buffered()
|
||||
|
||||
)
|
||||
return try {
|
||||
block(input)
|
||||
} finally {
|
||||
@ -82,26 +77,26 @@ public fun Binary.toByteArray(): ByteArray = if (this is ByteArrayBinary) {
|
||||
array.copyOfRange(start, start + size) // TODO do we need to ensure data safety here?
|
||||
} else {
|
||||
read {
|
||||
readByteArray()
|
||||
readBytes()
|
||||
}
|
||||
}
|
||||
|
||||
//TODO optimize for file-based Inputs
|
||||
public fun Source.readBinary(size: Int? = null): Binary {
|
||||
val array = if (size == null) readByteArray() else readByteArray(size)
|
||||
public fun Input.readBinary(size: Int? = null): Binary {
|
||||
val array = if (size == null) readBytes() else readBytes(size)
|
||||
return ByteArrayBinary(array)
|
||||
}
|
||||
|
||||
/**
|
||||
* Direct write of binary to the output. Returns the number of bytes written
|
||||
*/
|
||||
public fun Sink.writeBinary(binary: Binary): Int {
|
||||
public fun Output.writeBinary(binary: Binary): Int {
|
||||
return if (binary is ByteArrayBinary) {
|
||||
write(binary.array, binary.start, binary.start + binary.size)
|
||||
writeFully(binary.array, binary.start, binary.start + binary.size)
|
||||
binary.size
|
||||
} else {
|
||||
binary.read {
|
||||
transferTo(this@writeBinary).toInt()
|
||||
copyTo(this@writeBinary).toInt()
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,6 +1,6 @@
|
||||
package space.kscience.dataforge.io
|
||||
|
||||
import kotlinx.io.Sink
|
||||
import io.ktor.utils.io.core.Output
|
||||
import space.kscience.dataforge.meta.*
|
||||
|
||||
public class EnvelopeBuilder : Envelope {
|
||||
@ -33,7 +33,7 @@ public class EnvelopeBuilder : Envelope {
|
||||
/**
|
||||
* Construct a data binary from given builder
|
||||
*/
|
||||
public inline fun data(block: Sink.() -> Unit) {
|
||||
public inline fun data(block: Output.() -> Unit) {
|
||||
data = ByteArray { block() }.asBinary()
|
||||
}
|
||||
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
package space.kscience.dataforge.io
|
||||
|
||||
import kotlinx.io.Source
|
||||
import io.ktor.utils.io.core.Input
|
||||
import space.kscience.dataforge.context.Context
|
||||
import space.kscience.dataforge.io.EnvelopeFormatFactory.Companion.ENVELOPE_FORMAT_TYPE
|
||||
import space.kscience.dataforge.meta.Meta
|
||||
@ -15,7 +15,7 @@ public interface EnvelopeFormat : IOFormat<Envelope> {
|
||||
override val type: KType get() = typeOf<Envelope>()
|
||||
}
|
||||
|
||||
public fun EnvelopeFormat.read(input: Source): Envelope = readObject(input)
|
||||
public fun EnvelopeFormat.read(input: Input): Envelope = readObject(input)
|
||||
|
||||
@Type(ENVELOPE_FORMAT_TYPE)
|
||||
public interface EnvelopeFormatFactory : IOFormatFactory<Envelope>, EnvelopeFormat {
|
||||
|
||||
@ -1,8 +1,5 @@
|
||||
package space.kscience.dataforge.io
|
||||
|
||||
import kotlinx.io.bytestring.ByteString
|
||||
import kotlinx.io.bytestring.decodeToString
|
||||
import kotlinx.io.write
|
||||
import space.kscience.dataforge.io.Envelope.Companion.ENVELOPE_NODE_KEY
|
||||
import space.kscience.dataforge.io.PartDescriptor.Companion.DEFAULT_MULTIPART_DATA_SEPARATOR
|
||||
import space.kscience.dataforge.io.PartDescriptor.Companion.MULTIPART_DATA_TYPE
|
||||
@ -23,7 +20,7 @@ private class PartDescriptor : Scheme() {
|
||||
val PARTS_KEY = MULTIPART_KEY + "parts"
|
||||
val SEPARATOR_KEY = MULTIPART_KEY + "separator"
|
||||
|
||||
val DEFAULT_MULTIPART_DATA_SEPARATOR = "\r\n#~PART~#\r\n".toAsciiByteString()
|
||||
const val DEFAULT_MULTIPART_DATA_SEPARATOR = "\r\n#~PART~#\r\n"
|
||||
|
||||
const val MULTIPART_DATA_TYPE = "envelope.multipart"
|
||||
}
|
||||
@ -35,12 +32,12 @@ public typealias EnvelopeParts = List<EnvelopePart>
|
||||
|
||||
public fun EnvelopeBuilder.multipart(
|
||||
parts: EnvelopeParts,
|
||||
separator: ByteString = DEFAULT_MULTIPART_DATA_SEPARATOR,
|
||||
separator: String = DEFAULT_MULTIPART_DATA_SEPARATOR,
|
||||
) {
|
||||
dataType = MULTIPART_DATA_TYPE
|
||||
|
||||
var offsetCounter = 0
|
||||
val separatorSize = separator.size
|
||||
val separatorSize = separator.length
|
||||
val partDescriptors = parts.map { (binary, description) ->
|
||||
offsetCounter += separatorSize
|
||||
PartDescriptor {
|
||||
@ -54,14 +51,14 @@ public fun EnvelopeBuilder.multipart(
|
||||
|
||||
meta {
|
||||
if (separator != DEFAULT_MULTIPART_DATA_SEPARATOR) {
|
||||
SEPARATOR_KEY put separator.decodeToString()
|
||||
SEPARATOR_KEY put separator
|
||||
}
|
||||
setIndexed(PARTS_KEY, partDescriptors.map { it.toMeta() })
|
||||
}
|
||||
|
||||
data {
|
||||
parts.forEach {
|
||||
write(separator)
|
||||
writeRawString(separator)
|
||||
writeBinary(it.binary)
|
||||
}
|
||||
}
|
||||
@ -72,7 +69,7 @@ public fun EnvelopeBuilder.multipart(
|
||||
*/
|
||||
public fun EnvelopeBuilder.envelopes(
|
||||
envelopes: List<Envelope>,
|
||||
separator: ByteString = DEFAULT_MULTIPART_DATA_SEPARATOR,
|
||||
separator: String = DEFAULT_MULTIPART_DATA_SEPARATOR,
|
||||
) {
|
||||
val parts = envelopes.map {
|
||||
val binary = Binary(it, TaggedEnvelopeFormat)
|
||||
|
||||
@ -1,8 +1,6 @@
|
||||
package space.kscience.dataforge.io
|
||||
|
||||
import kotlinx.io.Sink
|
||||
import kotlinx.io.Source
|
||||
import kotlinx.io.readByteArray
|
||||
import io.ktor.utils.io.core.*
|
||||
import space.kscience.dataforge.context.Context
|
||||
import space.kscience.dataforge.context.Factory
|
||||
import space.kscience.dataforge.io.IOFormatFactory.Companion.IO_FORMAT_TYPE
|
||||
@ -23,7 +21,7 @@ public interface IOReader<out T> {
|
||||
*/
|
||||
public val type: KType
|
||||
|
||||
public fun readObject(source: Source): T
|
||||
public fun readObject(input: Input): T
|
||||
|
||||
public fun readObject(binary: Binary): T = binary.read { readObject(this) }
|
||||
|
||||
@ -34,21 +32,21 @@ public interface IOReader<out T> {
|
||||
public val binary: IOReader<Binary> = object : IOReader<Binary> {
|
||||
override val type: KType = typeOf<Binary>()
|
||||
|
||||
override fun readObject(source: Source): Binary = source.readByteArray().asBinary()
|
||||
override fun readObject(input: Input): Binary = input.readBytes().asBinary()
|
||||
|
||||
override fun readObject(binary: Binary): Binary = binary
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public inline fun <reified T> IOReader(crossinline read: Source.() -> T): IOReader<T> = object : IOReader<T> {
|
||||
public inline fun <reified T> IOReader(crossinline read: Input.() -> T): IOReader<T> = object : IOReader<T> {
|
||||
override val type: KType = typeOf<T>()
|
||||
|
||||
override fun readObject(source: Source): T = source.read()
|
||||
override fun readObject(input: Input): T = input.read()
|
||||
}
|
||||
|
||||
public fun interface IOWriter<in T> {
|
||||
public fun writeObject(sink: Sink, obj: T)
|
||||
public fun writeObject(output: Output, obj: T)
|
||||
}
|
||||
|
||||
/**
|
||||
@ -56,20 +54,20 @@ public fun interface IOWriter<in T> {
|
||||
*/
|
||||
public interface IOFormat<T> : IOReader<T>, IOWriter<T>
|
||||
|
||||
public fun <T : Any> Source.readObject(format: IOReader<T>): T = format.readObject(this@readObject)
|
||||
public fun <T : Any> Input.readObject(format: IOReader<T>): T = format.readObject(this@readObject)
|
||||
|
||||
public fun <T : Any> IOFormat<T>.readObjectFrom(binary: Binary): T = binary.read {
|
||||
readObject(this)
|
||||
}
|
||||
|
||||
/**
|
||||
* Read given binary as an object using given format
|
||||
* Read given binary as object using given format
|
||||
*/
|
||||
public fun <T : Any> Binary.readWith(format: IOReader<T>): T = read {
|
||||
readObject(format)
|
||||
}
|
||||
|
||||
public fun <T : Any> Sink.writeObject(format: IOWriter<T>, obj: T): Unit =
|
||||
public fun <T : Any> Output.writeObject(format: IOWriter<T>, obj: T): Unit =
|
||||
format.writeObject(this@writeObject, obj)
|
||||
|
||||
|
||||
@ -96,9 +94,9 @@ public object DoubleIOFormat : IOFormat<Double>, IOFormatFactory<Double> {
|
||||
|
||||
override val type: KType get() = typeOf<Double>()
|
||||
|
||||
override fun writeObject(sink: Sink, obj: Double) {
|
||||
sink.writeLong(obj.toBits())
|
||||
override fun writeObject(output: Output, obj: Double) {
|
||||
output.writeDouble(obj)
|
||||
}
|
||||
|
||||
override fun readObject(source: Source): Double = Double.fromBits(source.readLong())
|
||||
override fun readObject(input: Input): Double = input.readDouble()
|
||||
}
|
||||
@ -1,10 +1,10 @@
|
||||
@file:Suppress("UNUSED_PARAMETER")
|
||||
|
||||
package space.kscience.dataforge.io
|
||||
|
||||
|
||||
import kotlinx.io.Sink
|
||||
import kotlinx.io.Source
|
||||
import kotlinx.io.readString
|
||||
import kotlinx.io.writeString
|
||||
import io.ktor.utils.io.core.Input
|
||||
import io.ktor.utils.io.core.Output
|
||||
import kotlinx.serialization.json.Json
|
||||
import kotlinx.serialization.json.JsonElement
|
||||
import space.kscience.dataforge.context.Context
|
||||
@ -18,13 +18,13 @@ import space.kscience.dataforge.meta.toMeta
|
||||
*/
|
||||
public class JsonMetaFormat(private val json: Json = DEFAULT_JSON) : MetaFormat {
|
||||
|
||||
override fun writeMeta(sink: Sink, meta: Meta, descriptor: MetaDescriptor?) {
|
||||
override fun writeMeta(output: Output, meta: Meta, descriptor: MetaDescriptor?) {
|
||||
val jsonElement = meta.toJson(descriptor)
|
||||
sink.writeString(json.encodeToString(JsonElement.serializer(), jsonElement))
|
||||
output.writeUtf8String(json.encodeToString(JsonElement.serializer(), jsonElement))
|
||||
}
|
||||
|
||||
override fun readMeta(source: Source, descriptor: MetaDescriptor?): Meta {
|
||||
val str = source.readString()
|
||||
override fun readMeta(input: Input, descriptor: MetaDescriptor?): Meta {
|
||||
val str = input.readUtf8String()//readByteArray().decodeToString()
|
||||
val jsonElement = json.parseToJsonElement(str)
|
||||
return jsonElement.toMeta(descriptor)
|
||||
}
|
||||
@ -39,10 +39,10 @@ public class JsonMetaFormat(private val json: Json = DEFAULT_JSON) : MetaFormat
|
||||
|
||||
private val default = JsonMetaFormat()
|
||||
|
||||
override fun writeMeta(sink: Sink, meta: Meta, descriptor: MetaDescriptor?): Unit =
|
||||
default.run { writeMeta(sink, meta, descriptor) }
|
||||
override fun writeMeta(output: Output, meta: Meta, descriptor: MetaDescriptor?): Unit =
|
||||
default.run { writeMeta(output, meta, descriptor) }
|
||||
|
||||
override fun readMeta(source: Source, descriptor: MetaDescriptor?): Meta =
|
||||
default.run { readMeta(source, descriptor) }
|
||||
override fun readMeta(input: Input, descriptor: MetaDescriptor?): Meta =
|
||||
default.run { readMeta(input, descriptor) }
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,9 +1,9 @@
|
||||
package space.kscience.dataforge.io
|
||||
|
||||
|
||||
import kotlinx.io.Sink
|
||||
import kotlinx.io.Source
|
||||
import kotlinx.io.buffered
|
||||
import io.ktor.utils.io.core.ByteReadPacket
|
||||
import io.ktor.utils.io.core.Input
|
||||
import io.ktor.utils.io.core.Output
|
||||
import io.ktor.utils.io.core.use
|
||||
import space.kscience.dataforge.context.Context
|
||||
import space.kscience.dataforge.context.Global
|
||||
import space.kscience.dataforge.io.MetaFormatFactory.Companion.META_FORMAT_TYPE
|
||||
@ -23,19 +23,19 @@ public interface MetaFormat : IOFormat<Meta> {
|
||||
|
||||
override val type: KType get() = typeOf<Meta>()
|
||||
|
||||
override fun writeObject(sink: Sink, obj: Meta) {
|
||||
writeMeta(sink, obj, null)
|
||||
override fun writeObject(output: Output, obj: Meta) {
|
||||
writeMeta(output, obj, null)
|
||||
}
|
||||
|
||||
override fun readObject(source: Source): Meta = readMeta(source)
|
||||
override fun readObject(input: Input): Meta = readMeta(input)
|
||||
|
||||
public fun writeMeta(
|
||||
sink: Sink,
|
||||
output: Output,
|
||||
meta: Meta,
|
||||
descriptor: MetaDescriptor? = null,
|
||||
)
|
||||
|
||||
public fun readMeta(source: Source, descriptor: MetaDescriptor? = null): Meta
|
||||
public fun readMeta(input: Input, descriptor: MetaDescriptor? = null): Meta
|
||||
}
|
||||
|
||||
@Type(META_FORMAT_TYPE)
|
||||
@ -63,7 +63,9 @@ public fun Meta.toString(format: MetaFormat): String = ByteArray {
|
||||
|
||||
public fun Meta.toString(formatFactory: MetaFormatFactory): String = toString(formatFactory.build(Global, Meta.EMPTY))
|
||||
|
||||
public fun MetaFormat.parse(str: String): Meta = readObject(StringSource(str).buffered())
|
||||
public fun MetaFormat.parse(str: String): Meta {
|
||||
return ByteReadPacket(str.encodeToByteArray()).use { readObject(it) }
|
||||
}
|
||||
|
||||
public fun MetaFormatFactory.parse(str: String, formatMeta: Meta): Meta = build(Global, formatMeta).parse(str)
|
||||
|
||||
|
||||
@ -1,7 +1,6 @@
|
||||
package space.kscience.dataforge.io
|
||||
|
||||
import kotlinx.io.*
|
||||
import kotlinx.io.bytestring.decodeToString
|
||||
import io.ktor.utils.io.core.*
|
||||
import space.kscience.dataforge.context.Context
|
||||
import space.kscience.dataforge.context.Global
|
||||
import space.kscience.dataforge.meta.Meta
|
||||
@ -19,7 +18,7 @@ import space.kscience.dataforge.names.plus
|
||||
public class TaggedEnvelopeFormat(
|
||||
public val io: IOPlugin,
|
||||
public val version: VERSION = VERSION.DF02,
|
||||
public val metaFormatFactory: MetaFormatFactory = JsonMetaFormat,
|
||||
public val metaFormatFactory: MetaFormatFactory = JsonMetaFormat
|
||||
) : EnvelopeFormat {
|
||||
|
||||
// private val metaFormat = io.metaFormat(metaFormatKey)
|
||||
@ -27,60 +26,59 @@ public class TaggedEnvelopeFormat(
|
||||
|
||||
|
||||
private fun Tag.toBinary() = Binary {
|
||||
write(START_SEQUENCE)
|
||||
writeString(version.name)
|
||||
writeRawString(START_SEQUENCE)
|
||||
writeRawString(version.name)
|
||||
writeShort(metaFormatKey)
|
||||
writeUInt(metaSize)
|
||||
when (version) {
|
||||
VERSION.DF02 -> {
|
||||
writeUInt(dataSize.toUInt())
|
||||
}
|
||||
|
||||
VERSION.DF03 -> {
|
||||
writeULong(dataSize)
|
||||
}
|
||||
}
|
||||
write(END_SEQUENCE)
|
||||
writeRawString(END_SEQUENCE)
|
||||
}
|
||||
|
||||
override fun writeObject(
|
||||
sink: Sink,
|
||||
output: Output,
|
||||
obj: Envelope,
|
||||
) {
|
||||
val metaFormat = metaFormatFactory.build(io.context, Meta.EMPTY)
|
||||
val metaBytes = Binary(obj.meta, metaFormat)
|
||||
val metaBytes = Binary(obj.meta,metaFormat)
|
||||
val actualSize: ULong = (obj.data?.size ?: 0).toULong()
|
||||
val tag = Tag(metaFormatFactory.key, metaBytes.size.toUInt() + 2u, actualSize)
|
||||
sink.writeBinary(tag.toBinary())
|
||||
sink.writeBinary(metaBytes)
|
||||
sink.writeString("\r\n")
|
||||
output.writeBinary(tag.toBinary())
|
||||
output.writeBinary(metaBytes)
|
||||
output.writeRawString("\r\n")
|
||||
obj.data?.let {
|
||||
sink.writeBinary(it)
|
||||
output.writeBinary(it)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Read an envelope from input into memory
|
||||
*
|
||||
* @param source an input to read from
|
||||
* @param input an input to read from
|
||||
* @param formats a collection of meta formats to resolve
|
||||
*/
|
||||
override fun readObject(source: Source): Envelope {
|
||||
val tag = source.readTag(this.version)
|
||||
override fun readObject(input: Input): Envelope {
|
||||
val tag = input.readTag(this.version)
|
||||
|
||||
val metaFormat = io.resolveMetaFormat(tag.metaFormatKey)
|
||||
?: error("Meta format with key ${tag.metaFormatKey} not found")
|
||||
|
||||
val metaBinary = source.readBinary(tag.metaSize.toInt())
|
||||
val metaBinary = input.readBinary(tag.metaSize.toInt())
|
||||
|
||||
val meta: Meta = metaFormat.readObjectFrom(metaBinary)
|
||||
|
||||
val data = source.readBinary(tag.dataSize.toInt())
|
||||
val data = input.readBinary(tag.dataSize.toInt())
|
||||
|
||||
return SimpleEnvelope(meta, data)
|
||||
}
|
||||
|
||||
override fun readObject(binary: Binary): Envelope = binary.read {
|
||||
override fun readObject(binary: Binary): Envelope = binary.read{
|
||||
val tag = readTag(version)
|
||||
|
||||
val metaFormat = io.resolveMetaFormat(tag.metaFormatKey)
|
||||
@ -106,8 +104,8 @@ public class TaggedEnvelopeFormat(
|
||||
}
|
||||
|
||||
public companion object : EnvelopeFormatFactory {
|
||||
private val START_SEQUENCE = "#~".toAsciiByteString()
|
||||
private val END_SEQUENCE = "~#\r\n".toAsciiByteString()
|
||||
private const val START_SEQUENCE = "#~"
|
||||
private const val END_SEQUENCE = "~#\r\n"
|
||||
|
||||
override val name: Name = EnvelopeFormatFactory.ENVELOPE_FACTORY_NAME + "tagged"
|
||||
|
||||
@ -123,33 +121,35 @@ public class TaggedEnvelopeFormat(
|
||||
return TaggedEnvelopeFormat(io, version)
|
||||
}
|
||||
|
||||
private fun Source.readTag(version: VERSION): Tag {
|
||||
val start = readByteString(2)
|
||||
private fun Input.readTag(version: VERSION): Tag {
|
||||
val start = readRawString(2)
|
||||
if (start != START_SEQUENCE) error("The input is not an envelope")
|
||||
val versionString = readByteString(4)
|
||||
if (version.name.toAsciiByteString() != versionString) error("Wrong version of DataForge: expected $version but found $versionString")
|
||||
val versionString = readRawString(4)
|
||||
if (version.name != versionString) error("Wrong version of DataForge: expected $version but found $versionString")
|
||||
val metaFormatKey = readShort()
|
||||
val metaLength = readUInt()
|
||||
val dataLength: ULong = when (version) {
|
||||
VERSION.DF02 -> readUInt().toULong()
|
||||
VERSION.DF03 -> readULong()
|
||||
}
|
||||
val end = readByteString(4)
|
||||
val end = readRawString(4)
|
||||
if (end != END_SEQUENCE) error("The input is not an envelope")
|
||||
return Tag(metaFormatKey, metaLength, dataLength)
|
||||
}
|
||||
|
||||
override fun peekFormat(io: IOPlugin, binary: Binary): EnvelopeFormat? = try {
|
||||
binary.read {
|
||||
val header = readByteString(6)
|
||||
when (header.substring(2, 6).decodeToString()) {
|
||||
VERSION.DF02.name -> TaggedEnvelopeFormat(io, VERSION.DF02)
|
||||
VERSION.DF03.name -> TaggedEnvelopeFormat(io, VERSION.DF03)
|
||||
else -> null
|
||||
override fun peekFormat(io: IOPlugin, binary: Binary): EnvelopeFormat? {
|
||||
return try {
|
||||
binary.read {
|
||||
val header = readRawString(6)
|
||||
return@read when (header.substring(2..5)) {
|
||||
VERSION.DF02.name -> TaggedEnvelopeFormat(io, VERSION.DF02)
|
||||
VERSION.DF03.name -> TaggedEnvelopeFormat(io, VERSION.DF03)
|
||||
else -> null
|
||||
}
|
||||
}
|
||||
} catch (ex: Exception) {
|
||||
null
|
||||
}
|
||||
} catch (ex: Exception) {
|
||||
null
|
||||
}
|
||||
|
||||
private val default by lazy { build(Global, Meta.EMPTY) }
|
||||
@ -158,13 +158,16 @@ public class TaggedEnvelopeFormat(
|
||||
default.run { readObject(binary) }
|
||||
|
||||
override fun writeObject(
|
||||
sink: Sink,
|
||||
output: Output,
|
||||
obj: Envelope,
|
||||
): Unit = default.run {
|
||||
writeObject(sink, obj)
|
||||
writeObject(
|
||||
output,
|
||||
obj,
|
||||
)
|
||||
}
|
||||
|
||||
override fun readObject(source: Source): Envelope = default.readObject(source)
|
||||
override fun readObject(input: Input): Envelope = default.readObject(input)
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,9 +1,9 @@
|
||||
package space.kscience.dataforge.io
|
||||
|
||||
|
||||
import kotlinx.io.*
|
||||
import kotlinx.io.bytestring.ByteString
|
||||
import kotlinx.io.bytestring.encodeToByteString
|
||||
import io.ktor.utils.io.core.ByteReadPacket
|
||||
import io.ktor.utils.io.core.Input
|
||||
import io.ktor.utils.io.core.Output
|
||||
import io.ktor.utils.io.core.readUTF8UntilDelimiterTo
|
||||
import space.kscience.dataforge.context.Context
|
||||
import space.kscience.dataforge.context.Global
|
||||
import space.kscience.dataforge.meta.Meta
|
||||
@ -29,35 +29,34 @@ public class TaglessEnvelopeFormat(
|
||||
// }
|
||||
|
||||
override fun writeObject(
|
||||
sink: Sink,
|
||||
output: Output,
|
||||
obj: Envelope,
|
||||
) {
|
||||
val metaFormat = metaFormatFactory.build(this.io.context, meta)
|
||||
|
||||
//printing header
|
||||
sink.write(TAGLESS_ENVELOPE_HEADER)
|
||||
sink.writeString("\r\n")
|
||||
output.writeRawString(TAGLESS_ENVELOPE_HEADER + "\r\n")
|
||||
|
||||
//Printing meta
|
||||
if (!obj.meta.isEmpty()) {
|
||||
val metaBinary = Binary(obj.meta, metaFormat)
|
||||
sink.writeString(META_START + "-${metaFormatFactory.shortName}\r\n")
|
||||
sink.writeBinary(metaBinary)
|
||||
sink.writeString("\r\n")
|
||||
output.writeUtf8String(META_START + "-${metaFormatFactory.shortName}\r\n")
|
||||
output.writeBinary(metaBinary)
|
||||
output.writeRawString("\r\n")
|
||||
}
|
||||
|
||||
//Printing data
|
||||
obj.data?.let { data ->
|
||||
//val actualSize: Int = envelope.data?.size ?: 0
|
||||
sink.writeString(DATA_START + "\r\n")
|
||||
sink.writeBinary(data)
|
||||
output.writeUtf8String(DATA_START + "\r\n")
|
||||
output.writeBinary(data)
|
||||
}
|
||||
}
|
||||
|
||||
override fun readObject(source: Source): Envelope {
|
||||
override fun readObject(input: Input): Envelope {
|
||||
//read preamble
|
||||
source.discardWithSeparator(
|
||||
TAGLESS_ENVELOPE_HEADER,
|
||||
input.discardWithSeparator(
|
||||
TAGLESS_ENVELOPE_HEADER.encodeToByteArray(),
|
||||
atMost = 1024,
|
||||
)
|
||||
|
||||
@ -65,22 +64,22 @@ public class TaglessEnvelopeFormat(
|
||||
|
||||
var data: Binary? = null
|
||||
|
||||
source.discardWithSeparator(
|
||||
input.discardWithSeparator(
|
||||
SEPARATOR_PREFIX,
|
||||
atMost = 1024,
|
||||
)
|
||||
|
||||
var header: String = ByteArray {
|
||||
source.readWithSeparatorTo(this, "\n".encodeToByteString())
|
||||
input.readUTF8UntilDelimiterTo(this, "\n")
|
||||
}.decodeToString()
|
||||
|
||||
while (!source.exhausted()) {
|
||||
while (!input.endOfInput) {
|
||||
val block = ByteArray {
|
||||
source.readWithSeparatorTo(this, SEPARATOR_PREFIX)
|
||||
input.readWithSeparatorTo(this, SEPARATOR_PREFIX)
|
||||
}
|
||||
|
||||
val nextHeader = ByteArray {
|
||||
source.readWithSeparatorTo(this, "\n".encodeToByteString())
|
||||
input.readWithSeparatorTo(this, "\n".encodeToByteArray())
|
||||
}.decodeToString()
|
||||
|
||||
//terminate on end
|
||||
@ -90,7 +89,7 @@ public class TaglessEnvelopeFormat(
|
||||
if (header.startsWith("META")) {
|
||||
//TODO check format
|
||||
val metaFormat: MetaFormatFactory = JsonMetaFormat
|
||||
meta = metaFormat.readMeta(ByteArraySource(block).buffered())
|
||||
meta = metaFormat.readMeta(ByteReadPacket(block))
|
||||
}
|
||||
|
||||
if (header.startsWith("DATA")) {
|
||||
@ -112,9 +111,9 @@ public class TaglessEnvelopeFormat(
|
||||
|
||||
public const val TAGLESS_ENVELOPE_TYPE: String = "tagless"
|
||||
|
||||
public val SEPARATOR_PREFIX: ByteString = "\n#~".encodeToByteString()
|
||||
public val SEPARATOR_PREFIX: ByteArray = "\n#~".encodeToByteArray()
|
||||
|
||||
public val TAGLESS_ENVELOPE_HEADER: ByteString = "#~DFTL".encodeToByteString()
|
||||
public const val TAGLESS_ENVELOPE_HEADER: String = "#~DFTL"
|
||||
|
||||
// public const val META_START_PROPERTY: String = "metaSeparator"
|
||||
public const val META_START: String = "#~META"
|
||||
@ -135,18 +134,21 @@ public class TaglessEnvelopeFormat(
|
||||
override fun readObject(binary: Binary): Envelope = default.run { readObject(binary) }
|
||||
|
||||
override fun writeObject(
|
||||
sink: Sink,
|
||||
output: Output,
|
||||
obj: Envelope,
|
||||
): Unit = default.run {
|
||||
writeObject(sink, obj)
|
||||
writeObject(
|
||||
output,
|
||||
obj,
|
||||
)
|
||||
}
|
||||
|
||||
override fun readObject(source: Source): Envelope = default.readObject(source)
|
||||
override fun readObject(input: Input): Envelope = default.readObject(input)
|
||||
|
||||
override fun peekFormat(io: IOPlugin, binary: Binary): EnvelopeFormat? {
|
||||
return try {
|
||||
binary.read {
|
||||
val string = readByteString(TAGLESS_ENVELOPE_HEADER.size)
|
||||
val string = readRawString(TAGLESS_ENVELOPE_HEADER.length)
|
||||
return@read if (string == TAGLESS_ENVELOPE_HEADER) {
|
||||
TaglessEnvelopeFormat(io)
|
||||
} else {
|
||||
|
||||
@ -1,41 +1,40 @@
|
||||
package space.kscience.dataforge.io
|
||||
|
||||
import kotlinx.io.*
|
||||
import kotlinx.io.bytestring.ByteString
|
||||
import kotlinx.io.bytestring.decodeToString
|
||||
import kotlinx.io.bytestring.encodeToByteString
|
||||
import io.ktor.utils.io.bits.Memory
|
||||
import io.ktor.utils.io.charsets.Charsets
|
||||
import io.ktor.utils.io.charsets.decodeExactBytes
|
||||
import io.ktor.utils.io.core.*
|
||||
import io.ktor.utils.io.core.internal.ChunkBuffer
|
||||
import space.kscience.dataforge.meta.Meta
|
||||
import space.kscience.dataforge.misc.DFExperimental
|
||||
import kotlin.math.min
|
||||
|
||||
/**
|
||||
* Convert a string literal, containing only ASCII characters to a [ByteString].
|
||||
* Throws an error if there are non-ASCII characters.
|
||||
*/
|
||||
public fun String.toAsciiByteString(): ByteString {
|
||||
val bytes = ByteArray(length) {
|
||||
val char = get(it)
|
||||
val code = char.code
|
||||
if (code > Byte.MAX_VALUE) error("Symbol $char is not ASCII symbol") else code.toByte()
|
||||
}
|
||||
return ByteString(bytes)
|
||||
public fun Output.writeRawString(str: String) {
|
||||
writeFully(str.toByteArray(Charsets.ISO_8859_1))
|
||||
}
|
||||
|
||||
public inline fun Buffer(block: Sink.() -> Unit): Buffer = Buffer().apply(block)
|
||||
public fun Output.writeUtf8String(str: String) {
|
||||
writeFully(str.encodeToByteArray())
|
||||
}
|
||||
|
||||
//public fun Source.readSafeUtf8Line(): String = readUTF8Line() ?: error("Line not found")
|
||||
public fun Input.readRawString(size: Int): String {
|
||||
return Charsets.ISO_8859_1.newDecoder().decodeExactBytes(this, size)
|
||||
}
|
||||
|
||||
public inline fun ByteArray(block: Sink.() -> Unit): ByteArray =
|
||||
Buffer(block).readByteArray()
|
||||
public fun Input.readUtf8String(): String = readBytes().decodeToString()
|
||||
|
||||
public inline fun Binary(block: Sink.() -> Unit): Binary =
|
||||
public fun Input.readSafeUtf8Line(): String = readUTF8Line() ?: error("Line not found")
|
||||
|
||||
public inline fun ByteArray(block: Output.() -> Unit): ByteArray =
|
||||
buildPacket(block).readBytes()
|
||||
|
||||
public inline fun Binary(block: Output.() -> Unit): Binary =
|
||||
ByteArray(block).asBinary()
|
||||
|
||||
public operator fun Binary.get(range: IntRange): Binary = view(range.first, range.last - range.first)
|
||||
|
||||
/**
|
||||
* Return inferred [EnvelopeFormat] if only one format could read given file. If no format accepts the binary, return null. If
|
||||
* multiple formats accept binary, throw an error.
|
||||
* multiple formats accepts binary, throw an error.
|
||||
*/
|
||||
public fun IOPlugin.peekBinaryEnvelopeFormat(binary: Binary): EnvelopeFormat? {
|
||||
val formats = envelopeFormatFactories.mapNotNull { factory ->
|
||||
@ -97,139 +96,74 @@ private class RingByteArray(
|
||||
else -> inputArray.indices.all { inputArray[it] == get(it) }
|
||||
}
|
||||
|
||||
fun contentEquals(byteString: ByteString): Boolean = when {
|
||||
byteString.size != buffer.size -> false
|
||||
size < buffer.size -> false
|
||||
else -> (0 until byteString.size).all { byteString[it] == get(it) }
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
private fun RingByteArray.toArray(): ByteArray = ByteArray(size) { get(it) }
|
||||
|
||||
/**
|
||||
* Read [Source] into [output] until designated multibyte [separator] and optionally continues until
|
||||
* Read [Input] into [output] until designated multibyte [separator] and optionally continues until
|
||||
* the end of the line after it. Throw error if [separator] not found and [atMost] bytes are read.
|
||||
* Also fails if [separator] not found until the end of input.
|
||||
*
|
||||
* The Separator itself is not read into [Sink].
|
||||
* Separator itself is not read into Output.
|
||||
*
|
||||
* @param errorOnEof if true error is thrown if separator is never encountered
|
||||
*
|
||||
* @return bytes actually being read, including separator
|
||||
*/
|
||||
public fun Source.readWithSeparatorTo(
|
||||
output: Sink?,
|
||||
separator: ByteString,
|
||||
public fun Input.readWithSeparatorTo(
|
||||
output: Output,
|
||||
separator: ByteArray,
|
||||
atMost: Int = Int.MAX_VALUE,
|
||||
errorOnEof: Boolean = false,
|
||||
): Int {
|
||||
var counter = 0
|
||||
val rb = RingByteArray(ByteArray(separator.size))
|
||||
|
||||
while (!exhausted()) {
|
||||
val byte = readByte()
|
||||
counter++
|
||||
if (counter >= atMost) error("Maximum number of bytes to be read $atMost reached.")
|
||||
rb.push(byte)
|
||||
if (rb.contentEquals(separator)) {
|
||||
return counter
|
||||
} else if (rb.isFull()) {
|
||||
output?.writeByte(rb[0])
|
||||
takeWhile { buffer ->
|
||||
while (buffer.canRead()) {
|
||||
val byte = buffer.readByte()
|
||||
counter++
|
||||
if (counter >= atMost) error("Maximum number of bytes to be read $atMost reached.")
|
||||
rb.push(byte)
|
||||
if (rb.contentEquals(separator)) {
|
||||
return counter
|
||||
} else if (rb.isFull()) {
|
||||
output.writeByte(rb[0])
|
||||
}
|
||||
}
|
||||
!endOfInput
|
||||
}
|
||||
|
||||
if (errorOnEof) {
|
||||
error("Read to the end of input without encountering ${separator.decodeToString()}")
|
||||
} else {
|
||||
for (i in 1 until rb.size) {
|
||||
output?.writeByte(rb[i])
|
||||
for(i in 1 until rb.size){
|
||||
output.writeByte(rb[i])
|
||||
}
|
||||
counter += (rb.size - 1)
|
||||
return counter
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Discard all bytes until [separator] is encountered. Separator is discarded sa well.
|
||||
* Return the total number of bytes read.
|
||||
*/
|
||||
public fun Source.discardWithSeparator(
|
||||
separator: ByteString,
|
||||
public fun Input.discardLine(): Int {
|
||||
return discardUntilDelimiter('\n'.code.toByte()).also {
|
||||
discard(1)
|
||||
}.toInt() + 1
|
||||
}
|
||||
|
||||
public fun Input.discardWithSeparator(
|
||||
separator: ByteArray,
|
||||
atMost: Int = Int.MAX_VALUE,
|
||||
errorOnEof: Boolean = false,
|
||||
): Int = readWithSeparatorTo(null, separator, atMost, errorOnEof)
|
||||
): Int {
|
||||
val dummy: Output = object : Output(ChunkBuffer.Pool) {
|
||||
override fun closeDestination() {
|
||||
// Do nothing
|
||||
}
|
||||
|
||||
/**
|
||||
* Discard all symbol until newline is discovered. Carriage return is not discarded.
|
||||
*/
|
||||
public fun Source.discardLine(
|
||||
atMost: Int = Int.MAX_VALUE,
|
||||
errorOnEof: Boolean = false,
|
||||
): Int = discardWithSeparator("\n".encodeToByteString(), atMost, errorOnEof)
|
||||
|
||||
|
||||
/**
|
||||
* A [Source] based on [ByteArray]
|
||||
*/
|
||||
public class ByteArraySource(
|
||||
private val byteArray: ByteArray,
|
||||
private val offset: Int = 0,
|
||||
private val size: Int = byteArray.size - offset,
|
||||
) : RawSource {
|
||||
|
||||
init {
|
||||
require(offset >= 0) { "Offset must be positive" }
|
||||
require(offset + size <= byteArray.size) { "End index is ${offset + size}, but the array size is ${byteArray.size}" }
|
||||
override fun flush(source: Memory, offset: Int, length: Int) {
|
||||
// Do nothing
|
||||
}
|
||||
}
|
||||
|
||||
private var pointer = offset
|
||||
|
||||
override fun close() {
|
||||
// Do nothing
|
||||
}
|
||||
|
||||
override fun readAtMostTo(sink: Buffer, byteCount: Long): Long {
|
||||
if (pointer == offset + size) return -1
|
||||
val byteRead = min(byteCount.toInt(), (size + offset - pointer))
|
||||
sink.write(byteArray, pointer, pointer + byteRead)
|
||||
pointer += byteRead
|
||||
return byteRead.toLong()
|
||||
}
|
||||
return readWithSeparatorTo(dummy, separator, atMost, errorOnEof)
|
||||
}
|
||||
|
||||
/**
|
||||
* A [Source] based on [String]
|
||||
*/
|
||||
public class StringSource(
|
||||
public val string: String,
|
||||
public val offset: Int = 0,
|
||||
public val size: Int = string.length - offset,
|
||||
) : RawSource {
|
||||
|
||||
private var pointer = offset
|
||||
|
||||
override fun close() {
|
||||
// Do nothing
|
||||
}
|
||||
|
||||
override fun readAtMostTo(sink: Buffer, byteCount: Long): Long {
|
||||
if (pointer == offset + size) return -1
|
||||
val byteRead = min(byteCount.toInt(), (size + offset - pointer))
|
||||
sink.writeString(string, pointer, pointer + byteRead)
|
||||
pointer += byteRead
|
||||
return byteRead.toLong()
|
||||
}
|
||||
}
|
||||
|
||||
public fun Sink.writeDouble(value: Double) {
|
||||
writeLong(value.toBits())
|
||||
}
|
||||
|
||||
public fun Source.readDouble(): Double = Double.fromBits(readLong())
|
||||
|
||||
public fun Sink.writeFloat(value: Float) {
|
||||
writeInt(value.toBits())
|
||||
}
|
||||
|
||||
public fun Source.readFloat(): Float = Float.fromBits(readInt())
|
||||
@ -1,5 +1,6 @@
|
||||
package space.kscience.dataforge.io
|
||||
|
||||
import io.ktor.utils.io.core.readInt
|
||||
import kotlin.test.Test
|
||||
import kotlin.test.assertEquals
|
||||
|
||||
|
||||
@ -1,7 +1,6 @@
|
||||
package space.kscience.dataforge.io
|
||||
|
||||
import kotlinx.io.readByteArray
|
||||
import kotlinx.io.writeString
|
||||
import io.ktor.utils.io.core.readBytes
|
||||
import kotlin.test.Test
|
||||
import kotlin.test.assertEquals
|
||||
|
||||
@ -13,7 +12,7 @@ class EnvelopeFormatTest {
|
||||
"d" put 22.2
|
||||
}
|
||||
data {
|
||||
writeString("12345678")
|
||||
writeUtf8String("12345678")
|
||||
}
|
||||
}
|
||||
|
||||
@ -23,7 +22,7 @@ class EnvelopeFormatTest {
|
||||
val res = readFromByteArray(byteArray)
|
||||
assertEquals(envelope.meta, res.meta)
|
||||
val bytes = res.data?.read {
|
||||
readByteArray()
|
||||
readBytes()
|
||||
}
|
||||
assertEquals("12345678", bytes?.decodeToString())
|
||||
}
|
||||
@ -35,13 +34,13 @@ class EnvelopeFormatTest {
|
||||
val res = readFromByteArray(byteArray)
|
||||
assertEquals(envelope.meta, res.meta)
|
||||
val bytes = res.data?.read {
|
||||
readByteArray()
|
||||
readBytes()
|
||||
}
|
||||
assertEquals("12345678", bytes?.decodeToString())
|
||||
assertEquals("12345678", bytes?.decodeToString())
|
||||
}
|
||||
|
||||
@Test
|
||||
fun testManualDftl() {
|
||||
fun testManualDftl(){
|
||||
val envelopeString = """
|
||||
#~DFTL
|
||||
#~META
|
||||
@ -57,8 +56,8 @@ class EnvelopeFormatTest {
|
||||
val res = TaglessEnvelopeFormat.readFromByteArray(envelopeString.encodeToByteArray())
|
||||
assertEquals(envelope.meta, res.meta)
|
||||
val bytes = res.data?.read {
|
||||
readByteArray()
|
||||
readBytes()
|
||||
}
|
||||
assertEquals("12345678", bytes?.decodeToString())
|
||||
assertEquals("12345678", bytes?.decodeToString())
|
||||
}
|
||||
}
|
||||
@ -1,9 +1,8 @@
|
||||
package space.kscience.dataforge.io
|
||||
|
||||
import kotlinx.io.buffered
|
||||
import kotlinx.io.bytestring.encodeToByteString
|
||||
import kotlinx.io.readByteArray
|
||||
import kotlinx.io.readLine
|
||||
import io.ktor.utils.io.core.ByteReadPacket
|
||||
import io.ktor.utils.io.core.readBytes
|
||||
import io.ktor.utils.io.core.readUTF8Line
|
||||
import kotlin.test.Test
|
||||
import kotlin.test.assertEquals
|
||||
import kotlin.test.assertFails
|
||||
@ -12,9 +11,9 @@ class IOTest {
|
||||
@Test
|
||||
fun readBytes() {
|
||||
val bytes = ByteArray(8) { it.toByte() }
|
||||
val input = ByteArraySource(bytes).buffered()
|
||||
@Suppress("UNUSED_VARIABLE") val first = input.readByteArray(4)
|
||||
val second = input.readByteArray(4)
|
||||
val input = ByteReadPacket(bytes)
|
||||
@Suppress("UNUSED_VARIABLE") val first = input.readBytes(4)
|
||||
val second = input.readBytes(4)
|
||||
assertEquals(4.toByte(), second[0])
|
||||
}
|
||||
|
||||
@ -32,25 +31,25 @@ class IOTest {
|
||||
|
||||
binary.read {
|
||||
val array = ByteArray {
|
||||
val read = readWithSeparatorTo(this, "---".encodeToByteString()) + discardLine()
|
||||
val read = readWithSeparatorTo(this, "---".encodeToByteArray()) + discardLine()
|
||||
assertEquals(12, read)
|
||||
}
|
||||
assertEquals("""
|
||||
aaa
|
||||
bbb
|
||||
""".trimIndent(),array.decodeToString().trim())
|
||||
assertEquals("ccc", readLine()?.trim())
|
||||
assertEquals("ccc", readUTF8Line()?.trim())
|
||||
}
|
||||
|
||||
assertFails {
|
||||
binary.read {
|
||||
discardWithSeparator("---".encodeToByteString(), atMost = 3 )
|
||||
discardWithSeparator("---".encodeToByteArray(), atMost = 3 )
|
||||
}
|
||||
}
|
||||
|
||||
assertFails {
|
||||
binary.read{
|
||||
discardWithSeparator("-+-".encodeToByteString(), errorOnEof = true)
|
||||
discardWithSeparator("-+-".encodeToByteArray(), errorOnEof = true)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -1,6 +1,5 @@
|
||||
package space.kscience.dataforge.io
|
||||
|
||||
import kotlinx.io.writeString
|
||||
import space.kscience.dataforge.context.Global
|
||||
import space.kscience.dataforge.meta.get
|
||||
import space.kscience.dataforge.meta.int
|
||||
@ -19,9 +18,9 @@ class MultipartTest {
|
||||
"value" put it
|
||||
}
|
||||
data {
|
||||
writeString("Hello World $it")
|
||||
writeUtf8String("Hello World $it")
|
||||
repeat(300) {
|
||||
writeString("$it ")
|
||||
writeRawString("$it ")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,11 +1,12 @@
|
||||
package space.kscience.dataforge.io
|
||||
|
||||
import kotlinx.io.buffered
|
||||
import io.ktor.utils.io.core.ByteReadPacket
|
||||
import io.ktor.utils.io.core.use
|
||||
|
||||
|
||||
fun <T : Any> IOFormat<T>.writeToByteArray(obj: T): ByteArray = ByteArray {
|
||||
writeObject(this, obj)
|
||||
}
|
||||
fun <T : Any> IOFormat<T>.readFromByteArray(array: ByteArray): T = ByteArraySource(array).buffered().use {
|
||||
fun <T : Any> IOFormat<T>.readFromByteArray(array: ByteArray): T = ByteReadPacket(array).use {
|
||||
readObject(it)
|
||||
}
|
||||
@ -1,11 +1,8 @@
|
||||
package space.kscience.dataforge.io
|
||||
|
||||
|
||||
import io.ktor.utils.io.core.*
|
||||
import io.ktor.utils.io.streams.asOutput
|
||||
import kotlinx.coroutines.runBlocking
|
||||
import kotlinx.io.Sink
|
||||
import kotlinx.io.Source
|
||||
import kotlinx.io.asSink
|
||||
import kotlinx.io.buffered
|
||||
import space.kscience.dataforge.meta.Meta
|
||||
import space.kscience.dataforge.meta.descriptors.MetaDescriptor
|
||||
import space.kscience.dataforge.meta.isEmpty
|
||||
@ -26,18 +23,18 @@ internal class PathBinary(
|
||||
override val size: Int = Files.size(path).toInt() - fileOffset,
|
||||
) : Binary {
|
||||
|
||||
override fun <R> read(offset: Int, atMost: Int, block: Source.() -> R): R = runBlocking {
|
||||
override fun <R> read(offset: Int, atMost: Int, block: Input.() -> R): R = runBlocking {
|
||||
readSuspend(offset, atMost, block)
|
||||
}
|
||||
|
||||
override suspend fun <R> readSuspend(offset: Int, atMost: Int, block: suspend Source.() -> R): R {
|
||||
override suspend fun <R> readSuspend(offset: Int, atMost: Int, block: suspend Input.() -> R): R {
|
||||
val actualOffset = offset + fileOffset
|
||||
val actualSize = min(atMost, size - offset)
|
||||
val array = path.inputStream().use {
|
||||
it.skip(actualOffset.toLong())
|
||||
it.readNBytes(actualSize)
|
||||
}
|
||||
return ByteArraySource(array).buffered().use { it.block() }
|
||||
return ByteReadPacket(array).block()
|
||||
}
|
||||
|
||||
override fun view(offset: Int, binarySize: Int) = PathBinary(path, fileOffset + offset, binarySize)
|
||||
@ -45,36 +42,36 @@ internal class PathBinary(
|
||||
|
||||
public fun Path.asBinary(): Binary = PathBinary(this)
|
||||
|
||||
public fun <R> Path.read(block: Source.() -> R): R = asBinary().read(block = block)
|
||||
public fun <R> Path.read(block: Input.() -> R): R = asBinary().read(block = block)
|
||||
|
||||
/**
|
||||
* Write a live output to a newly created file. If file does not exist, throws error
|
||||
*/
|
||||
public fun Path.write(block: Sink.() -> Unit): Unit {
|
||||
public fun Path.write(block: Output.() -> Unit): Unit {
|
||||
val stream = Files.newOutputStream(this, StandardOpenOption.WRITE, StandardOpenOption.CREATE_NEW)
|
||||
stream.asSink().buffered().use(block)
|
||||
stream.asOutput().use(block)
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a new file or append to exiting one with given output [block]
|
||||
*/
|
||||
public fun Path.append(block: Sink.() -> Unit): Unit {
|
||||
public fun Path.append(block: Output.() -> Unit): Unit {
|
||||
val stream = Files.newOutputStream(
|
||||
this,
|
||||
StandardOpenOption.WRITE, StandardOpenOption.APPEND, StandardOpenOption.CREATE
|
||||
)
|
||||
stream.asSink().buffered().use(block)
|
||||
stream.asOutput().use(block)
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a new file or replace existing one using given output [block]
|
||||
*/
|
||||
public fun Path.rewrite(block: Sink.() -> Unit): Unit {
|
||||
public fun Path.rewrite(block: Output.() -> Unit): Unit {
|
||||
val stream = Files.newOutputStream(
|
||||
this,
|
||||
StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.CREATE
|
||||
)
|
||||
stream.asSink().buffered().use(block)
|
||||
stream.asOutput().use(block)
|
||||
}
|
||||
|
||||
@DFExperimental
|
||||
@ -263,7 +260,7 @@ public fun IOPlugin.writeEnvelopeDirectory(
|
||||
val dataFile = path.resolve(IOPlugin.DATA_FILE_NAME)
|
||||
dataFile.write {
|
||||
envelope.data?.read {
|
||||
val copied = transferTo(this@write)
|
||||
val copied = copyTo(this@write)
|
||||
if (copied != envelope.data?.size?.toLong()) {
|
||||
error("The number of copied bytes does not equal data size")
|
||||
}
|
||||
|
||||
@ -1,11 +1,9 @@
|
||||
package space.kscience.dataforge.io
|
||||
|
||||
import kotlinx.io.Source
|
||||
import kotlinx.io.asSource
|
||||
import kotlinx.io.buffered
|
||||
import io.ktor.utils.io.core.Input
|
||||
import io.ktor.utils.io.streams.asInput
|
||||
|
||||
public fun IOPlugin.resource(name: String): Binary? = context.javaClass.getResource(name)?.readBytes()?.asBinary()
|
||||
|
||||
public fun IOPlugin.resource(name: String): Binary? = { }.javaClass.getResource(name)?.readBytes()?.asBinary()
|
||||
|
||||
public inline fun <R> IOPlugin.readResource(name: String, block: Source.() -> R): R =
|
||||
{ }.javaClass.getResource(name)?.openStream()?.asSource()?.buffered()?.block() ?: error("Can't read resource $name")
|
||||
public inline fun <R> IOPlugin.readResource(name: String, block: Input.() -> R): R =
|
||||
context.javaClass.getResource(name)?.openStream()?.asInput()?.block() ?: error("Can't read resource $name")
|
||||
@ -1,5 +1,6 @@
|
||||
package space.kscience.dataforge.io
|
||||
|
||||
import io.ktor.utils.io.core.writeDouble
|
||||
import space.kscience.dataforge.context.Global
|
||||
import space.kscience.dataforge.misc.DFExperimental
|
||||
import java.nio.file.Files
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
package space.kscience.dataforge.io
|
||||
|
||||
import io.ktor.utils.io.core.writeDouble
|
||||
import space.kscience.dataforge.context.Global
|
||||
import space.kscience.dataforge.misc.DFExperimental
|
||||
import java.nio.file.Files
|
||||
|
||||
@ -45,9 +45,9 @@ public final class space/kscience/dataforge/meta/False : space/kscience/dataforg
|
||||
|
||||
public final class space/kscience/dataforge/meta/JsonMetaKt {
|
||||
public static final fun getJSON_ARRAY_KEY (Lspace/kscience/dataforge/meta/Meta$Companion;)Ljava/lang/String;
|
||||
public static final fun toJson (Lspace/kscience/dataforge/meta/Meta;Lspace/kscience/dataforge/meta/descriptors/MetaDescriptor;)Lkotlinx/serialization/json/JsonElement;
|
||||
public static final fun toJson (Lspace/kscience/dataforge/meta/Meta;Lspace/kscience/dataforge/meta/descriptors/MetaDescriptor;)Lkotlinx/serialization/json/JsonObject;
|
||||
public static final fun toJson (Lspace/kscience/dataforge/meta/Value;Lspace/kscience/dataforge/meta/descriptors/MetaDescriptor;)Lkotlinx/serialization/json/JsonElement;
|
||||
public static synthetic fun toJson$default (Lspace/kscience/dataforge/meta/Meta;Lspace/kscience/dataforge/meta/descriptors/MetaDescriptor;ILjava/lang/Object;)Lkotlinx/serialization/json/JsonElement;
|
||||
public static synthetic fun toJson$default (Lspace/kscience/dataforge/meta/Meta;Lspace/kscience/dataforge/meta/descriptors/MetaDescriptor;ILjava/lang/Object;)Lkotlinx/serialization/json/JsonObject;
|
||||
public static synthetic fun toJson$default (Lspace/kscience/dataforge/meta/Value;Lspace/kscience/dataforge/meta/descriptors/MetaDescriptor;ILjava/lang/Object;)Lkotlinx/serialization/json/JsonElement;
|
||||
public static final fun toMeta (Lkotlinx/serialization/json/JsonElement;Lspace/kscience/dataforge/meta/descriptors/MetaDescriptor;)Lspace/kscience/dataforge/meta/SealedMeta;
|
||||
public static final fun toMeta (Lkotlinx/serialization/json/JsonObject;Lspace/kscience/dataforge/meta/descriptors/MetaDescriptor;)Lspace/kscience/dataforge/meta/SealedMeta;
|
||||
@ -639,6 +639,17 @@ public final class space/kscience/dataforge/meta/ValueType : java/lang/Enum {
|
||||
public static fun values ()[Lspace/kscience/dataforge/meta/ValueType;
|
||||
}
|
||||
|
||||
public final class space/kscience/dataforge/meta/ValueType$$serializer : kotlinx/serialization/internal/GeneratedSerializer {
|
||||
public static final field INSTANCE Lspace/kscience/dataforge/meta/ValueType$$serializer;
|
||||
public fun childSerializers ()[Lkotlinx/serialization/KSerializer;
|
||||
public synthetic fun deserialize (Lkotlinx/serialization/encoding/Decoder;)Ljava/lang/Object;
|
||||
public fun deserialize (Lkotlinx/serialization/encoding/Decoder;)Lspace/kscience/dataforge/meta/ValueType;
|
||||
public fun getDescriptor ()Lkotlinx/serialization/descriptors/SerialDescriptor;
|
||||
public synthetic fun serialize (Lkotlinx/serialization/encoding/Encoder;Ljava/lang/Object;)V
|
||||
public fun serialize (Lkotlinx/serialization/encoding/Encoder;Lspace/kscience/dataforge/meta/ValueType;)V
|
||||
public fun typeParametersSerializers ()[Lkotlinx/serialization/KSerializer;
|
||||
}
|
||||
|
||||
public final class space/kscience/dataforge/meta/ValueType$Companion {
|
||||
public final fun serializer ()Lkotlinx/serialization/KSerializer;
|
||||
}
|
||||
|
||||
@ -10,35 +10,17 @@ kscience{
|
||||
useSerialization{
|
||||
protobuf()
|
||||
}
|
||||
// dependencies {
|
||||
// api(projects.dataforgeContext)
|
||||
// api(projects.dataforgeData)
|
||||
// api(projects.dataforgeIo)
|
||||
// }
|
||||
// dependencies(jvmTest){
|
||||
// implementation(spclibs.logback.classic)
|
||||
// implementation(projects.dataforgeIo.dataforgeIoYaml)
|
||||
// }
|
||||
dependencies {
|
||||
api(projects.dataforgeContext)
|
||||
api(projects.dataforgeData)
|
||||
api(projects.dataforgeIo)
|
||||
}
|
||||
dependencies(jvmTest){
|
||||
implementation(spclibs.logback.classic)
|
||||
implementation(projects.dataforgeIo.dataforgeIoYaml)
|
||||
}
|
||||
}
|
||||
|
||||
readme{
|
||||
maturity = space.kscience.gradle.Maturity.EXPERIMENTAL
|
||||
}
|
||||
|
||||
kotlin{
|
||||
sourceSets{
|
||||
commonMain{
|
||||
dependencies {
|
||||
api(projects.dataforgeContext)
|
||||
api(projects.dataforgeData)
|
||||
api(projects.dataforgeIo)
|
||||
}
|
||||
}
|
||||
getByName("jvmTest"){
|
||||
dependencies {
|
||||
implementation(spclibs.logback.classic)
|
||||
implementation(projects.dataforgeIo.dataforgeIoYaml)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,6 +1,9 @@
|
||||
package space.kscience.dataforge.workspace
|
||||
|
||||
import kotlinx.io.*
|
||||
import io.ktor.utils.io.core.Input
|
||||
import io.ktor.utils.io.core.Output
|
||||
import io.ktor.utils.io.core.readBytes
|
||||
import io.ktor.utils.io.core.writeFully
|
||||
import kotlinx.serialization.ExperimentalSerializationApi
|
||||
import kotlinx.serialization.KSerializer
|
||||
import kotlinx.serialization.json.Json
|
||||
@ -27,10 +30,10 @@ public class JsonIOFormat<T : Any>(override val type: KType) : IOFormat<T> {
|
||||
@Suppress("UNCHECKED_CAST")
|
||||
private val serializer: KSerializer<T> = serializer(type) as KSerializer<T>
|
||||
|
||||
override fun readObject(source: Source): T = Json.decodeFromString(serializer, source.readString())
|
||||
override fun readObject(input: Input): T = Json.decodeFromString(serializer, input.readUtf8String())
|
||||
|
||||
override fun writeObject(sink: Sink, obj: T) {
|
||||
sink.writeString(Json.encodeToString(serializer, obj))
|
||||
override fun writeObject(output: Output, obj: T) {
|
||||
output.writeUtf8String(Json.encodeToString(serializer, obj))
|
||||
}
|
||||
}
|
||||
|
||||
@ -40,10 +43,10 @@ public class ProtobufIOFormat<T : Any>(override val type: KType) : IOFormat<T> {
|
||||
@Suppress("UNCHECKED_CAST")
|
||||
private val serializer: KSerializer<T> = serializer(type) as KSerializer<T>
|
||||
|
||||
override fun readObject(source: Source): T = ProtoBuf.decodeFromByteArray(serializer, source.readByteArray())
|
||||
override fun readObject(input: Input): T = ProtoBuf.decodeFromByteArray(serializer, input.readBytes())
|
||||
|
||||
override fun writeObject(sink: Sink, obj: T) {
|
||||
sink.write(ProtoBuf.encodeToByteArray(serializer, obj))
|
||||
override fun writeObject(output: Output, obj: T) {
|
||||
output.writeFully(ProtoBuf.encodeToByteArray(serializer, obj))
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -1,10 +1,13 @@
|
||||
package space.kscience.dataforge.workspace
|
||||
|
||||
import io.ktor.utils.io.streams.asOutput
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.withContext
|
||||
import space.kscience.dataforge.data.DataTree
|
||||
import space.kscience.dataforge.data.DataTreeItem
|
||||
import space.kscience.dataforge.io.*
|
||||
import space.kscience.dataforge.io.EnvelopeFormat
|
||||
import space.kscience.dataforge.io.IOFormat
|
||||
import space.kscience.dataforge.io.TaggedEnvelopeFormat
|
||||
import space.kscience.dataforge.misc.DFExperimental
|
||||
import java.nio.file.Files
|
||||
import java.nio.file.Path
|
||||
@ -25,15 +28,11 @@ private suspend fun <T : Any> ZipOutputStream.writeNode(
|
||||
val envelope = treeItem.data.toEnvelope(dataFormat)
|
||||
val entry = ZipEntry(name)
|
||||
putNextEntry(entry)
|
||||
|
||||
//TODO remove additional copy
|
||||
val bytes = ByteArray {
|
||||
writeObject(envelopeFormat, envelope)
|
||||
asOutput().run {
|
||||
envelopeFormat.writeObject(this, envelope)
|
||||
flush()
|
||||
}
|
||||
write(bytes)
|
||||
|
||||
}
|
||||
|
||||
is DataTreeItem.Node -> {
|
||||
val entry = ZipEntry("$name/")
|
||||
putNextEntry(entry)
|
||||
|
||||
@ -1,18 +1,12 @@
|
||||
package space.kscience.dataforge.workspace
|
||||
|
||||
import io.ktor.utils.io.core.Input
|
||||
import io.ktor.utils.io.core.Output
|
||||
import kotlinx.coroutines.runBlocking
|
||||
import kotlinx.coroutines.test.runTest
|
||||
import kotlinx.io.Sink
|
||||
import kotlinx.io.Source
|
||||
import kotlinx.io.readString
|
||||
import kotlinx.io.writeString
|
||||
import space.kscience.dataforge.context.Context
|
||||
import space.kscience.dataforge.context.Global
|
||||
import space.kscience.dataforge.data.*
|
||||
import space.kscience.dataforge.io.Envelope
|
||||
import space.kscience.dataforge.io.IOFormat
|
||||
import space.kscience.dataforge.io.io
|
||||
import space.kscience.dataforge.io.readEnvelopeFile
|
||||
import space.kscience.dataforge.io.*
|
||||
import space.kscience.dataforge.io.yaml.YamlPlugin
|
||||
import space.kscience.dataforge.meta.get
|
||||
import space.kscience.dataforge.misc.DFExperimental
|
||||
@ -42,11 +36,11 @@ class FileDataTest {
|
||||
object StringIOFormat : IOFormat<String> {
|
||||
override val type: KType get() = typeOf<String>()
|
||||
|
||||
override fun writeObject(sink: Sink, obj: String) {
|
||||
sink.writeString(obj)
|
||||
override fun writeObject(output: Output, obj: String) {
|
||||
output.writeUtf8String(obj)
|
||||
}
|
||||
|
||||
override fun readObject(source: Source): String = source.readString()
|
||||
override fun readObject(input: Input): String = input.readUtf8String()
|
||||
}
|
||||
|
||||
@Test
|
||||
@ -65,9 +59,9 @@ class FileDataTest {
|
||||
|
||||
@Test
|
||||
@DFExperimental
|
||||
fun testZipWriteRead() = runTest {
|
||||
with(Global.io) {
|
||||
val zip = Files.createTempFile("df_data_node", ".zip")
|
||||
fun testZipWriteRead() = with(Global.io) {
|
||||
val zip = Files.createTempFile("df_data_node", ".zip")
|
||||
runBlocking {
|
||||
dataNode.writeZip(zip, StringIOFormat)
|
||||
println(zip.toUri().toString())
|
||||
val reconstructed = readDataDirectory(zip) { _, _ -> StringIOFormat }
|
||||
|
||||
@ -6,5 +6,4 @@ kotlin.mpp.stability.nowarn=true
|
||||
kotlin.incremental.js.ir=true
|
||||
kotlin.native.ignoreDisabledTargets=true
|
||||
|
||||
toolsVersion=0.15.0-kotlin-1.9.20-Beta
|
||||
#kotlin.experimental.tryK2=true
|
||||
toolsVersion=0.14.9-kotlin-1.8.20
|
||||
|
||||
2
gradle/wrapper/gradle-wrapper.properties
vendored
2
gradle/wrapper/gradle-wrapper.properties
vendored
@ -1,5 +1,5 @@
|
||||
distributionBase=GRADLE_USER_HOME
|
||||
distributionPath=wrapper/dists
|
||||
distributionUrl=https\://services.gradle.org/distributions/gradle-8.3-bin.zip
|
||||
distributionUrl=https\://services.gradle.org/distributions/gradle-8.2-bin.zip
|
||||
zipStoreBase=GRADLE_USER_HOME
|
||||
zipStorePath=wrapper/dists
|
||||
|
||||
Loading…
Reference in New Issue
Block a user