kscience/dataforge-core
7
0
Fork 0
Code Issues Pull Requests 1 Packages Projects Releases 12 Wiki Activity

Refactor io.

Browse Source
This commit is contained in:
Alexander Nozik 2023-10-08 18:20:23 +03:00
parent dc2bf5da83
commit cbbcd18df3
13 changed files with 70 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
CHANGELOG.md
View File

@ -3,15 +3,20 @@
## Unreleased
### Added
- Added separate `Meta`, `SealedMeta` and `ObservableMutableMeta` builders.
### Changed
- Kotlin 1.9
- `MutableMeta` builder now returns a simplified version of meta that does not hold listeners.
- Ktor-io is replaced with kotlinx-io.
- More concise names for read/write methods in IO.
### Deprecated
### Removed
### Fixed
- Memory leak in SealedMeta builder
### Security

2
build.gradle.kts
View File

@ -9,7 +9,7 @@ plugins {
allprojects {
group = "space.kscience"
version = "0.6.2"
version = "0.6.3-dev-1"
}
subprojects {

18
dataforge-io/dataforge-io-yaml/src/commonMain/kotlin/space/kscience/dataforge/io/yaml/FrontMatterEnvelopeFormat.kt
View File

@ -19,7 +19,7 @@ public class FrontMatterEnvelopeFormat(
private val metaFormatFactory: MetaFormatFactory = YamlMetaFormat,
) : EnvelopeFormat {
override fun readObject(binary: Binary): Envelope = binary.read {
override fun readFrom(binary: Binary): Envelope = binary.read {
var offset = 0
offset += discardWithSeparator(
@ -39,20 +39,20 @@ public class FrontMatterEnvelopeFormat(
offset += discardLine()
val meta = readMetaFormat.readObject(packet.asBinary())
val meta = readMetaFormat.readFrom(packet.asBinary())
Envelope(meta, binary.view(offset))
}
override fun readObject(source: Source): Envelope = readObject(source.readBinary())
override fun readFrom(source: Source): Envelope = readFrom(source.readBinary())
override fun writeObject(
override fun writeTo(
sink: Sink,
obj: Envelope,
) {
val metaFormat = metaFormatFactory.build(io.context, meta)
val formatSuffix = if (metaFormat is YamlMetaFormat) "" else metaFormatFactory.shortName
sink.writeString("$SEPARATOR${formatSuffix}\r\n")
metaFormat.run { metaFormat.writeObject(sink, obj.meta) }
metaFormat.run { metaFormat.writeTo(sink, obj.meta) }
sink.writeString("$SEPARATOR\r\n")
//Printing data
obj.data?.let { data ->
@ -83,15 +83,15 @@ public class FrontMatterEnvelopeFormat(
private val default by lazy { build(Global, Meta.EMPTY) }
override fun readObject(binary: Binary): Envelope = default.readObject(binary)
override fun readFrom(binary: Binary): Envelope = default.readFrom(binary)
override fun writeObject(
override fun writeTo(
sink: Sink,
obj: Envelope,
): Unit = default.writeObject(sink, obj)
): Unit = default.writeTo(sink, obj)
override fun readObject(source: Source): Envelope = default.readObject(source)
override fun readFrom(source: Source): Envelope = default.readFrom(source)
}
}

2
dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/EnvelopeFormat.kt
View File

@ -15,7 +15,7 @@ public interface EnvelopeFormat : IOFormat<Envelope> {
override val type: KType get() = typeOf<Envelope>()
}
public fun EnvelopeFormat.read(input: Source): Envelope = readObject(input)
public fun EnvelopeFormat.read(input: Source): Envelope = readFrom(input)
@Type(ENVELOPE_FORMAT_TYPE)
public interface EnvelopeFormatFactory : IOFormatFactory<Envelope>, EnvelopeFormat {

33
dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/IOFormat.kt
View File

@ -23,9 +23,9 @@ public interface IOReader<out T> {
*/
public val type: KType
public fun readObject(source: Source): T
public fun readFrom(source: Source): T
public fun readObject(binary: Binary): T = binary.read { readObject(this) }
public fun readFrom(binary: Binary): T = binary.read { readFrom(this) }
public companion object {
/**
@ -34,9 +34,9 @@ public interface IOReader<out T> {
public val binary: IOReader<Binary> = object : IOReader<Binary> {
override val type: KType = typeOf<Binary>()
override fun readObject(source: Source): Binary = source.readByteArray().asBinary()
override fun readFrom(source: Source): Binary = source.readByteArray().asBinary()
override fun readObject(binary: Binary): Binary = binary
override fun readFrom(binary: Binary): Binary = binary
}
}
}
@ -44,11 +44,11 @@ public interface IOReader<out T> {
public inline fun <reified T> IOReader(crossinline read: Source.() -> T): IOReader<T> = object : IOReader<T> {
override val type: KType = typeOf<T>()
override fun readObject(source: Source): T = source.read()
override fun readFrom(source: Source): T = source.read()
}
public fun interface IOWriter<in T> {
public fun writeObject(sink: Sink, obj: T)
public fun writeTo(sink: Sink, obj: T)
}
/**
@ -56,21 +56,20 @@ public fun interface IOWriter<in T> {
*/
public interface IOFormat<T> : IOReader<T>, IOWriter<T>
public fun <T : Any> Source.readObject(format: IOReader<T>): T = format.readObject(this@readObject)
public fun <T : Any> IOFormat<T>.readObjectFrom(binary: Binary): T = binary.read {
readObject(this)
}
public fun <T : Any> Source.readWith(format: IOReader<T>): T = format.readFrom(this)
/**
* Read given binary as an object using given format
*/
public fun <T : Any> Binary.readWith(format: IOReader<T>): T = read {
readObject(format)
readWith(format)
}
public fun <T : Any> Sink.writeObject(format: IOWriter<T>, obj: T): Unit =
format.writeObject(this@writeObject, obj)
/**
* Write an object to the [Sink] with given [format]
*/
public fun <T : Any> Sink.writeWith(format: IOWriter<T>, obj: T): Unit =
format.writeTo(this, obj)
@Type(IO_FORMAT_TYPE)
@ -87,7 +86,7 @@ public interface IOFormatFactory<T : Any> : Factory<IOFormat<T>>, Named {
}
}
public fun <T : Any> Binary(obj: T, format: IOWriter<T>): Binary = Binary { format.writeObject(this, obj) }
public fun <T : Any> Binary(obj: T, format: IOWriter<T>): Binary = Binary { format.writeTo(this, obj) }
public object DoubleIOFormat : IOFormat<Double>, IOFormatFactory<Double> {
override fun build(context: Context, meta: Meta): IOFormat<Double> = this
@ -96,9 +95,9 @@ public object DoubleIOFormat : IOFormat<Double>, IOFormatFactory<Double> {
override val type: KType get() = typeOf<Double>()
override fun writeObject(sink: Sink, obj: Double) {
override fun writeTo(sink: Sink, obj: Double) {
sink.writeLong(obj.toBits())
}
override fun readObject(source: Source): Double = Double.fromBits(source.readLong())
override fun readFrom(source: Source): Double = Double.fromBits(source.readLong())
}

8
dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/MetaFormat.kt
View File

@ -23,11 +23,11 @@ public interface MetaFormat : IOFormat<Meta> {
override val type: KType get() = typeOf<Meta>()
override fun writeObject(sink: Sink, obj: Meta) {
override fun writeTo(sink: Sink, obj: Meta) {
writeMeta(sink, obj, null)
}
override fun readObject(source: Source): Meta = readMeta(source)
override fun readFrom(source: Source): Meta = readMeta(source)
public fun writeMeta(
sink: Sink,
@ -57,13 +57,13 @@ public interface MetaFormatFactory : IOFormatFactory<Meta>, MetaFormat {
public fun Meta.toString(format: MetaFormat): String = ByteArray {
format.run {
writeObject(this@ByteArray, this@toString)
writeTo(this@ByteArray, this@toString)
}
}.decodeToString()
public fun Meta.toString(formatFactory: MetaFormatFactory): String = toString(formatFactory.build(Global, Meta.EMPTY))
public fun MetaFormat.parse(str: String): Meta = readObject(StringSource(str).buffered())
public fun MetaFormat.parse(str: String): Meta = readFrom(StringSource(str).buffered())
public fun MetaFormatFactory.parse(str: String, formatMeta: Meta): Meta = build(Global, formatMeta).parse(str)

20
dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/TaggedEnvelopeFormat.kt
View File

@ -43,7 +43,7 @@ public class TaggedEnvelopeFormat(
write(END_SEQUENCE)
}
override fun writeObject(
override fun writeTo(
sink: Sink,
obj: Envelope,
) {
@ -65,7 +65,7 @@ public class TaggedEnvelopeFormat(
* @param source an input to read from
* @param formats a collection of meta formats to resolve
*/
override fun readObject(source: Source): Envelope {
override fun readFrom(source: Source): Envelope {
val tag = source.readTag(this.version)
val metaFormat = io.resolveMetaFormat(tag.metaFormatKey)
@ -73,14 +73,14 @@ public class TaggedEnvelopeFormat(
val metaBinary = source.readBinary(tag.metaSize.toInt())
val meta: Meta = metaFormat.readObjectFrom(metaBinary)
val meta: Meta = metaFormat.readFrom(metaBinary)
val data = source.readBinary(tag.dataSize.toInt())
return SimpleEnvelope(meta, data)
}
override fun readObject(binary: Binary): Envelope = binary.read {
override fun readFrom(binary: Binary): Envelope = binary.read {
val tag = readTag(version)
val metaFormat = io.resolveMetaFormat(tag.metaFormatKey)
@ -88,7 +88,7 @@ public class TaggedEnvelopeFormat(
val metaBinary = readBinary(tag.metaSize.toInt())
val meta: Meta = metaFormat.readObjectFrom(metaBinary)
val meta: Meta = metaFormat.readFrom(metaBinary)
SimpleEnvelope(meta, binary.view((version.tagSize + tag.metaSize).toInt(), tag.dataSize.toInt()))
@ -154,17 +154,17 @@ public class TaggedEnvelopeFormat(
private val default by lazy { build(Global, Meta.EMPTY) }
override fun readObject(binary: Binary): Envelope =
default.run { readObject(binary) }
override fun readFrom(binary: Binary): Envelope =
default.run { readFrom(binary) }
override fun writeObject(
override fun writeTo(
sink: Sink,
obj: Envelope,
): Unit = default.run {
writeObject(sink, obj)
writeTo(sink, obj)
}
override fun readObject(source: Source): Envelope = default.readObject(source)
override fun readFrom(source: Source): Envelope = default.readFrom(source)
}
}

12
dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/TaglessEnvelopeFormat.kt
View File

@ -28,7 +28,7 @@ public class TaglessEnvelopeFormat(
// writeFully("#? $key: $value;\r\n".encodeToByteArray())
// }
override fun writeObject(
override fun writeTo(
sink: Sink,
obj: Envelope,
) {
@ -54,7 +54,7 @@ public class TaglessEnvelopeFormat(
}
}
override fun readObject(source: Source): Envelope {
override fun readFrom(source: Source): Envelope {
//read preamble
source.discardWithSeparator(
TAGLESS_ENVELOPE_HEADER,
@ -132,16 +132,16 @@ public class TaglessEnvelopeFormat(
private val default by lazy { build(Global, Meta.EMPTY) }
override fun readObject(binary: Binary): Envelope = default.run { readObject(binary) }
override fun readFrom(binary: Binary): Envelope = default.run { readFrom(binary) }
override fun writeObject(
override fun writeTo(
sink: Sink,
obj: Envelope,
): Unit = default.run {
writeObject(sink, obj)
writeTo(sink, obj)
}
override fun readObject(source: Source): Envelope = default.readObject(source)
override fun readFrom(source: Source): Envelope = default.readFrom(source)
override fun peekFormat(io: IOPlugin, binary: Binary): EnvelopeFormat? {
return try {

2
dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/ioMisc.kt
View File

@ -57,7 +57,7 @@ public fun IOPlugin.readEnvelope(
binary: Binary,
readNonEnvelopes: Boolean = false,
formatPicker: IOPlugin.(Binary) -> EnvelopeFormat? = IOPlugin::peekBinaryEnvelopeFormat,
): Envelope = formatPicker(binary)?.readObject(binary) ?: if (readNonEnvelopes) {
): Envelope = formatPicker(binary)?.readFrom(binary) ?: if (readNonEnvelopes) {
// if no format accepts file, read it as binary
Envelope(Meta.EMPTY, binary)
} else error("Can't infer format for $binary")

4
dataforge-io/src/jvmMain/kotlin/space/kscience/dataforge/io/fileIO.kt
View File

@ -78,7 +78,7 @@ public fun Path.rewrite(block: Sink.() -> Unit): Unit {
}
@DFExperimental
public fun EnvelopeFormat.readFile(path: Path): Envelope = readObject(path.asBinary())
public fun EnvelopeFormat.readFile(path: Path): Envelope = readFrom(path.asBinary())
/**
* Resolve IOFormat based on type
@ -238,7 +238,7 @@ public fun IOPlugin.writeEnvelopeFile(
envelopeFormat: EnvelopeFormat = TaggedEnvelopeFormat,
) {
path.rewrite {
envelopeFormat.writeObject(this, envelope)
envelopeFormat.writeTo(this, envelope)
}
}

14
dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/FileWorkspaceCache.kt
View File

@ -27,10 +27,10 @@ public class JsonIOFormat<T : Any>(override val type: KType) : IOFormat<T> {
@Suppress("UNCHECKED_CAST")
private val serializer: KSerializer<T> = serializer(type) as KSerializer<T>
override fun readObject(input: Source): T = Json.decodeFromString(serializer, input.readString())
override fun readFrom(source: Source): T = Json.decodeFromString(serializer, source.readString())
override fun writeObject(output: Sink, obj: T) {
output.writeString(Json.encodeToString(serializer, obj))
override fun writeTo(sink: Sink, obj: T) {
sink.writeString(Json.encodeToString(serializer, obj))
}
}
@ -40,10 +40,10 @@ public class ProtobufIOFormat<T : Any>(override val type: KType) : IOFormat<T> {
@Suppress("UNCHECKED_CAST")
private val serializer: KSerializer<T> = serializer(type) as KSerializer<T>
override fun readObject(input: Source): T = ProtoBuf.decodeFromByteArray(serializer, input.readByteArray())
override fun readFrom(source: Source): T = ProtoBuf.decodeFromByteArray(serializer, source.readByteArray())
override fun writeObject(output: Sink, obj: T) {
output.write(ProtoBuf.encodeToByteArray(serializer, obj))
override fun writeTo(sink: Sink, obj: T) {
sink.write(ProtoBuf.encodeToByteArray(serializer, obj))
}
}
@ -85,7 +85,7 @@ public class FileWorkspaceCache(public val cacheDirectory: Path) : WorkspaceCach
val envelope = Envelope {
meta = data.meta
data {
writeObject(format, result)
writeWith(format, result)
}
}
io.writeEnvelopeFile(path, envelope)

2
dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/zipData.kt
View File

@ -28,7 +28,7 @@ private suspend fun <T : Any> ZipOutputStream.writeNode(
//TODO remove additional copy
val bytes = ByteArray {
writeObject(envelopeFormat, envelope)
writeWith(envelopeFormat, envelope)
}
write(bytes)

11
dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/FileDataTest.kt
View File

@ -9,7 +9,10 @@ import kotlinx.io.writeString
import space.kscience.dataforge.context.Context
import space.kscience.dataforge.context.Global
import space.kscience.dataforge.data.*
import space.kscience.dataforge.io.*
import space.kscience.dataforge.io.Envelope
import space.kscience.dataforge.io.IOFormat
import space.kscience.dataforge.io.io
import space.kscience.dataforge.io.readEnvelopeFile
import space.kscience.dataforge.io.yaml.YamlPlugin
import space.kscience.dataforge.meta.get
import space.kscience.dataforge.misc.DFExperimental
@ -39,11 +42,11 @@ class FileDataTest {
object StringIOFormat : IOFormat<String> {
override val type: KType get() = typeOf<String>()
override fun writeObject(output: Sink, obj: String) {
output.writeString(obj)
override fun writeTo(sink: Sink, obj: String) {
sink.writeString(obj)
}
override fun readObject(input: Source): String = input.readString()
override fun readFrom(source: Source): String = source.readString()
}
@Test