kscience/dataforge-core
8
0
Fork 0
Code Issues Pull Requests 1 Packages Projects Releases 12 Wiki Activity

- Remove all unnecessary properties for IOFormat

Browse Source 
- Separate interfaces for `IOReader` and `IOWriter`
This commit is contained in:
Alexander Nozik 2022-05-16 18:57:48 +03:00
parent a546552540
commit 0fc2198832
No known key found for this signature in database
GPG Key ID: F7FCF2DD25C71357
20 changed files with 142 additions and 261 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
CHANGELOG.md
View File

@ -17,6 +17,8 @@
- `ActiveDataSet` renamed to `DataSource`
- `selectOne`->`getByType`
- Data traversal in `DataSet` is done via iterator
- Remove all unnecessary properties for `IOFormat`
- Separate interfaces for `IOReader` and `IOWriter`
### Deprecated

8
build.gradle.kts
View File

@ -1,3 +1,5 @@
import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
plugins {
id("ru.mipt.npm.gradle.project")
}
@ -5,6 +7,12 @@ plugins {
allprojects {
group = "space.kscience"
version = "0.6.0-dev-7"
tasks.withType<KotlinCompile>{
kotlinOptions{
freeCompilerArgs = freeCompilerArgs + "-Xcontext-receivers"
}
}
}
subprojects {

6
dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/provider/Provider.kt
View File

@ -75,10 +75,8 @@ public inline fun <reified T : Any> Provider.provide(path: String, targetOverrid
/**
* Typed top level content
*/
public fun <T : Any> Provider.top(target: String, type: KClass<out T>): Map<Name, T> {
return content(target).mapValues {
type.safeCast(it.value) ?: error("The type of element $it is ${it::class} but $type is expected")
}
public fun <T : Any> Provider.top(target: String, type: KClass<out T>): Map<Name, T> = content(target).mapValues {
type.safeCast(it.value) ?: error("The type of element ${it.value} is ${it.value::class} but $type is expected")
}
/**

8
dataforge-io/dataforge-io-yaml/src/commonMain/kotlin/space/kscience/dataforge/io/yaml/FrontMatterEnvelopeFormat.kt
View File

@ -7,10 +7,7 @@ import io.ktor.utils.io.core.readBytes
import space.kscience.dataforge.context.Context
import space.kscience.dataforge.context.Global
import space.kscience.dataforge.io.*
import space.kscience.dataforge.io.IOFormat.Companion.META_KEY
import space.kscience.dataforge.io.IOFormat.Companion.NAME_KEY
import space.kscience.dataforge.meta.Meta
import space.kscience.dataforge.misc.DFExperimental
import space.kscience.dataforge.names.Name
import space.kscience.dataforge.names.plus
@ -65,11 +62,6 @@ public class FrontMatterEnvelopeFormat(
}
}
override fun toMeta(): Meta = Meta {
NAME_KEY put name.toString()
META_KEY put meta
}
public companion object : EnvelopeFormatFactory {
public const val SEPARATOR: String = "---"

8
dataforge-io/dataforge-io-yaml/src/commonMain/kotlin/space/kscience/dataforge/io/yaml/YamlMetaFormat.kt
View File

@ -4,8 +4,6 @@ import io.ktor.utils.io.core.Input
import io.ktor.utils.io.core.Output
import net.mamoe.yamlkt.*
import space.kscience.dataforge.context.Context
import space.kscience.dataforge.io.IOFormat.Companion.META_KEY
import space.kscience.dataforge.io.IOFormat.Companion.NAME_KEY
import space.kscience.dataforge.io.MetaFormat
import space.kscience.dataforge.io.MetaFormatFactory
import space.kscience.dataforge.io.readUtf8String
@ -14,7 +12,6 @@ import space.kscience.dataforge.meta.Meta
import space.kscience.dataforge.meta.descriptors.MetaDescriptor
import space.kscience.dataforge.meta.descriptors.get
import space.kscience.dataforge.meta.isLeaf
import space.kscience.dataforge.misc.DFExperimental
import space.kscience.dataforge.names.NameToken
import space.kscience.dataforge.names.withIndex
import space.kscience.dataforge.values.ListValue
@ -108,11 +105,6 @@ public class YamlMetaFormat(private val meta: Meta) : MetaFormat {
return yaml.toMeta()
}
override fun toMeta(): Meta = Meta {
NAME_KEY put FrontMatterEnvelopeFormat.name.toString()
META_KEY put meta
}
public companion object : MetaFormatFactory {
override fun build(context: Context, meta: Meta): MetaFormat = YamlMetaFormat(meta)

2
dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/EnvelopeFormat.kt
View File

@ -17,8 +17,6 @@ import kotlin.reflect.typeOf
public data class PartialEnvelope(val meta: Meta, val dataOffset: Int, val dataSize: ULong?)
public interface EnvelopeFormat : IOFormat<Envelope> {
override val type: KType get() = typeOf<Envelope>()
public val defaultMetaFormat: MetaFormatFactory get() = JsonMetaFormat
public fun readPartial(input: Input): PartialEnvelope

21
dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/EnvelopeParts.kt
View File

@ -1,5 +1,6 @@
package space.kscience.dataforge.io
import space.kscience.dataforge.context.invoke
import space.kscience.dataforge.io.Envelope.Companion.ENVELOPE_NODE_KEY
import space.kscience.dataforge.io.PartDescriptor.Companion.DEFAULT_MULTIPART_DATA_SEPARATOR
import space.kscience.dataforge.io.PartDescriptor.Companion.MULTIPART_DATA_TYPE
@ -35,7 +36,7 @@ public typealias EnvelopeParts = List<EnvelopePart>
public fun EnvelopeBuilder.multipart(
parts: EnvelopeParts,
separator: String = DEFAULT_MULTIPART_DATA_SEPARATOR
separator: String = DEFAULT_MULTIPART_DATA_SEPARATOR,
) {
dataType = MULTIPART_DATA_TYPE
@ -67,17 +68,25 @@ public fun EnvelopeBuilder.multipart(
}
}
/**
* Put a list of envelopes as parts of given envelope
*/
public fun EnvelopeBuilder.envelopes(
envelopes: List<Envelope>,
format: EnvelopeFormat = TaggedEnvelopeFormat,
separator: String = DEFAULT_MULTIPART_DATA_SEPARATOR
formatFactory: EnvelopeFormatFactory = TaggedEnvelopeFormat,
formatMeta: Meta? = null,
separator: String = DEFAULT_MULTIPART_DATA_SEPARATOR,
) {
val parts = envelopes.map {
val binary = format.toBinary(it)
val format = formatMeta?.let { formatFactory(formatMeta) } ?: formatFactory
val binary = Binary(it, format)
EnvelopePart(binary, null)
}
meta{
set(MULTIPART_KEY + PART_FORMAT_KEY, format.toMeta())
meta {
(MULTIPART_KEY + PART_FORMAT_KEY) put {
IOFormatFactory.NAME_KEY put formatFactory.name.toString()
formatMeta?.let { IOFormatFactory.META_KEY put formatMeta }
}
}
multipart(parts, separator)
}

105
dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/IOFormat.kt
View File

@ -1,12 +1,13 @@
package space.kscience.dataforge.io
import io.ktor.utils.io.core.*
import io.ktor.utils.io.core.Input
import io.ktor.utils.io.core.Output
import io.ktor.utils.io.core.readDouble
import io.ktor.utils.io.core.writeDouble
import space.kscience.dataforge.context.Context
import space.kscience.dataforge.context.Factory
import space.kscience.dataforge.io.IOFormat.Companion.NAME_KEY
import space.kscience.dataforge.io.IOFormatFactory.Companion.IO_FORMAT_TYPE
import space.kscience.dataforge.meta.Meta
import space.kscience.dataforge.meta.MetaRepr
import space.kscience.dataforge.misc.Named
import space.kscience.dataforge.misc.Type
import space.kscience.dataforge.names.Name
@ -14,92 +15,53 @@ import space.kscience.dataforge.names.asName
import kotlin.reflect.KType
import kotlin.reflect.typeOf
public fun interface IOReader<out T : Any> {
public fun readObject(input: Input): T
}
public fun interface IOWriter<in T : Any> {
public fun writeObject(output: Output, obj: T)
}
/**
* And interface for reading and writing objects into with IO streams
*/
public interface IOFormat<T : Any> : MetaRepr {
public val type: KType
public interface IOFormat<T : Any> : IOReader<T>, IOWriter<T>
public fun writeObject(output: Output, obj: T)
public fun readObject(input: Input): T
public fun <T : Any> Input.readObject(format: IOReader<T>): T = format.readObject(this@readObject)
public companion object {
public val NAME_KEY: Name = "name".asName()
public val META_KEY: Name = "meta".asName()
}
}
public fun <T : Any> Input.readWith(format: IOFormat<T>): T = format.readObject(this@readWith)
public fun <T: Any> IOFormat<T>.readObject(binary: Binary): T = binary.read {
public fun <T : Any> IOFormat<T>.readObjectFrom(binary: Binary): T = binary.read {
readObject(this)
}
/**
* Read given binary as object using given format
*/
public fun <T : Any> Binary.readWith(format: IOFormat<T>): T = read {
readWith(format)
public fun <T : Any> Binary.readWith(format: IOReader<T>): T = read {
readObject(format)
}
public fun <T : Any> Output.writeWith(format: IOFormat<T>, obj: T): Unit =
format.run { writeObject(this@writeWith, obj) }
public fun <T : Any> Output.writeObject(format: IOWriter<T>, obj: T): Unit =
format.run { writeObject(this@writeObject, obj) }
public inline fun <reified T : Any> IOFormat.Companion.listOf(
format: IOFormat<T>,
): IOFormat<List<T>> = object : IOFormat<List<T>> {
override val type: KType = typeOf<List<T>>()
override fun writeObject(output: Output, obj: List<T>) {
output.writeInt(obj.size)
format.run {
obj.forEach {
writeObject(output, it)
}
}
}
override fun readObject(input: Input): List<T> {
val size = input.readInt()
return format.run {
List(size) { readObject(input) }
}
}
override fun toMeta(): Meta = Meta {
NAME_KEY put "list"
"contentFormat" put format.toMeta()
}
}
//public fun ObjectPool<Buffer>.fill(block: Buffer.() -> Unit): Buffer {
// val buffer = borrow()
// return try {
// buffer.apply(block)
// } catch (ex: Exception) {
// //recycle(buffer)
// throw ex
// }
//}
@Type(IO_FORMAT_TYPE)
public interface IOFormatFactory<T : Any> : Factory<IOFormat<T>>, Named, MetaRepr {
public interface IOFormatFactory<T : Any> : Factory<IOFormat<T>>, Named {
/**
* Explicit type for dynamic type checks
*/
public val type: KType
override fun toMeta(): Meta = Meta {
NAME_KEY put name.toString()
}
public companion object {
public const val IO_FORMAT_TYPE: String = "io.format"
public val NAME_KEY: Name = "name".asName()
public val META_KEY: Name = "meta".asName()
}
}
public fun <T : Any> IOFormat<T>.toBinary(obj: T): Binary = Binary { writeObject(this, obj) }
public fun <T : Any> Binary(obj: T, format: IOWriter<T>): Binary = Binary { format.writeObject(this, obj) }
public object DoubleIOFormat : IOFormat<Double>, IOFormatFactory<Double> {
override fun build(context: Context, meta: Meta): IOFormat<Double> = this
@ -114,20 +76,3 @@ public object DoubleIOFormat : IOFormat<Double>, IOFormatFactory<Double> {
override fun readObject(input: Input): Double = input.readDouble()
}
//public object ValueIOFormat : IOFormat<Value>, IOFormatFactory<Value> {
// override fun invoke(meta: Meta, context: Context): IOFormat<Value> = this
//
// override val name: Name = "value".asName()
//
// override val type: KType get() = typeOf<Value>()
//
// override fun writeObject(output: Output, obj: Value) {
// BinaryMetaFormat.run { output.writeValue(obj) }
// }
//
// override fun readObject(input: Input): Value {
// return (BinaryMetaFormat.run { input.readMetaItem() } as? MetaItemValue)?.value
// ?: error("The item is not a value")
// }
//}

10
dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/IOPlugin.kt
View File

@ -2,8 +2,6 @@ package space.kscience.dataforge.io
import space.kscience.dataforge.context.*
import space.kscience.dataforge.io.EnvelopeFormatFactory.Companion.ENVELOPE_FORMAT_TYPE
import space.kscience.dataforge.io.IOFormat.Companion.META_KEY
import space.kscience.dataforge.io.IOFormat.Companion.NAME_KEY
import space.kscience.dataforge.io.IOFormatFactory.Companion.IO_FORMAT_TYPE
import space.kscience.dataforge.io.MetaFormatFactory.Companion.META_FORMAT_TYPE
import space.kscience.dataforge.meta.Meta
@ -20,12 +18,12 @@ public class IOPlugin(meta: Meta) : AbstractPlugin(meta) {
}
public fun <T : Any> resolveIOFormat(item: Meta, type: KClass<out T>): IOFormat<T>? {
val key = item.string ?: item[NAME_KEY]?.string ?: error("Format name not defined")
val key = item.string ?: item[IOFormatFactory.NAME_KEY]?.string ?: error("Format name not defined")
val name = Name.parse(key)
return ioFormatFactories.find { it.name == name }?.let {
@Suppress("UNCHECKED_CAST")
if (it.type != type) error("Format type ${it.type} is not the same as requested type $type")
else it.build(context, item[META_KEY] ?: Meta.EMPTY) as IOFormat<T>
else it.build(context, item[IOFormatFactory.META_KEY] ?: Meta.EMPTY) as IOFormat<T>
}
}
@ -47,8 +45,8 @@ public class IOPlugin(meta: Meta) : AbstractPlugin(meta) {
envelopeFormatFactories.find { it.name == name }?.build(context, meta)
public fun resolveEnvelopeFormat(item: Meta): EnvelopeFormat? {
val name = item.string ?: item[NAME_KEY]?.string ?: error("Envelope format name not defined")
val meta = item[META_KEY] ?: Meta.EMPTY
val name = item.string ?: item[IOFormatFactory.NAME_KEY]?.string ?: error("Envelope format name not defined")
val meta = item[IOFormatFactory.META_KEY] ?: Meta.EMPTY
return resolveEnvelopeFormat(Name.parse(name), meta)
}

9
dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/JsonMetaFormat.kt
View File

@ -8,30 +8,21 @@ import io.ktor.utils.io.core.Output
import kotlinx.serialization.json.Json
import kotlinx.serialization.json.JsonObject
import space.kscience.dataforge.context.Context
import space.kscience.dataforge.io.IOFormat.Companion.NAME_KEY
import space.kscience.dataforge.meta.Meta
import space.kscience.dataforge.meta.descriptors.MetaDescriptor
import space.kscience.dataforge.meta.toJson
import space.kscience.dataforge.meta.toMeta
import kotlin.reflect.KType
import kotlin.reflect.typeOf
/**
* A Json format for Meta representation
*/
public class JsonMetaFormat(private val json: Json = DEFAULT_JSON) : MetaFormat {
override val type: KType get() = typeOf<Meta>()
override fun writeMeta(output: Output, meta: Meta, descriptor: MetaDescriptor?) {
val jsonObject = meta.toJson(descriptor)
output.writeUtf8String(json.encodeToString(JsonObject.serializer(), jsonObject))
}
override fun toMeta(): Meta = Meta {
NAME_KEY put name.toString()
}
override fun readMeta(input: Input, descriptor: MetaDescriptor?): Meta {
val str = input.readUtf8String()//readByteArray().decodeToString()
val jsonElement = json.parseToJsonElement(str)

1
dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/MetaFormat.kt
View File

@ -20,7 +20,6 @@ import kotlin.reflect.typeOf
* A format for meta serialization
*/
public interface MetaFormat : IOFormat<Meta> {
override val type: KType get() = typeOf<Meta>()
override fun writeObject(output: Output, obj: Meta) {
writeMeta(output, obj, null)

15
dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/TaggedEnvelopeFormat.kt
View File

@ -3,8 +3,6 @@ package space.kscience.dataforge.io
import io.ktor.utils.io.core.*
import space.kscience.dataforge.context.Context
import space.kscience.dataforge.context.Global
import space.kscience.dataforge.io.IOFormat.Companion.META_KEY
import space.kscience.dataforge.io.IOFormat.Companion.NAME_KEY
import space.kscience.dataforge.meta.Meta
import space.kscience.dataforge.meta.enum
import space.kscience.dataforge.meta.get
@ -49,7 +47,7 @@ public class TaggedEnvelopeFormat(
formatMeta: Meta,
) {
val metaFormat = metaFormatFactory.build(this@TaggedEnvelopeFormat.io.context, formatMeta)
val metaBytes = metaFormat.toBinary(envelope.meta)
val metaBytes = Binary(envelope.meta,metaFormat)
val actualSize: ULong = (envelope.data?.size ?: 0).toULong()
val tag = Tag(metaFormatFactory.key, metaBytes.size.toUInt() + 2u, actualSize)
output.writeBinary(tag.toBinary())
@ -74,7 +72,7 @@ public class TaggedEnvelopeFormat(
val metaBinary = input.readBinary(tag.metaSize.toInt())
val meta: Meta = metaFormat.readObject(metaBinary)
val meta: Meta = metaFormat.readObjectFrom(metaBinary)
val data = input.readBinary(tag.dataSize.toInt())
@ -89,7 +87,7 @@ public class TaggedEnvelopeFormat(
val metaBinary = input.readBinary(tag.metaSize.toInt())
val meta: Meta = metaFormat.readObject(metaBinary)
val meta: Meta = metaFormat.readObjectFrom(metaBinary)
return PartialEnvelope(meta, (version.tagSize + tag.metaSize).toInt(), tag.dataSize)
@ -106,13 +104,6 @@ public class TaggedEnvelopeFormat(
DF03(24u)
}
override fun toMeta(): Meta = Meta {
NAME_KEY put name.toString()
META_KEY put {
"version" put version
}
}
public companion object : EnvelopeFormatFactory {
private const val START_SEQUENCE = "#~"
private const val END_SEQUENCE = "~#\r\n"

17
dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/TaglessEnvelopeFormat.kt
View File

@ -3,8 +3,6 @@ package space.kscience.dataforge.io
import io.ktor.utils.io.core.*
import space.kscience.dataforge.context.Context
import space.kscience.dataforge.context.Global
import space.kscience.dataforge.io.IOFormat.Companion.META_KEY
import space.kscience.dataforge.io.IOFormat.Companion.NAME_KEY
import space.kscience.dataforge.meta.Meta
import space.kscience.dataforge.meta.get
import space.kscience.dataforge.meta.isEmpty
@ -50,11 +48,11 @@ public class TaglessEnvelopeFormat(
//Printing meta
if (!envelope.meta.isEmpty()) {
val metaBytes = metaFormat.toBinary(envelope.meta)
val metaBinary = Binary(envelope.meta, metaFormat)
output.writeProperty(META_LENGTH_PROPERTY,
metaBytes.size + 2)
metaBinary.size + 2)
output.writeUtf8String(this.metaStart + "\r\n")
output.writeBinary(metaBytes)
output.writeBinary(metaBinary)
output.writeRawString("\r\n")
}
@ -102,7 +100,7 @@ public class TaglessEnvelopeFormat(
val metaFormat = properties[META_TYPE_PROPERTY]?.let { io.resolveMetaFormat(it) } ?: JsonMetaFormat
val metaSize = properties[META_LENGTH_PROPERTY]?.toInt()
meta = if (metaSize != null) {
metaFormat.readObject(input.readBinary(metaSize))
metaFormat.readObjectFrom(input.readBinary(metaSize))
} else {
error("Can't partially read an envelope with undefined meta size")
}
@ -170,7 +168,7 @@ public class TaglessEnvelopeFormat(
val metaSize = properties[META_LENGTH_PROPERTY]?.toInt()
meta = if (metaSize != null) {
offset += metaSize
metaFormat.readObject(input.readBinary(metaSize))
metaFormat.readObjectFrom(input.readBinary(metaSize))
} else {
error("Can't partially read an envelope with undefined meta size")
}
@ -187,11 +185,6 @@ public class TaglessEnvelopeFormat(
return PartialEnvelope(meta, offset, dataSize)
}
override fun toMeta(): Meta = Meta {
NAME_KEY put name.toString()
META_KEY put meta
}
public companion object : EnvelopeFormatFactory {
private val propertyPattern = "#\\?\\s*([\\w.]*)\\s*:\\s*([^;]*);?".toRegex()

2
dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/ioMisc.kt
View File

@ -183,7 +183,7 @@ public fun Input.discardWithSeparator(
atMost: Int = Int.MAX_VALUE,
skipUntilEndOfLine: Boolean = false,
): Int {
val dummy: Output = object :Output(ChunkBuffer.Pool){
val dummy: Output = object : Output(ChunkBuffer.Pool) {
override fun closeDestination() {
// Do nothing
}

23
dataforge-io/src/commonTest/kotlin/space/kscience/dataforge/io/MultipartTest.kt
View File

@ -32,18 +32,17 @@ class MultipartTest {
@Test
fun testParts() {
TaglessEnvelopeFormat.run {
val singleEnvelopeData = toBinary(envelopes[0])
val singleEnvelopeSize = singleEnvelopeData.size
val bytes = toBinary(partsEnvelope)
assertTrue(envelopes.size * singleEnvelopeSize < bytes.size)
val reconstructed = bytes.readWith(this)
println(reconstructed.meta)
val parts = reconstructed.parts()
val envelope = parts[2].envelope(io)
assertEquals(2, envelope.meta["value"].int)
println(reconstructed.data!!.size)
}
val format = TaglessEnvelopeFormat
val singleEnvelopeData = Binary(envelopes[0], format)
val singleEnvelopeSize = singleEnvelopeData.size
val bytes = Binary(partsEnvelope, format)
assertTrue(envelopes.size * singleEnvelopeSize < bytes.size)
val reconstructed = bytes.readWith(format)
println(reconstructed.meta)
val parts = reconstructed.parts()
val envelope = parts[2].envelope(io)
assertEquals(2, envelope.meta["value"].int)
println(reconstructed.data!!.size)
}
}

45
dataforge-io/src/jvmMain/kotlin/space/kscience/dataforge/io/fileIO.kt
View File

@ -97,7 +97,36 @@ public val IOPlugin.Companion.DATA_FILE_NAME: String get() = "@data"
/**
* Read file containing meta using given [formatOverride] or file extension to infer meta type.
* If [path] is a directory search for file starting with `meta` in it
* If [path] is a directory search for file starting with `meta` in it.
*
* Returns null if meta could not be resolved
*/
public fun IOPlugin.readMetaFileOrNull(
path: Path,
formatOverride: MetaFormat? = null,
descriptor: MetaDescriptor? = null,
): Meta? {
if (!Files.exists(path)) return null
val actualPath: Path = if (Files.isDirectory(path)) {
Files.list(path).asSequence().find { it.fileName.startsWith(IOPlugin.META_FILE_NAME) }
?: return null
} else {
path
}
val extension = actualPath.fileName.toString().substringAfterLast('.')
val metaFormat = formatOverride ?: resolveMetaFormat(extension) ?: return null
return actualPath.read {
metaFormat.readMeta(this, descriptor)
}
}
/**
* Read file containing meta using given [formatOverride] or file extension to infer meta type.
* If [path] is a directory search for file starting with `meta` in it.
*
* Fails if nothing works.
*/
public fun IOPlugin.readMetaFile(
path: Path,
@ -120,6 +149,7 @@ public fun IOPlugin.readMetaFile(
}
}
/**
* Write meta to file using [metaFormat]. If [path] is a directory, write a file with name equals name of [metaFormat].
* Like "meta.json"
@ -196,22 +226,11 @@ public fun IOPlugin.readEnvelopeFile(
return SimpleEnvelope(meta, data)
}
return formatPicker(path)?.let { format ->
format.readFile(path)
} ?: if (readNonEnvelopes) { // if no format accepts file, read it as binary
return formatPicker(path)?.readFile(path) ?: if (readNonEnvelopes) { // if no format accepts file, read it as binary
SimpleEnvelope(Meta.EMPTY, path.asBinary())
} else error("Can't infer format for file $path")
}
/**
* Write a binary into file. Throws an error if file already exists
*/
public fun <T : Any> IOFormat<T>.writeToFile(path: Path, obj: T) {
path.write {
writeObject(this, obj)
}
}
/**
* Write envelope file to given [path] using [envelopeFormat] and optional [metaFormat]
*/

11
dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/envelopeData.kt
View File

@ -4,19 +4,18 @@ import space.kscience.dataforge.data.Data
import space.kscience.dataforge.data.await
import space.kscience.dataforge.io.*
import space.kscience.dataforge.misc.DFInternal
import kotlin.reflect.typeOf
/**
* Convert an [Envelope] to a data via given format. The actual parsing is done lazily.
*/
@OptIn(DFInternal::class)
public fun <T : Any> Envelope.toData(format: IOFormat<T>): Data<T> {
return Data(format.type, meta) {
data?.readWith(format) ?: error("Can't convert envelope without data to Data")
}
public inline fun <reified T : Any> Envelope.toData(format: IOReader<T>): Data<T> = Data(typeOf<T>(), meta) {
data?.readWith(format) ?: error("Can't convert envelope without data to Data")
}
public suspend fun <T : Any> Data<T>.toEnvelope(format: IOFormat<T>): Envelope {
public suspend fun <T : Any> Data<T>.toEnvelope(format: IOWriter<T>): Envelope {
val obj = await()
val binary = format.toBinary(obj)
val binary = Binary(obj, format)
return SimpleEnvelope(meta, binary)
}

85
dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/fileData.kt
View File

@ -2,11 +2,12 @@ package space.kscience.dataforge.workspace
import io.ktor.utils.io.streams.asOutput
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.runBlocking
import kotlinx.coroutines.withContext
import space.kscience.dataforge.data.*
import space.kscience.dataforge.io.*
import space.kscience.dataforge.meta.*
import space.kscience.dataforge.meta.Meta
import space.kscience.dataforge.meta.get
import space.kscience.dataforge.meta.string
import space.kscience.dataforge.misc.DFExperimental
import java.nio.file.FileSystem
import java.nio.file.Files
@ -15,28 +16,13 @@ import java.nio.file.StandardOpenOption
import java.nio.file.spi.FileSystemProvider
import java.util.zip.ZipEntry
import java.util.zip.ZipOutputStream
import kotlin.reflect.KType
import kotlin.reflect.typeOf
import kotlin.streams.toList
//public typealias FileFormatResolver<T> = (Path, Meta) -> IOFormat<T>
public interface FileFormatResolver<T : Any> {
public val type: KType
public operator fun invoke(path: Path, meta: Meta): IOFormat<T>
}
public typealias FileFormatResolver<T> = (path: Path, meta: Meta) -> IOReader<T>
@PublishedApi
internal inline fun <reified T : Any> IOPlugin.formatResolver(): FileFormatResolver<T> =
object : FileFormatResolver<T> {
override val type: KType = typeOf<T>()
@OptIn(DFExperimental::class)
override fun invoke(path: Path, meta: Meta): IOFormat<T> =
resolveIOFormat<T>() ?: error("Can't resolve IO format for ${T::class}")
}
private fun newZFS(path: Path): FileSystem {
val fsProvider = FileSystemProvider.installedProviders().find { it.scheme == "jar" }
?: error("Zip file system provider not found")
@ -46,14 +32,9 @@ private fun newZFS(path: Path): FileSystem {
/**
* Read data with supported envelope format and binary format. If envelope format is null, then read binary directly from file.
* The operation is blocking since it must read meta header. The reading of envelope body is lazy
* @param type explicit type of data read
* @param dataFormat binary format
* @param envelopeFormat the format of envelope. If null, file is read directly
* @param metaFile the relative file for optional meta override
* @param metaFileFormat the meta format for override
*/
@DFExperimental
public fun <T : Any> IOPlugin.readDataFile(
public inline fun <reified T : Any> IOPlugin.readDataFile(
path: Path,
formatResolver: FileFormatResolver<T>,
): Data<T> {
@ -62,34 +43,26 @@ public fun <T : Any> IOPlugin.readDataFile(
return envelope.toData(format)
}
@DFExperimental
public inline fun <reified T : Any> IOPlugin.readDataFile(path: Path): Data<T> = readDataFile(path, formatResolver())
/**
* Add file/directory-based data tree item
*/
@DFExperimental
public suspend fun <T : Any> DataSetBuilder<T>.file(
plugin: IOPlugin,
context(IOPlugin) @DFExperimental
public fun DataSetBuilder<Any>.file(
path: Path,
formatResolver: FileFormatResolver<T>,
formatResolver: FileFormatResolver<Any>,
) {
//If path is a single file or a special directory, read it as single datum
if (!Files.isDirectory(path) || Files.list(path).allMatch { it.fileName.toString().startsWith("@") }) {
plugin.run {
val data = readDataFile(path, formatResolver)
val name = data.meta[Envelope.ENVELOPE_NAME_KEY].string
?: path.fileName.toString().replace(".df", "")
data(name, data)
}
val data = readDataFile(path, formatResolver)
val name = data.meta[Envelope.ENVELOPE_NAME_KEY].string
?: path.fileName.toString().replace(".df", "")
data(name, data)
} else {
//otherwise, read as directory
plugin.run {
val data = readDataDirectory(path, formatResolver)
val name = data.meta[Envelope.ENVELOPE_NAME_KEY].string
?: path.fileName.toString().replace(".df", "")
node(name, data)
}
val data = readDataDirectory(path, formatResolver)
val name = data.meta[Envelope.ENVELOPE_NAME_KEY].string
?: path.fileName.toString().replace(".df", "")
node(name, data)
}
}
@ -97,10 +70,10 @@ public suspend fun <T : Any> DataSetBuilder<T>.file(
* Read the directory as a data node. If [path] is a zip archive, read it as directory
*/
@DFExperimental
public suspend fun <T : Any> IOPlugin.readDataDirectory(
public fun IOPlugin.readDataDirectory(
path: Path,
formatResolver: FileFormatResolver<T>,
): DataTree<T> {
formatResolver: FileFormatResolver<Any>,
): DataTree<Any> {
//read zipped data node
if (path.fileName != null && path.fileName.toString().endsWith(".zip")) {
//Using explicit Zip file system to avoid bizarre compatibility bugs
@ -108,24 +81,18 @@ public suspend fun <T : Any> IOPlugin.readDataDirectory(
return readDataDirectory(fs.rootDirectories.first(), formatResolver)
}
if (!Files.isDirectory(path)) error("Provided path $path is not a directory")
return DataTree(formatResolver.type) {
return DataTree<Any> {
Files.list(path).toList().forEach { path ->
val fileName = path.fileName.toString()
if (fileName.startsWith(IOPlugin.META_FILE_NAME)) {
meta(readMetaFile(path))
} else if (!fileName.startsWith("@")) {
runBlocking {
file(this@readDataDirectory, path, formatResolver)
}
file(path, formatResolver)
}
}
}
}
@DFExperimental
public suspend inline fun <reified T : Any> IOPlugin.readDataDirectory(path: Path): DataTree<T> =
readDataDirectory(path, formatResolver())
/**
* Write data tree to existing directory or create a new one using default [java.nio.file.FileSystem] provider
*/
@ -133,7 +100,7 @@ public suspend inline fun <reified T : Any> IOPlugin.readDataDirectory(path: Pat
public suspend fun <T : Any> IOPlugin.writeDataDirectory(
path: Path,
tree: DataTree<T>,
format: IOFormat<T>,
format: IOWriter<T>,
envelopeFormat: EnvelopeFormat? = null,
metaFormat: MetaFormatFactory? = null,
) {
@ -179,11 +146,9 @@ private suspend fun <T : Any> ZipOutputStream.writeNode(
val envelope = treeItem.data.toEnvelope(dataFormat)
val entry = ZipEntry(name)
putNextEntry(entry)
envelopeFormat.run {
asOutput().run {
writeEnvelope(this, envelope)
flush()
}
asOutput().run {
envelopeFormat.writeEnvelope(this, envelope)
flush()
}
}
is DataTreeItem.Node -> {

21
dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/FileDataTest.kt
View File

@ -9,12 +9,8 @@ import space.kscience.dataforge.io.IOFormat
import space.kscience.dataforge.io.io
import space.kscience.dataforge.io.readUtf8String
import space.kscience.dataforge.io.writeUtf8String
import space.kscience.dataforge.meta.Meta
import space.kscience.dataforge.misc.DFExperimental
import java.nio.file.Files
import java.nio.file.Path
import kotlin.reflect.KType
import kotlin.reflect.typeOf
import kotlin.test.Test
import kotlin.test.assertEquals
@ -35,8 +31,6 @@ class FileDataTest {
object StringIOFormat : IOFormat<String> {
override val type: KType = typeOf<String>()
override fun writeObject(output: Output, obj: String) {
output.writeUtf8String(obj)
}
@ -45,17 +39,6 @@ class FileDataTest {
return input.readUtf8String()
}
override fun toMeta(): Meta = Meta {
IOFormat.NAME_KEY put "string"
}
}
object StringFormatResolver : FileFormatResolver<String> {
override val type: KType = typeOf<String>()
override fun invoke(path: Path, meta: Meta): IOFormat<String> = StringIOFormat
}
@Test
@ -66,7 +49,7 @@ class FileDataTest {
runBlocking {
writeDataDirectory(dir, dataNode, StringIOFormat)
println(dir.toUri().toString())
val reconstructed = readDataDirectory(dir, StringFormatResolver)
val reconstructed = readDataDirectory(dir) { _, _ -> StringIOFormat }
assertEquals(dataNode["dir.a"]?.meta, reconstructed["dir.a"]?.meta)
assertEquals(dataNode["b"]?.await(), reconstructed["b"]?.await())
}
@ -82,7 +65,7 @@ class FileDataTest {
runBlocking {
writeZip(zip, dataNode, StringIOFormat)
println(zip.toUri().toString())
val reconstructed = readDataDirectory(zip, StringFormatResolver)
val reconstructed = readDataDirectory(zip) { _, _ -> StringIOFormat }
assertEquals(dataNode["dir.a"]?.meta, reconstructed["dir.a"]?.meta)
assertEquals(dataNode["b"]?.await(), reconstructed["b"]?.await())
}

2
gradle.properties
View File

@ -4,4 +4,4 @@ kotlin.code.style=official
kotlin.mpp.stability.nowarn=true
toolsVersion=0.11.4-kotlin-1.6.20
toolsVersion=0.11.5-kotlin-1.6.21