kscience/dataforge-core
7
0
Fork 0
Code Issues Pull Requests 1 Packages Projects Releases 12 Wiki Activity

Refactoring to kotlinx-io complete

Browse Source
This commit is contained in:
Alexander Nozik 2023-07-15 10:49:42 +03:00
parent a699c36f8e
commit a136db16ff
6 changed files with 55 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
dataforge-workspace/build.gradle.kts
View File

@ -10,17 +10,35 @@ kscience{
useSerialization{
protobuf()
}
dependencies {
api(projects.dataforgeContext)
api(projects.dataforgeData)
api(projects.dataforgeIo)
}
dependencies(jvmTest){
implementation(spclibs.logback.classic)
implementation(projects.dataforgeIo.dataforgeIoYaml)
}
// dependencies {
// api(projects.dataforgeContext)
// api(projects.dataforgeData)
// api(projects.dataforgeIo)
// }
// dependencies(jvmTest){
// implementation(spclibs.logback.classic)
// implementation(projects.dataforgeIo.dataforgeIoYaml)
// }
}
readme{
maturity = space.kscience.gradle.Maturity.EXPERIMENTAL
}
kotlin{
sourceSets{
commonMain{
dependencies {
api(projects.dataforgeContext)
api(projects.dataforgeData)
api(projects.dataforgeIo)
}
}
getByName("jvmTest"){
dependencies {
implementation(spclibs.logback.classic)
implementation(projects.dataforgeIo.dataforgeIoYaml)
}
}
}
}

13
dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/FileWorkspaceCache.kt
View File

@ -1,5 +1,6 @@
package space.kscience.dataforge.workspace
import kotlinx.io.*
import kotlinx.serialization.ExperimentalSerializationApi
import kotlinx.serialization.KSerializer
import kotlinx.serialization.json.Json
@ -26,10 +27,10 @@ public class JsonIOFormat<T : Any>(override val type: KType) : IOFormat<T> {
@Suppress("UNCHECKED_CAST")
private val serializer: KSerializer<T> = serializer(type) as KSerializer<T>
override fun readObject(input: Input): T = Json.decodeFromString(serializer, input.readUtf8String())
override fun readObject(input: Source): T = Json.decodeFromString(serializer, input.readString())
override fun writeObject(output: Output, obj: T) {
output.writeUtf8String(Json.encodeToString(serializer, obj))
override fun writeObject(output: Sink, obj: T) {
output.writeString(Json.encodeToString(serializer, obj))
}
}
@ -39,10 +40,10 @@ public class ProtobufIOFormat<T : Any>(override val type: KType) : IOFormat<T> {
@Suppress("UNCHECKED_CAST")
private val serializer: KSerializer<T> = serializer(type) as KSerializer<T>
override fun readObject(input: Input): T = ProtoBuf.decodeFromByteArray(serializer, input.readBytes())
override fun readObject(input: Source): T = ProtoBuf.decodeFromByteArray(serializer, input.readByteArray())
override fun writeObject(output: Output, obj: T) {
output.writeFully(ProtoBuf.encodeToByteArray(serializer, obj))
override fun writeObject(output: Sink, obj: T) {
output.write(ProtoBuf.encodeToByteArray(serializer, obj))
}
}

15
dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/zipData.kt
View File

@ -1,13 +1,10 @@
package space.kscience.dataforge.workspace
import io.ktor.utils.io.streams.asOutput
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.withContext
import space.kscience.dataforge.data.DataTree
import space.kscience.dataforge.data.DataTreeItem
import space.kscience.dataforge.io.EnvelopeFormat
import space.kscience.dataforge.io.IOFormat
import space.kscience.dataforge.io.TaggedEnvelopeFormat
import space.kscience.dataforge.io.*
import space.kscience.dataforge.misc.DFExperimental
import java.nio.file.Files
import java.nio.file.Path
@ -28,11 +25,15 @@ private suspend fun <T : Any> ZipOutputStream.writeNode(
val envelope = treeItem.data.toEnvelope(dataFormat)
val entry = ZipEntry(name)
putNextEntry(entry)
asOutput().run {
envelopeFormat.writeObject(this, envelope)
flush()
//TODO remove additional copy
val bytes = ByteArray {
writeObject(envelopeFormat, envelope)
}
write(bytes)
}
is DataTreeItem.Node -> {
val entry = ZipEntry("$name/")
putNextEntry(entry)

17
dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/FileDataTest.kt
View File

@ -1,8 +1,11 @@
package space.kscience.dataforge.workspace
import io.ktor.utils.io.core.Input
import io.ktor.utils.io.core.Output
import kotlinx.coroutines.runBlocking
import kotlinx.coroutines.test.runTest
import kotlinx.io.Sink
import kotlinx.io.Source
import kotlinx.io.readString
import kotlinx.io.writeString
import space.kscience.dataforge.context.Context
import space.kscience.dataforge.context.Global
import space.kscience.dataforge.data.*
@ -36,11 +39,11 @@ class FileDataTest {
object StringIOFormat : IOFormat<String> {
override val type: KType get() = typeOf<String>()
override fun writeObject(output: Output, obj: String) {
output.writeUtf8String(obj)
override fun writeObject(output: Sink, obj: String) {
output.writeString(obj)
}
override fun readObject(input: Input): String = input.readUtf8String()
override fun readObject(input: Source): String = input.readString()
}
@Test
@ -59,9 +62,9 @@ class FileDataTest {
@Test
@DFExperimental
fun testZipWriteRead() = with(Global.io) {
fun testZipWriteRead() = runTest {
with(Global.io) {
val zip = Files.createTempFile("df_data_node", ".zip")
runBlocking {
dataNode.writeZip(zip, StringIOFormat)
println(zip.toUri().toString())
val reconstructed = readDataDirectory(zip) { _, _ -> StringIOFormat }

1
gradle.properties
View File

@ -7,3 +7,4 @@ kotlin.incremental.js.ir=true
kotlin.native.ignoreDisabledTargets=true
toolsVersion=0.14.9-kotlin-1.9.0
kotlin.experimental.tryK2=true

2
gradle/wrapper/gradle-wrapper.properties vendored
View File

@ -1,5 +1,5 @@
distributionBase=GRADLE_USER_HOME
distributionPath=wrapper/dists
distributionUrl=https\://services.gradle.org/distributions/gradle-8.2-bin.zip
distributionUrl=https\://services.gradle.org/distributions/gradle-8.2.1-bin.zip
zipStoreBase=GRADLE_USER_HOME
zipStorePath=wrapper/dists