kscience/dataforge-core
7
0
Fork 0
Code Issues Pull Requests 1 Packages Projects Releases 12 Wiki Activity

Refactoring to kotlinx-io complete

Browse Source
This commit is contained in:
Alexander Nozik 2023-07-15 10:49:42 +03:00
parent a699c36f8e
commit a136db16ff
6 changed files with 55 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
dataforge-workspace/build.gradle.kts
View File

@ -10,17 +10,35 @@ kscience{
useSerialization{ useSerialization{
protobuf() protobuf()
} }
dependencies { // dependencies {
api(projects.dataforgeContext) // api(projects.dataforgeContext)
api(projects.dataforgeData) // api(projects.dataforgeData)
api(projects.dataforgeIo) // api(projects.dataforgeIo)
} // }
dependencies(jvmTest){ // dependencies(jvmTest){
implementation(spclibs.logback.classic) // implementation(spclibs.logback.classic)
implementation(projects.dataforgeIo.dataforgeIoYaml) // implementation(projects.dataforgeIo.dataforgeIoYaml)
} // }
} }
readme{ readme{
maturity = space.kscience.gradle.Maturity.EXPERIMENTAL maturity = space.kscience.gradle.Maturity.EXPERIMENTAL
} }
kotlin{
sourceSets{
commonMain{
dependencies {
api(projects.dataforgeContext)
api(projects.dataforgeData)
api(projects.dataforgeIo)
}
}
getByName("jvmTest"){
dependencies {
implementation(spclibs.logback.classic)
implementation(projects.dataforgeIo.dataforgeIoYaml)
}
}
}
}

13
dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/FileWorkspaceCache.kt
View File

@ -1,5 +1,6 @@
package space.kscience.dataforge.workspace package space.kscience.dataforge.workspace
import kotlinx.io.*
import kotlinx.serialization.ExperimentalSerializationApi import kotlinx.serialization.ExperimentalSerializationApi
import kotlinx.serialization.KSerializer import kotlinx.serialization.KSerializer
import kotlinx.serialization.json.Json import kotlinx.serialization.json.Json
@ -26,10 +27,10 @@ public class JsonIOFormat<T : Any>(override val type: KType) : IOFormat<T> {
@Suppress("UNCHECKED_CAST") @Suppress("UNCHECKED_CAST")
private val serializer: KSerializer<T> = serializer(type) as KSerializer<T> private val serializer: KSerializer<T> = serializer(type) as KSerializer<T>
override fun readObject(input: Input): T = Json.decodeFromString(serializer, input.readUtf8String()) override fun readObject(input: Source): T = Json.decodeFromString(serializer, input.readString())
override fun writeObject(output: Output, obj: T) { override fun writeObject(output: Sink, obj: T) {
output.writeUtf8String(Json.encodeToString(serializer, obj)) output.writeString(Json.encodeToString(serializer, obj))
} }
} }
@ -39,10 +40,10 @@ public class ProtobufIOFormat<T : Any>(override val type: KType) : IOFormat<T> {
@Suppress("UNCHECKED_CAST") @Suppress("UNCHECKED_CAST")
private val serializer: KSerializer<T> = serializer(type) as KSerializer<T> private val serializer: KSerializer<T> = serializer(type) as KSerializer<T>
override fun readObject(input: Input): T = ProtoBuf.decodeFromByteArray(serializer, input.readBytes()) override fun readObject(input: Source): T = ProtoBuf.decodeFromByteArray(serializer, input.readByteArray())
override fun writeObject(output: Output, obj: T) { override fun writeObject(output: Sink, obj: T) {
output.writeFully(ProtoBuf.encodeToByteArray(serializer, obj)) output.write(ProtoBuf.encodeToByteArray(serializer, obj))
} }
} }

15
dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/zipData.kt
View File

@ -1,13 +1,10 @@
package space.kscience.dataforge.workspace package space.kscience.dataforge.workspace
import io.ktor.utils.io.streams.asOutput
import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.withContext import kotlinx.coroutines.withContext
import space.kscience.dataforge.data.DataTree import space.kscience.dataforge.data.DataTree
import space.kscience.dataforge.data.DataTreeItem import space.kscience.dataforge.data.DataTreeItem
import space.kscience.dataforge.io.EnvelopeFormat import space.kscience.dataforge.io.*
import space.kscience.dataforge.io.IOFormat
import space.kscience.dataforge.io.TaggedEnvelopeFormat
import space.kscience.dataforge.misc.DFExperimental import space.kscience.dataforge.misc.DFExperimental
import java.nio.file.Files import java.nio.file.Files
import java.nio.file.Path import java.nio.file.Path
@ -28,11 +25,15 @@ private suspend fun <T : Any> ZipOutputStream.writeNode(
val envelope = treeItem.data.toEnvelope(dataFormat) val envelope = treeItem.data.toEnvelope(dataFormat)
val entry = ZipEntry(name) val entry = ZipEntry(name)
putNextEntry(entry) putNextEntry(entry)
asOutput().run {
envelopeFormat.writeObject(this, envelope) //TODO remove additional copy
flush() val bytes = ByteArray {
writeObject(envelopeFormat, envelope)
} }
write(bytes)
} }
is DataTreeItem.Node -> { is DataTreeItem.Node -> {
val entry = ZipEntry("$name/") val entry = ZipEntry("$name/")
putNextEntry(entry) putNextEntry(entry)

17
dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/FileDataTest.kt
View File

@ -1,8 +1,11 @@
package space.kscience.dataforge.workspace package space.kscience.dataforge.workspace
import io.ktor.utils.io.core.Input
import io.ktor.utils.io.core.Output
import kotlinx.coroutines.runBlocking import kotlinx.coroutines.runBlocking
import kotlinx.coroutines.test.runTest
import kotlinx.io.Sink
import kotlinx.io.Source
import kotlinx.io.readString
import kotlinx.io.writeString
import space.kscience.dataforge.context.Context import space.kscience.dataforge.context.Context
import space.kscience.dataforge.context.Global import space.kscience.dataforge.context.Global
import space.kscience.dataforge.data.* import space.kscience.dataforge.data.*
@ -36,11 +39,11 @@ class FileDataTest {
object StringIOFormat : IOFormat<String> { object StringIOFormat : IOFormat<String> {
override val type: KType get() = typeOf<String>() override val type: KType get() = typeOf<String>()
override fun writeObject(output: Output, obj: String) { override fun writeObject(output: Sink, obj: String) {
output.writeUtf8String(obj) output.writeString(obj)
} }
override fun readObject(input: Input): String = input.readUtf8String() override fun readObject(input: Source): String = input.readString()
} }
@Test @Test
@ -59,9 +62,9 @@ class FileDataTest {
@Test @Test
@DFExperimental @DFExperimental
fun testZipWriteRead() = with(Global.io) { fun testZipWriteRead() = runTest {
with(Global.io) {
val zip = Files.createTempFile("df_data_node", ".zip") val zip = Files.createTempFile("df_data_node", ".zip")
runBlocking {
dataNode.writeZip(zip, StringIOFormat) dataNode.writeZip(zip, StringIOFormat)
println(zip.toUri().toString()) println(zip.toUri().toString())
val reconstructed = readDataDirectory(zip) { _, _ -> StringIOFormat } val reconstructed = readDataDirectory(zip) { _, _ -> StringIOFormat }

1
gradle.properties
View File

@ -7,3 +7,4 @@ kotlin.incremental.js.ir=true
kotlin.native.ignoreDisabledTargets=true kotlin.native.ignoreDisabledTargets=true
toolsVersion=0.14.9-kotlin-1.9.0 toolsVersion=0.14.9-kotlin-1.9.0
kotlin.experimental.tryK2=true

2
gradle/wrapper/gradle-wrapper.properties vendored
View File

@ -1,5 +1,5 @@
distributionBase=GRADLE_USER_HOME distributionBase=GRADLE_USER_HOME
distributionPath=wrapper/dists distributionPath=wrapper/dists
distributionUrl=https\://services.gradle.org/distributions/gradle-8.2-bin.zip distributionUrl=https\://services.gradle.org/distributions/gradle-8.2.1-bin.zip
zipStoreBase=GRADLE_USER_HOME zipStoreBase=GRADLE_USER_HOME
zipStorePath=wrapper/dists zipStorePath=wrapper/dists