diff --git a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/TaskData.kt b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/TaskData.kt index 73702b36..080ffec3 100644 --- a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/TaskData.kt +++ b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/TaskData.kt @@ -42,6 +42,9 @@ private class TaskDataImpl( // } } +/** + * Adopt data into this workspace + */ public fun Workspace.wrapData(data: Data, name: Name, taskName: Name, taskMeta: Meta): TaskData = TaskDataImpl(this, data, name, taskName, taskMeta) diff --git a/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/FileWorkspaceCache.kt b/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/FileWorkspaceCache.kt index acda84e4..16af238b 100644 --- a/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/FileWorkspaceCache.kt +++ b/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/FileWorkspaceCache.kt @@ -4,30 +4,36 @@ import space.kscience.dataforge.context.request import space.kscience.dataforge.data.Data import space.kscience.dataforge.data.await import space.kscience.dataforge.io.* +import space.kscience.dataforge.misc.DFExperimental +import space.kscience.dataforge.misc.DFInternal import space.kscience.dataforge.names.Name +import space.kscience.dataforge.names.withIndex import java.nio.file.Path import kotlin.io.path.deleteIfExists +import kotlin.io.path.div import kotlin.io.path.exists import kotlin.reflect.KType -public class FileWorkspaceCache : WorkspaceCache { +public class FileWorkspaceCache(public val cacheDirectory: Path) : WorkspaceCache { private fun TaskData<*>.checkType(taskType: KType): TaskData = this as TaskData + @OptIn(DFExperimental::class, DFInternal::class) override suspend fun evaluate(result: TaskResult): TaskResult { val io = result.workspace.context.request(IOPlugin) - val format: IOFormat = io.resolveIOFormat(result.dataType, result.taskMeta) + val format: IOFormat = io.resolveIOFormat(result.dataType, result.taskMeta) ?: error("Can't resolve IOFormat for ${result.dataType}") - fun cachedDataPath(dataName: Name): Path { - TODO() - } + fun cachedDataPath(dataName: Name): Path = cacheDirectory / + result.taskName.withIndex(result.taskMeta.hashCode().toString(16)).toString() / + dataName.toString() - fun cachedData(data: TaskData): TaskData { + fun evaluateDatum(data: TaskData): TaskData { val path = cachedDataPath(data.name) - val cachedData: Data = Data(data.type, meta = data.meta, dependencies = data.dependencies) { + val datum: Data = Data(data.type, meta = data.meta, dependencies = data.dependencies) { + // return cached data if it is present if (path.exists()) { try { val envelope: Envelope = io.readEnvelopeFile(path) @@ -40,28 +46,26 @@ public class FileWorkspaceCache : WorkspaceCache { } } - return@Data data.await().also { + //waiting for data in current scope because Envelope is synchronous + return@Data data.await().also { result -> val envelope = Envelope { meta = data.meta data { - writeObject(format, it) + writeObject(format, result) } } io.writeEnvelopeFile(path, envelope) } } - return data.workspace.wrapData(cachedData, data.name, data.taskName, data.taskMeta) + return data.workspace.wrapData(datum, data.name, data.taskName, data.taskMeta) } return object : TaskResult by result { - override fun iterator(): Iterator> = iterator { - result.iterator().forEach { - yield(cachedData(it)) - } - } + override fun iterator(): Iterator> = + iterator().asSequence().map { evaluateDatum(it) }.iterator() - override fun get(name: Name): TaskData? = result[name]?.let { cachedData(it) } + override fun get(name: Name): TaskData? = result[name]?.let { evaluateDatum(it) } } } } \ No newline at end of file diff --git a/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/CachingWorkspaceTest.kt b/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/CachingWorkspaceTest.kt index 8021c29f..8bcbf736 100644 --- a/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/CachingWorkspaceTest.kt +++ b/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/CachingWorkspaceTest.kt @@ -28,7 +28,7 @@ internal class CachingWorkspaceTest { } } - val doSecond by task{ + @Suppress("UNUSED_VARIABLE") val doSecond by task{ pipeFrom(doFirst) { _, name, _ -> println("Done second on $name with flag=${taskMeta["flag"].boolean ?: false}") } diff --git a/gradle.properties b/gradle.properties index 89b1a74b..cf8696b2 100644 --- a/gradle.properties +++ b/gradle.properties @@ -6,4 +6,4 @@ kotlin.mpp.stability.nowarn=true kotlin.incremental.js.ir=true kotlin.native.ignoreDisabledTargets=true -toolsVersion=0.14.2-kotlin-1.8.10 +toolsVersion=0.14.3-kotlin-1.8.20-RC