diff --git a/CHANGELOG.md b/CHANGELOG.md index 5e4a3ce2..83462d70 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,12 +8,13 @@ ### Changed - Simplify inheritance logic in `MutableTypedMeta` -- Full rework of `DataTree` and associated interfaces (`DataSource`, `DataSink`, etc). +- Full rework of `DataTree` and associated interfaces (`DataSource`, `DataSink`, etc.). ### Deprecated - MetaProvider `spec` is replaced by `readable`. `listOfSpec` replaced with `listOfReadable` ### Removed +- Remove implicit io format resolver in `IOPlugin` and `FileWorkspaceCache`. There are no guarantees that only one format is present in the contrxt for each type. ### Fixed - Fixed NameToken parsing. diff --git a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/IOPlugin.kt b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/IOPlugin.kt index f431a731..11b5e5e3 100644 --- a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/IOPlugin.kt +++ b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/IOPlugin.kt @@ -6,28 +6,11 @@ import space.kscience.dataforge.io.IOFormatFactory.Companion.IO_FORMAT_TYPE import space.kscience.dataforge.io.MetaFormatFactory.Companion.META_FORMAT_TYPE import space.kscience.dataforge.meta.Meta import space.kscience.dataforge.meta.string -import space.kscience.dataforge.misc.UnsafeKType import space.kscience.dataforge.names.Name -import kotlin.reflect.KType -import kotlin.reflect.typeOf public class IOPlugin(meta: Meta) : AbstractPlugin(meta) { override val tag: PluginTag get() = Companion.tag - public val ioFormatFactories: Collection<IOFormatFactory<*>> by lazy { - context.gather<IOFormatFactory<*>>(IO_FORMAT_TYPE).values - } - - @Suppress("UNCHECKED_CAST") - @UnsafeKType - public fun <T> resolveIOFormat(type: KType, meta: Meta): IOFormat<T>? = - ioFormatFactories.singleOrNull { it.type == type }?.build(context, meta) as? IOFormat<T> - - @OptIn(UnsafeKType::class) - public inline fun <reified T> resolveIOFormat(meta: Meta = Meta.EMPTY): IOFormat<T>? = - resolveIOFormat(typeOf<T>(), meta) - - public val metaFormatFactories: Collection<MetaFormatFactory> by lazy { context.gather<MetaFormatFactory>(META_FORMAT_TYPE).values } diff --git a/dataforge-io/src/jvmMain/kotlin/space/kscience/dataforge/io/fileIO.kt b/dataforge-io/src/jvmMain/kotlin/space/kscience/dataforge/io/fileIO.kt index 7df23eb5..2d54e061 100644 --- a/dataforge-io/src/jvmMain/kotlin/space/kscience/dataforge/io/fileIO.kt +++ b/dataforge-io/src/jvmMain/kotlin/space/kscience/dataforge/io/fileIO.kt @@ -15,8 +15,6 @@ import java.nio.file.Path import java.nio.file.StandardOpenOption import kotlin.io.path.inputStream import kotlin.math.min -import kotlin.reflect.full.isSupertypeOf -import kotlin.reflect.typeOf import kotlin.streams.asSequence @@ -79,14 +77,6 @@ public fun Path.rewrite(block: Sink.() -> Unit): Unit { public fun EnvelopeFormat.readFile(path: Path): Envelope = readFrom(path.asBinary()) -/** - * Resolve IOFormat based on type - */ -@Suppress("UNCHECKED_CAST") -public inline fun <reified T : Any> IOPlugin.resolveIOFormat(): IOFormat<T>? = - ioFormatFactories.find { it.type.isSupertypeOf(typeOf<T>()) } as IOFormat<T>? - - public val IOPlugin.Companion.META_FILE_NAME: String get() = "@meta" public val IOPlugin.Companion.DATA_FILE_NAME: String get() = "@data" diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/Scheme.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/Scheme.kt index bc05cb5d..12eb4c68 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/Scheme.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/Scheme.kt @@ -227,7 +227,7 @@ public fun <T : Scheme> MutableMetaProvider.scheme( ): ReadWriteProperty<Any?, T> = object : ReadWriteProperty<Any?, T> { override fun getValue(thisRef: Any?, property: KProperty<*>): T { val name = key ?: property.name.asName() - val node = get(name)?: MutableMeta().also { set(name,it) } + val node = get(name) ?: MutableMeta().also { set(name, it) } return spec.write(node) } diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/names/NameIndexComparator.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/names/NameIndexComparator.kt index 742f8ebb..bb95cf65 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/names/NameIndexComparator.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/names/NameIndexComparator.kt @@ -27,4 +27,4 @@ public object NameIndexComparator : Comparator<String?> { public fun Meta.getIndexedList(name: Name): List<Meta> = getIndexed(name).entries.sortedWith( //sort by index compareBy(space.kscience.dataforge.names.NameIndexComparator) { it.key } -).map{it.value} \ No newline at end of file +).map { it.value } \ No newline at end of file diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/names/NameToken.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/names/NameToken.kt index 3994ef27..d6a760f1 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/names/NameToken.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/names/NameToken.kt @@ -82,13 +82,13 @@ public class NameToken(public val body: String, public val index: String? = null else -> indexEnd = index } - else -> if(indexEnd>=0) error("Symbols not allowed after index in NameToken: $string") + else -> if (indexEnd >= 0) error("Symbols not allowed after index in NameToken: $string") } } - if(indexStart>=0 && indexEnd<0) error("Opening bracket without closing bracket not allowed in NameToken: $string") + if (indexStart >= 0 && indexEnd < 0) error("Opening bracket without closing bracket not allowed in NameToken: $string") return NameToken( - if(indexStart>=0) string.substring(0, indexStart) else string, - if(indexStart>=0) string.substring(indexStart + 1, indexEnd) else null + if (indexStart >= 0) string.substring(0, indexStart) else string, + if (indexStart >= 0) string.substring(indexStart + 1, indexEnd) else null ) } } diff --git a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/Task.kt b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/Task.kt index 06134ce6..5e0ff572 100644 --- a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/Task.kt +++ b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/Task.kt @@ -29,10 +29,10 @@ public interface Task<T> : Described { public val fingerprint: String get() = hashCode().toString(radix = 16) /** - * Compute a [TaskResult] using given meta. In general, the result is lazy and represents both computation model - * and a handler for actual result + * Compute a [TaskResult] using given meta. In general, the result is lazy and represents both the computation model + * and a handler for the actual result * - * @param workspace a workspace to run task in + * @param workspace a workspace to run the task in * @param taskName the name of the task in this workspace * @param taskMeta configuration for current stage computation */ diff --git a/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/FileWorkspaceCache.kt b/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/FileWorkspaceCache.kt index 4d2578e5..ce32848a 100644 --- a/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/FileWorkspaceCache.kt +++ b/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/FileWorkspaceCache.kt @@ -15,6 +15,7 @@ import space.kscience.dataforge.data.Data import space.kscience.dataforge.data.await import space.kscience.dataforge.data.named import space.kscience.dataforge.io.* +import space.kscience.dataforge.meta.Meta import space.kscience.dataforge.misc.DFExperimental import space.kscience.dataforge.misc.UnsafeKType import space.kscience.dataforge.names.withIndex @@ -24,11 +25,7 @@ import kotlin.io.path.div import kotlin.io.path.exists import kotlin.reflect.KType -public class JsonIOFormat<T>(private val type: KType) : IOFormat<T> { - - @Suppress("UNCHECKED_CAST") - private val serializer: KSerializer<T> = serializer(type) as KSerializer<T> - +public class JsonIOFormat<T>(public val serializer: KSerializer<T>) : IOFormat<T> { override fun readFrom(source: Source): T = Json.decodeFromString(serializer, source.readString()) override fun writeTo(sink: Sink, obj: T) { @@ -36,12 +33,11 @@ public class JsonIOFormat<T>(private val type: KType) : IOFormat<T> { } } +/** + * An [IOFormat] based on Protobuf representation of the serializeable object. + */ @OptIn(ExperimentalSerializationApi::class) -public class ProtobufIOFormat<T>(private val type: KType) : IOFormat<T> { - - @Suppress("UNCHECKED_CAST") - private val serializer: KSerializer<T> = serializer(type) as KSerializer<T> - +public class ProtobufIOFormat<T>(public val serializer: KSerializer<T>) : IOFormat<T> { override fun readFrom(source: Source): T = ProtoBuf.decodeFromByteArray(serializer, source.readByteArray()) override fun writeTo(sink: Sink, obj: T) { @@ -49,19 +45,39 @@ public class ProtobufIOFormat<T>(private val type: KType) : IOFormat<T> { } } +public interface IOFormatResolveStrategy { + public fun <T> resolve(type: KType, meta: Meta): IOFormat<T> -public class FileWorkspaceCache(public val cacheDirectory: Path) : WorkspaceCache { + public companion object { + public val PROTOBUF: IOFormatResolveStrategy = object : IOFormatResolveStrategy { + @Suppress("UNCHECKED_CAST") + override fun <T> resolve( + type: KType, + meta: Meta + ): IOFormat<T> = ProtobufIOFormat(serializer(type) as KSerializer<T>) + } - // private fun <T : Any> TaskData<*>.checkType(taskType: KType): TaskData<T> = this as TaskData<T> + public val JSON: IOFormatResolveStrategy = object : IOFormatResolveStrategy { + @Suppress("UNCHECKED_CAST") + override fun <T> resolve( + type: KType, + meta: Meta + ): IOFormat<T> = JsonIOFormat(serializer(type) as KSerializer<T>) + } + } +} + +public class FileWorkspaceCache( + public val cacheDirectory: Path, + private val ioFormatResolveStrategy: IOFormatResolveStrategy, +) : WorkspaceCache { @OptIn(DFExperimental::class, UnsafeKType::class) override suspend fun <T> cache(result: TaskResult<T>): TaskResult<T> { val io = result.workspace.context.request(IOPlugin) - val format: IOFormat<T> = io.resolveIOFormat(result.dataType, result.taskMeta) - ?: ProtobufIOFormat(result.dataType) - ?: error("Can't resolve IOFormat for ${result.dataType}") + val format: IOFormat<T> = ioFormatResolveStrategy.resolve<T>(result.dataType, result.taskMeta) val cachingAction: Action<T, T> = CachingAction(result.dataType) { data -> @@ -104,4 +120,7 @@ public class FileWorkspaceCache(public val cacheDirectory: Path) : WorkspaceCach } } -public fun WorkspaceBuilder.fileCache(cacheDir: Path): Unit = cache(FileWorkspaceCache(cacheDir)) \ No newline at end of file +public fun WorkspaceBuilder.fileCache( + cacheDir: Path, + ioFormatResolveStrategy: IOFormatResolveStrategy = IOFormatResolveStrategy.PROTOBUF +): Unit = cache(FileWorkspaceCache(cacheDir, ioFormatResolveStrategy)) \ No newline at end of file