From 707b59e6fc7b20d56871a289c5e71ddf0ae637ef Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Fri, 24 Feb 2023 10:05:01 +0300 Subject: [PATCH] Update version --- build.gradle.kts | 2 +- .../kscience/dataforge/names/NameToken.kt | 2 - .../kscience/dataforge/workspace/fileData.kt | 53 ++++++++++++------- 3 files changed, 35 insertions(+), 22 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index 87a6f55c..f7d70463 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -9,7 +9,7 @@ plugins { allprojects { group = "space.kscience" - version = "0.6.1-dev-3" + version = "0.6.1-dev-4" } subprojects { diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/names/NameToken.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/names/NameToken.kt index 2710e42c..0dc83c57 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/names/NameToken.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/names/NameToken.kt @@ -1,7 +1,6 @@ package space.kscience.dataforge.names import kotlinx.serialization.Serializable -import space.kscience.dataforge.misc.DFExperimental /** * A single name token. Body is not allowed to be empty. @@ -67,7 +66,6 @@ public class NameToken(public val body: String, public val index: String? = null /** * Parse name token from a string */ - @DFExperimental public fun parse(string: String): NameToken { val body = string.substringBefore('[') val index = string.substringAfter('[', "") diff --git a/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/fileData.kt b/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/fileData.kt index c352d635..683dd1ea 100644 --- a/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/fileData.kt +++ b/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/fileData.kt @@ -6,6 +6,7 @@ import kotlinx.coroutines.launch import kotlinx.coroutines.withContext import space.kscience.dataforge.context.error import space.kscience.dataforge.context.logger +import space.kscience.dataforge.context.warn import space.kscience.dataforge.data.* import space.kscience.dataforge.io.* import space.kscience.dataforge.meta.Meta @@ -34,7 +35,7 @@ import kotlin.reflect.typeOf //public typealias FileFormatResolver = (Path, Meta) -> IOFormat -public typealias FileFormatResolver = (path: Path, meta: Meta) -> IOReader +public typealias FileFormatResolver = (path: Path, meta: Meta) -> IOReader? /** * A data based on a filesystem [Path] @@ -44,15 +45,15 @@ public class FileData internal constructor(private val data: Data, public // public val path: String? get() = meta[META_FILE_PATH_KEY].string // public val extension: String? get() = meta[META_FILE_EXTENSION_KEY].string // - public val createdTime: Instant? get() = meta[META_FILE_CREATE_TIME_KEY].string?.let { Instant.parse(it) } - public val updatedTime: Instant? get() = meta[META_FILE_UPDATE_TIME_KEY].string?.let { Instant.parse(it) } + public val createdTime: Instant? get() = meta[FILE_CREATE_TIME_KEY].string?.let { Instant.parse(it) } + public val updatedTime: Instant? get() = meta[FILE_UPDATE_TIME_KEY].string?.let { Instant.parse(it) } public companion object { - public val META_FILE_KEY: Name = "file".asName() - public val META_FILE_PATH_KEY: Name = META_FILE_KEY + "path" - public val META_FILE_EXTENSION_KEY: Name = META_FILE_KEY + "extension" - public val META_FILE_CREATE_TIME_KEY: Name = META_FILE_KEY + "created" - public val META_FILE_UPDATE_TIME_KEY: Name = META_FILE_KEY + "updated" + public val FILE_KEY: Name = "file".asName() + public val FILE_PATH_KEY: Name = FILE_KEY + "path" + public val FILE_EXTENSION_KEY: Name = FILE_KEY + "extension" + public val FILE_CREATE_TIME_KEY: Name = FILE_KEY + "created" + public val FILE_UPDATE_TIME_KEY: Name = FILE_KEY + "updated" } } @@ -66,20 +67,20 @@ public class FileData internal constructor(private val data: Data, public public fun IOPlugin.readDataFile( path: Path, formatResolver: FileFormatResolver, -): FileData { +): FileData? { val envelope = readEnvelopeFile(path, true) - val format = formatResolver(path, envelope.meta) + val format = formatResolver(path, envelope.meta) ?: return null val updatedMeta = envelope.meta.copy { - FileData.META_FILE_PATH_KEY put path.toString() - FileData.META_FILE_EXTENSION_KEY put path.extension + FileData.FILE_PATH_KEY put path.toString() + FileData.FILE_EXTENSION_KEY put path.extension val attributes = path.readAttributes() - FileData.META_FILE_UPDATE_TIME_KEY put attributes.lastModifiedTime().toInstant().toString() - FileData.META_FILE_CREATE_TIME_KEY put attributes.creationTime().toInstant().toString() + FileData.FILE_UPDATE_TIME_KEY put attributes.lastModifiedTime().toInstant().toString() + FileData.FILE_CREATE_TIME_KEY put attributes.creationTime().toInstant().toString() } return FileData( Data(format.type, updatedMeta) { - envelope.data?.readWith(format) ?: error("Can't convert envelope without content to Data") + (envelope.data ?: Binary.EMPTY).readWith(format) }, path ) @@ -119,6 +120,9 @@ public fun IOPlugin.readDataDirectory( } if (!Files.isDirectory(path)) error("Provided path $path is not a directory") return DataTree(type) { + meta { + FileData.FILE_PATH_KEY put path.toString() + } directory(path, formatResolver) } } @@ -139,7 +143,6 @@ public fun IOPlugin.readRawDirectory( ): DataTree = readDataDirectory(path) { _, _ -> IOReader.binary } -@OptIn(DFExperimental::class) private fun Path.toName() = Name(map { NameToken.parse(it.nameWithoutExtension) }) @DFInternal @@ -197,6 +200,14 @@ public inline fun IOPlugin.monitorDataDirectory( noinline formatResolver: FileFormatResolver, ): DataSource = monitorDataDirectory(typeOf(), path, formatResolver) +/** + * Read and monitor raw binary data tree from the directory. All files are read as-is (save for meta files). + */ +@DFExperimental +public fun IOPlugin.monitorRawDirectory( + path: Path, +): DataSource = monitorDataDirectory(path) { _, _ -> IOReader.binary } + /** * Write data tree to existing directory or create a new one using default [java.nio.file.FileSystem] provider */ @@ -238,22 +249,26 @@ public suspend fun IOPlugin.writeDataDirectory( /** * Add file/directory-based data tree item */ -context(IOPlugin) @OptIn(DFInternal::class) +context(IOPlugin) +@OptIn(DFInternal::class) @DFExperimental public fun DataSetBuilder.file( path: Path, formatResolver: FileFormatResolver, ) { try { - //If path is a single file or a special directory, read it as single datum if (!Files.isDirectory(path) || Files.list(path).allMatch { it.fileName.toString().startsWith("@") }) { val data = readDataFile(path, formatResolver) + if (data == null) { + logger.warn { "File format is not resolved for $path. Skipping." } + return + } val name = data.meta[Envelope.ENVELOPE_NAME_KEY].string ?: path.nameWithoutExtension data(name, data) } else { //otherwise, read as directory - val data = readDataDirectory(dataType, path, formatResolver) + val data: DataTree = readDataDirectory(dataType, path, formatResolver) val name = data.meta[Envelope.ENVELOPE_NAME_KEY].string ?: path.nameWithoutExtension node(name, data) }