From e52924a5e2e57f3dbe6e1604ed8e2ad96960f599 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Wed, 27 Jan 2021 22:20:20 +0300 Subject: [PATCH] Directory and file accessors --- build.gradle.kts | 16 ++++-- gradle/wrapper/gradle-wrapper.properties | 2 +- numass-data-model/build.gradle.kts | 4 ++ .../kotlin/ru/inr/mass/data/api/NumassSet.kt | 16 +++--- numass-data-proto/build.gradle.kts | 8 ++- .../inr/mass/data/proto/NumassDirectorySet.kt | 51 +++++++++++++++++++ .../inr/mass/data/proto/NumassProtoPlugin.kt | 2 +- .../inr/mass/data/proto/ProtoNumassPoint.kt | 23 +-------- .../ru/inr/mass/data/proto/readFiles.kt | 13 +++++ .../kotlin/ru/inr/mass/data/proto/utils.kt | 10 ++++ 10 files changed, 108 insertions(+), 37 deletions(-) create mode 100644 numass-data-proto/src/main/kotlin/ru/inr/mass/data/proto/NumassDirectorySet.kt create mode 100644 numass-data-proto/src/main/kotlin/ru/inr/mass/data/proto/readFiles.kt create mode 100644 numass-data-proto/src/main/kotlin/ru/inr/mass/data/proto/utils.kt diff --git a/build.gradle.kts b/build.gradle.kts index 16175a8..08895dd 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -2,13 +2,19 @@ plugins { id("ru.mipt.npm.project") } -group = "ru.inr.mass" -version = "0.1.0-SHAPSHOT" +allprojects { + group = "ru.inr.mass" + version = "0.1.0-SHAPSHOT" +} -val dataforgeVersion by extra("0.3.0-dev") - -val spaceRepo by extra("https://maven.pkg.jetbrains.space/mipt-npm/p/numass/maven") +val dataforgeVersion by extra("0.3.0-dev-1") apiValidation{ validationDisabled = true +} + +val vcs by project.extra("https://mipt-npm.jetbrains.space/p/numass/code/numass/") + +ksciencePublish{ + spaceRepo = "https://maven.pkg.jetbrains.space/mipt-npm/p/numass/maven" } \ No newline at end of file diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index da9702f..28ff446 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-6.8-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-6.8.1-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/numass-data-model/build.gradle.kts b/numass-data-model/build.gradle.kts index a230a9d..26e86cd 100644 --- a/numass-data-model/build.gradle.kts +++ b/numass-data-model/build.gradle.kts @@ -3,6 +3,10 @@ plugins { id("ru.mipt.npm.kscience") } +kscience{ + publish() +} + val dataforgeVersion: String by rootProject.extra dependencies { diff --git a/numass-data-model/src/main/kotlin/ru/inr/mass/data/api/NumassSet.kt b/numass-data-model/src/main/kotlin/ru/inr/mass/data/api/NumassSet.kt index 2b27c68..03e7721 100644 --- a/numass-data-model/src/main/kotlin/ru/inr/mass/data/api/NumassSet.kt +++ b/numass-data-model/src/main/kotlin/ru/inr/mass/data/api/NumassSet.kt @@ -5,7 +5,6 @@ */ package ru.inr.mass.data.api -import hep.dataforge.context.Named import hep.dataforge.meta.Meta import hep.dataforge.meta.get import hep.dataforge.meta.long @@ -13,14 +12,13 @@ import hep.dataforge.names.Name import hep.dataforge.names.toName import hep.dataforge.provider.Provider import java.time.Instant -import java.util.* /** - * A single set of numass points previously called file. + * A single set of numass measurements together with metadata. * * @author [Alexander Nozik](mailto:altavir@gmail.com) */ -public interface NumassSet : Named, Iterable, Provider { +public interface NumassSet : Iterable, Provider { public val meta: Meta @@ -32,7 +30,9 @@ public interface NumassSet : Named, Iterable, Provider { * @return */ public val startTime: Instant - get() = meta[NumassPoint.START_TIME_KEY].long?.let { Instant.ofEpochMilli(it) } ?: firstPoint.startTime + get() = meta[NumassPoint.START_TIME_KEY].long?.let { + Instant.ofEpochMilli(it) + } ?: firstPoint.startTime //suspend fun getHvData(): Table? @@ -44,9 +44,9 @@ public interface NumassSet : Named, Iterable, Provider { override val defaultTarget: String get() = NUMASS_POINT_PROVIDER_KEY override fun content(target: String): Map { - return if(target == NUMASS_POINT_PROVIDER_KEY){ - points.associate { "point[${it.voltage}]".toName() to it } - }else { + return if (target == NUMASS_POINT_PROVIDER_KEY) { + points.associateBy { "point[${it.voltage}]".toName() } + } else { super.content(target) } } diff --git a/numass-data-proto/build.gradle.kts b/numass-data-proto/build.gradle.kts index 394423a..365153a 100644 --- a/numass-data-proto/build.gradle.kts +++ b/numass-data-proto/build.gradle.kts @@ -1,4 +1,6 @@ -import com.google.protobuf.gradle.* +import com.google.protobuf.gradle.proto +import com.google.protobuf.gradle.protobuf +import com.google.protobuf.gradle.protoc plugins { java @@ -7,6 +9,10 @@ plugins { id("com.google.protobuf") version "0.8.14" } +kscience{ + publish() +} + val dataforgeVersion: String by rootProject.extra dependencies { diff --git a/numass-data-proto/src/main/kotlin/ru/inr/mass/data/proto/NumassDirectorySet.kt b/numass-data-proto/src/main/kotlin/ru/inr/mass/data/proto/NumassDirectorySet.kt new file mode 100644 index 0000000..38dcc78 --- /dev/null +++ b/numass-data-proto/src/main/kotlin/ru/inr/mass/data/proto/NumassDirectorySet.kt @@ -0,0 +1,51 @@ +package ru.inr.mass.data.proto + +import hep.dataforge.context.Context +import hep.dataforge.context.logger +import hep.dataforge.io.io +import hep.dataforge.io.readEnvelopeFile +import hep.dataforge.meta.Meta +import ru.inr.mass.data.api.NumassPoint +import ru.inr.mass.data.api.NumassSet +import java.nio.file.Files +import java.nio.file.Path +import kotlin.io.path.ExperimentalPathApi +import kotlin.io.path.div +import kotlin.io.path.exists +import kotlin.io.path.isDirectory +import kotlin.streams.toList + +@OptIn(ExperimentalPathApi::class) +public class NumassDirectorySet internal constructor( + public val context: Context, + public val path: Path, +) : NumassSet { + + override val meta: Meta by lazy { + val metaFilePath = path / "meta" + if (metaFilePath.exists()) { + val envelope = context.io.readEnvelopeFile(path) ?: error("Envelope could not be read from $path") + envelope.meta + } else { + Meta.EMPTY + } + } + + override val points: List by lazy> { + Files.list(path).filter { it.fileName.startsWith("p") }.map { path -> + try { + context.readNumassFile(path) + } catch (e: Exception) { + context.logger.error(e) { "Error reading Numass point file $path" } + null + } + }.toList().filterNotNull() + } +} + +@OptIn(ExperimentalPathApi::class) +public fun Context.readNumassDirectory(path: Path): NumassDirectorySet { + if(!path.exists()) error("Path $path does not exist") + if(!path.isDirectory()) error("The path $path is not a directory") + return NumassDirectorySet(this, path) +} \ No newline at end of file diff --git a/numass-data-proto/src/main/kotlin/ru/inr/mass/data/proto/NumassProtoPlugin.kt b/numass-data-proto/src/main/kotlin/ru/inr/mass/data/proto/NumassProtoPlugin.kt index 00d16b6..883a7fd 100644 --- a/numass-data-proto/src/main/kotlin/ru/inr/mass/data/proto/NumassProtoPlugin.kt +++ b/numass-data-proto/src/main/kotlin/ru/inr/mass/data/proto/NumassProtoPlugin.kt @@ -9,7 +9,7 @@ import hep.dataforge.meta.Meta import kotlin.reflect.KClass public class NumassProtoPlugin : AbstractPlugin() { - val io by require(IOPlugin) + public val io: IOPlugin by require(IOPlugin) override val tag: PluginTag get() = Companion.tag public companion object : PluginFactory { diff --git a/numass-data-proto/src/main/kotlin/ru/inr/mass/data/proto/ProtoNumassPoint.kt b/numass-data-proto/src/main/kotlin/ru/inr/mass/data/proto/ProtoNumassPoint.kt index 761666c..1102f22 100644 --- a/numass-data-proto/src/main/kotlin/ru/inr/mass/data/proto/ProtoNumassPoint.kt +++ b/numass-data-proto/src/main/kotlin/ru/inr/mass/data/proto/ProtoNumassPoint.kt @@ -16,10 +16,7 @@ package ru.inr.mass.data.proto -import hep.dataforge.context.Context import hep.dataforge.io.Envelope -import hep.dataforge.io.io -import hep.dataforge.io.readEnvelopeFile import hep.dataforge.meta.* import kotlinx.coroutines.flow.* import kotlinx.coroutines.runBlocking @@ -30,7 +27,6 @@ import ru.inr.mass.data.api.* import java.io.ByteArrayInputStream import java.io.ByteArrayOutputStream import java.io.InputStream -import java.nio.file.Path import java.time.Duration import java.time.Instant import java.util.zip.Inflater @@ -76,7 +72,6 @@ public class ProtoNumassPoint( public companion object { - /** * Get valid data stream utilizing compression if it is present */ @@ -114,24 +109,10 @@ public class ProtoNumassPoint( } return proto?.let { ProtoNumassPoint(envelope.meta) { it } } } - - public fun fromFile(context: Context, path: Path): ProtoNumassPoint? { - val envelope = context.io.readEnvelopeFile(path) ?: error("Envelope could not be read from $path") - return fromEnvelope(envelope) - } - - public fun fromFile(context: Context, path: String): ProtoNumassPoint? { - return fromFile(context,Path.of(path)) - } - - public fun ofEpochNanos(nanos: Long): Instant { - val seconds = Math.floorDiv(nanos, 1e9.toInt().toLong()) - val reminder = (nanos % 1e9).toInt() - return Instant.ofEpochSecond(seconds, reminder.toLong()) - } } } + public class ProtoBlock( override val channel: Int, private val block: NumassProto.Point.Channel.Block, @@ -139,7 +120,7 @@ public class ProtoBlock( ) : NumassBlock { override val startTime: Instant - get() = ProtoNumassPoint.ofEpochNanos(block.time) + get() = epochNanoTime(block.time) override val length: Duration = when { block.length > 0 -> Duration.ofNanos(block.length) diff --git a/numass-data-proto/src/main/kotlin/ru/inr/mass/data/proto/readFiles.kt b/numass-data-proto/src/main/kotlin/ru/inr/mass/data/proto/readFiles.kt new file mode 100644 index 0000000..446c765 --- /dev/null +++ b/numass-data-proto/src/main/kotlin/ru/inr/mass/data/proto/readFiles.kt @@ -0,0 +1,13 @@ +package ru.inr.mass.data.proto + +import hep.dataforge.context.Context +import hep.dataforge.io.io +import hep.dataforge.io.readEnvelopeFile +import java.nio.file.Path + +public fun Context.readNumassFile(path: Path): ProtoNumassPoint? { + val envelope = io.readEnvelopeFile(path) ?: error("Envelope could not be read from $path") + return ProtoNumassPoint.fromEnvelope(envelope) +} + +public fun Context.readNumassFile(path: String): ProtoNumassPoint? = readNumassFile(Path.of(path)) \ No newline at end of file diff --git a/numass-data-proto/src/main/kotlin/ru/inr/mass/data/proto/utils.kt b/numass-data-proto/src/main/kotlin/ru/inr/mass/data/proto/utils.kt new file mode 100644 index 0000000..49c7609 --- /dev/null +++ b/numass-data-proto/src/main/kotlin/ru/inr/mass/data/proto/utils.kt @@ -0,0 +1,10 @@ +package ru.inr.mass.data.proto + +import java.time.Instant + + +internal fun epochNanoTime(nanos: Long): Instant { + val seconds = Math.floorDiv(nanos, 1e9.toInt().toLong()) + val reminder = (nanos % 1e9).toInt() + return Instant.ofEpochSecond(seconds, reminder.toLong()) +} \ No newline at end of file