From 6bdb3583a19e6db9137dff14e7ca9ac616fa5af2 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Tue, 23 Nov 2021 14:08:31 +0300 Subject: [PATCH] Quick-patch analyzers. They need more work --- build.gradle.kts | 4 +- numass-analysis/build.gradle.kts | 21 ++ .../mass/data/analysis/AbstractAnalyzer.kt | 112 +++++++ .../inr/mass/data/analysis/DebunchAnalyzer.kt | 35 ++ .../inr/mass/data/analysis/NumassAnalyzer.kt | 268 ++++++++++++++++ .../inr/mass/data/analysis/NumassGenerator.kt | 141 ++++++++ .../inr/mass/data/analysis/SimpleAnalyzer.kt | 72 +++++ .../inr/mass/data/analysis/SmartAnalyzer.kt | 104 ++++++ .../ru/inr/mass/data/analysis/TimeAnalyzer.kt | 300 ++++++++++++++++++ .../ru/inr/mass/data/analysis/Values.kt | 5 + numass-data-model/build.gradle.kts | 7 +- .../kotlin/ru/inr/mass/data/api/MetaBlock.kt | 10 +- .../ru/inr/mass/data/api/NumassBlock.kt | 13 +- .../ru/inr/mass/data/api/NumassPoint.kt | 8 +- .../kotlin/ru/inr/mass/data/api/NumassSet.kt | 15 +- .../ru/inr/mass/data/api/SimpleNumassPoint.kt | 26 -- numass-data-proto/build.gradle.kts | 5 +- .../inr/mass/data/proto/ProtoNumassPoint.kt | 35 +- .../data/proto/TaggedNumassEnvelopeFormat.kt | 3 +- .../mass/data/proto/TestNumassDirectory.kt | 3 +- numass-workspace/build.gradle.kts | 1 + .../kotlin/ru/inr/mass/scripts/run_2020_12.kt | 4 +- .../ru/inr/mass/workspace/NumassPlugin.kt | 70 ++++ .../kotlin/ru/inr/mass/workspace/workspace.kt | 1 - settings.gradle.kts | 16 +- 25 files changed, 1201 insertions(+), 78 deletions(-) create mode 100644 numass-analysis/build.gradle.kts create mode 100644 numass-analysis/src/commonMain/kotlin/ru/inr/mass/data/analysis/AbstractAnalyzer.kt create mode 100644 numass-analysis/src/commonMain/kotlin/ru/inr/mass/data/analysis/DebunchAnalyzer.kt create mode 100644 numass-analysis/src/commonMain/kotlin/ru/inr/mass/data/analysis/NumassAnalyzer.kt create mode 100644 numass-analysis/src/commonMain/kotlin/ru/inr/mass/data/analysis/NumassGenerator.kt create mode 100644 numass-analysis/src/commonMain/kotlin/ru/inr/mass/data/analysis/SimpleAnalyzer.kt create mode 100644 numass-analysis/src/commonMain/kotlin/ru/inr/mass/data/analysis/SmartAnalyzer.kt create mode 100644 numass-analysis/src/commonMain/kotlin/ru/inr/mass/data/analysis/TimeAnalyzer.kt create mode 100644 numass-analysis/src/commonMain/kotlin/ru/inr/mass/data/analysis/Values.kt delete mode 100644 numass-data-model/src/commonMain/kotlin/ru/inr/mass/data/api/SimpleNumassPoint.kt create mode 100644 numass-workspace/src/main/kotlin/ru/inr/mass/workspace/NumassPlugin.kt diff --git a/build.gradle.kts b/build.gradle.kts index 71e7398..26c568a 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -12,10 +12,10 @@ allprojects { version = "0.1.0-dev-1" } -val dataforgeVersion by extra("0.4.0") +val dataforgeVersion by extra("0.5.2-dev-2") val kmathVersion by extra("0.3.0-dev-15") ksciencePublish{ - vcs("https://mipt-npm.jetbrains.space/p/numass/code/numass/") + git("https://mipt-npm.jetbrains.space/p/numass/code/numass/") space("https://maven.pkg.jetbrains.space/mipt-npm/p/numass/maven") } \ No newline at end of file diff --git a/numass-analysis/build.gradle.kts b/numass-analysis/build.gradle.kts new file mode 100644 index 0000000..fef9fbd --- /dev/null +++ b/numass-analysis/build.gradle.kts @@ -0,0 +1,21 @@ +plugins { + id("ru.mipt.npm.gradle.mpp") + `maven-publish` +} + + +val dataforgeVersion: String by rootProject.extra + +kotlin.sourceSets { + commonMain { + dependencies { + api(project(":numass-data-model")) + api("space.kscience:tables-kt:0.1.1-dev-2") + api("space.kscience:kmath-complex:0.3.0-dev-17") + api("space.kscience:kmath-stat:0.3.0-dev-17") + api("space.kscience:kmath-histograms:0.3.0-dev-17") + } + } +} + + diff --git a/numass-analysis/src/commonMain/kotlin/ru/inr/mass/data/analysis/AbstractAnalyzer.kt b/numass-analysis/src/commonMain/kotlin/ru/inr/mass/data/analysis/AbstractAnalyzer.kt new file mode 100644 index 0000000..d910297 --- /dev/null +++ b/numass-analysis/src/commonMain/kotlin/ru/inr/mass/data/analysis/AbstractAnalyzer.kt @@ -0,0 +1,112 @@ +/* + * Copyright 2017 Alexander Nozik. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ru.inr.mass.data.analysis + +import kotlinx.coroutines.flow.* +import ru.inr.mass.data.api.NumassBlock +import ru.inr.mass.data.api.NumassEvent +import ru.inr.mass.data.api.NumassSet +import ru.inr.mass.data.api.SignalProcessor +import space.kscience.dataforge.meta.Meta +import space.kscience.dataforge.meta.get +import space.kscience.dataforge.meta.int +import space.kscience.dataforge.tables.RowTable +import space.kscience.dataforge.tables.Table +import space.kscience.dataforge.values.Value + +/** + * Created by darksnake on 11.07.2017. + */ +public abstract class AbstractAnalyzer( + private val processor: SignalProcessor? = null, +) : NumassAnalyzer { + + /** + * Return unsorted stream of events including events from frames. + * In theory, events after processing could be unsorted due to mixture of frames and events. + * In practice usually block have either frame or events, but not both. + * + * @param block + * @return + */ + override fun getEvents(block: NumassBlock, meta: Meta): Flow { + val range = meta.getRange() + return getAllEvents(block).filter { event -> + event.amplitude.toInt() in range + } + } + + protected fun Meta.getRange(): IntRange { + val loChannel = get("window.lo")?.int ?: 0 + val upChannel = get("window.up")?.int ?: Int.MAX_VALUE + return loChannel until upChannel + } + + protected fun getAllEvents(block: NumassBlock): Flow { + return when { + block.framesCount == 0L -> block.events + processor == null -> throw IllegalArgumentException("Signal processor needed to analyze frames") + else -> flow { + emitAll(block.events) + emitAll(block.frames.flatMapConcat { processor.analyze(it) }) + } + } + } + +// /** +// * Get table format for summary table +// * +// * @param config +// * @return +// */ +// protected open fun getTableFormat(config: Meta): ValueTableHeader { +// return TableFormatBuilder() +// .addNumber(HV_KEY, X_VALUE_KEY) +// .addNumber(NumassAnalyzer.LENGTH_KEY) +// .addNumber(NumassAnalyzer.COUNT_KEY) +// .addNumber(NumassAnalyzer.COUNT_RATE_KEY, Y_VALUE_KEY) +// .addNumber(NumassAnalyzer.COUNT_RATE_ERROR_KEY, Y_ERROR_KEY) +// .addColumn(NumassAnalyzer.WINDOW_KEY) +// .addTime() +// .build() +// } + + override suspend fun analyzeSet(set: NumassSet, config: Meta): Table = RowTable( + NumassAnalyzer.length, + NumassAnalyzer.count, + NumassAnalyzer.cr, + NumassAnalyzer.crError, +// NumassAnalyzer.window, +// NumassAnalyzer.timestamp + ) { + + set.points.forEach { point -> + analyzeParent(point, config) + } + } + + public companion object { +// public val NAME_LIST: List = listOf( +// NumassAnalyzer.LENGTH_KEY, +// NumassAnalyzer.COUNT_KEY, +// NumassAnalyzer.COUNT_RATE_KEY, +// NumassAnalyzer.COUNT_RATE_ERROR_KEY, +// NumassAnalyzer.WINDOW_KEY, +// NumassAnalyzer.TIME_KEY +// ) + } +} diff --git a/numass-analysis/src/commonMain/kotlin/ru/inr/mass/data/analysis/DebunchAnalyzer.kt b/numass-analysis/src/commonMain/kotlin/ru/inr/mass/data/analysis/DebunchAnalyzer.kt new file mode 100644 index 0000000..69e82a7 --- /dev/null +++ b/numass-analysis/src/commonMain/kotlin/ru/inr/mass/data/analysis/DebunchAnalyzer.kt @@ -0,0 +1,35 @@ +/* + * Copyright 2017 Alexander Nozik. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ru.inr.mass.data.analysis + +import ru.inr.mass.data.api.NumassBlock +import ru.inr.mass.data.api.SignalProcessor +import space.kscience.dataforge.meta.Meta +import space.kscience.dataforge.meta.descriptors.MetaDescriptor + +/** + * Block analyzer that can perform debunching + * Created by darksnake on 11.07.2017. + */ +public class DebunchAnalyzer(processor: SignalProcessor? = null) : AbstractAnalyzer(processor) { + + override suspend fun analyze(block: NumassBlock, config: Meta): NumassAnalyzerResult { + TODO() + } + + override val descriptor: MetaDescriptor? = null +} diff --git a/numass-analysis/src/commonMain/kotlin/ru/inr/mass/data/analysis/NumassAnalyzer.kt b/numass-analysis/src/commonMain/kotlin/ru/inr/mass/data/analysis/NumassAnalyzer.kt new file mode 100644 index 0000000..26c530d --- /dev/null +++ b/numass-analysis/src/commonMain/kotlin/ru/inr/mass/data/analysis/NumassAnalyzer.kt @@ -0,0 +1,268 @@ +/* + * Copyright 2017 Alexander Nozik. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ru.inr.mass.data.analysis + +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.collect +import ru.inr.mass.data.analysis.NumassAnalyzer.Companion.MAX_CHANNEL +import ru.inr.mass.data.analysis.NumassAnalyzer.Companion.channel +import ru.inr.mass.data.analysis.NumassAnalyzer.Companion.count +import ru.inr.mass.data.analysis.NumassAnalyzer.Companion.cr +import ru.inr.mass.data.analysis.NumassAnalyzer.Companion.crError +import ru.inr.mass.data.api.* +import ru.inr.mass.data.api.NumassPoint.Companion.HV_KEY +import ru.inr.mass.data.api.NumassPoint.Companion.LENGTH_KEY +import space.kscience.dataforge.meta.* +import space.kscience.dataforge.meta.descriptors.Described +import space.kscience.dataforge.names.asName +import space.kscience.dataforge.tables.* +import space.kscience.dataforge.values.* +import space.kscience.kmath.histogram.Counter +import space.kscience.kmath.histogram.LongCounter +import kotlin.math.max +import kotlin.math.min +import kotlin.math.pow +import kotlin.math.sqrt + +public class NumassAnalyzerResult : Scheme() { + public var count: Long? by long(NumassAnalyzer.count.name.asName()) + public var countRate: Double? by double(NumassAnalyzer.cr.name.asName()) + public var countRateError: Double? by double(NumassAnalyzer.crError.name.asName()) + public var length: Long? by long(NumassAnalyzer.length.name.asName()) + + public var voltage: Double? by double(HV_KEY.asName()) + + public var window: UIntRange? + get() = meta["window"]?.value?.list?.let { + it[0].int.toUInt().rangeTo(it[1].int.toUInt()) + } + set(value) { + meta["window"] = value?.let { ListValue(it.first.toInt(), it.first.toInt()) } + } + + public companion object : SchemeSpec(::NumassAnalyzerResult) +} + + +/** + * A general raw data analysis utility. Could have different implementations + * Created by darksnake on 06-Jul-17. + */ +public interface NumassAnalyzer : Described { + + /** + * Perform analysis on block. The values for count rate, its error and point length in nanos must + * exist, but occasionally additional values could also be presented. + * + * @param block + * @return + */ + public suspend fun analyze(block: NumassBlock, config: Meta = Meta.EMPTY): NumassAnalyzerResult + + /** + * Analysis result for point including hv information + * @param point + * @param config + * @return + */ + public suspend fun analyzeParent(point: ParentBlock, config: Meta = Meta.EMPTY): NumassAnalyzerResult { +// //Add properties to config +// val newConfig = config.builder.apply { +// if (point is NumassPoint) { +// setValue("voltage", point.voltage) +// setValue("index", point.index) +// } +// setValue("channel", point.channel) +// } + val res = analyze(point, config) + if (point is NumassPoint) { + res.voltage = point.voltage + } + + return res + } + + /** + * Return unsorted stream of events including events from frames + * + * @param block + * @return + */ + public fun getEvents(block: NumassBlock, meta: Meta = Meta.EMPTY): Flow + + /** + * Analyze the whole set. And return results as a table + * + * @param set + * @param config + * @return + */ + public suspend fun analyzeSet(set: NumassSet, config: Meta): Table + + /** + * Get the approximate number of events in block. Not all analyzers support precise event counting + * + * @param block + * @param config + * @return + */ + public suspend fun getCount(block: NumassBlock, config: Meta): Long = + analyze(block, config).getValue(count.name)?.long ?: 0L + + /** + * Get approximate effective point length in nanos. It is not necessary corresponds to real point length. + * + * @param block + * @param config + * @return + */ + public suspend fun getLength(block: NumassBlock, config: Meta = Meta.EMPTY): Long = + analyze(block, config).getValue(LENGTH_KEY)?.long ?: 0L + + public companion object { + + public val channel: ColumnHeader by ColumnHeader.value(ValueType.NUMBER) + public val count: ColumnHeader by ColumnHeader.value(ValueType.NUMBER) + public val length: ColumnHeader by ColumnHeader.value(ValueType.NUMBER) + public val cr: ColumnHeader by ColumnHeader.value(ValueType.NUMBER) + public val crError: ColumnHeader by ColumnHeader.value(ValueType.NUMBER) + public val window: ColumnHeader by ColumnHeader.value(ValueType.LIST) + public val timestamp: ColumnHeader by ColumnHeader.value(ValueType.NUMBER) +// +// val AMPLITUDE_ADAPTER: ValuesAdapter = Adapters.buildXYAdapter(CHANNEL_KEY, COUNT_RATE_KEY) + + public val MAX_CHANNEL: UInt = 10000U + } +} + +public suspend fun NumassAnalyzer.getAmplitudeSpectrum( + block: NumassBlock, + range: UIntRange = 0U..MAX_CHANNEL, + config: Meta = Meta.EMPTY, +): Table { + val seconds = block.getLength().inWholeMilliseconds.toDouble() / 1000.0 + return getEvents(block, config).getAmplitudeSpectrum(seconds, range) +} + +/** + * Calculate number of counts in the given channel + * + * @param spectrum + * @param loChannel + * @param upChannel + * @return + */ +internal fun Table.countInWindow(loChannel: Short, upChannel: Short): Long = rows.filter { row -> + row[channel]?.int in loChannel until upChannel +}.sumOf { it[count]?.long ?: 0L } + +/** + * Calculate the amplitude spectrum for a given block. The s + * + * @param this@getAmplitudeSpectrum + * @param length length in seconds, used for count rate calculation + * @param config + * @return + */ +private suspend fun Flow.getAmplitudeSpectrum( + length: Double, + range: UIntRange = 0U..MAX_CHANNEL, +): Table { + + //optimized for fastest computation + val spectrum: MutableMap = HashMap() + collect { event -> + val channel = event.amplitude + spectrum.getOrPut(channel.toUInt()) { + LongCounter() + }.add(1L) + } + + return RowTable(channel, count, cr, crError) { + range.forEach { ch -> + val countValue: Long = spectrum[ch]?.value ?: 0L + valueRow( + channel to ch, + count to countValue, + cr to (countValue.toDouble() / length), + crError to sqrt(countValue.toDouble()) / length + ) + } + } +} + +/** + * Apply window and binning to a spectrum. Empty bins are filled with zeroes + */ +private fun Table.withBinning( + binSize: UInt, range: UIntRange = 0U..MAX_CHANNEL, +): Table = RowTable(channel, count, cr, crError) { +// var chan = loChannel +// ?: this.getColumn(NumassAnalyzer.CHANNEL_KEY).stream().mapToInt { it.int }.min().orElse(0) +// +// val top = upChannel +// ?: this.getColumn(NumassAnalyzer.CHANNEL_KEY).stream().mapToInt { it.int }.max().orElse(1) + + val binSizeColumn = newColumn("binSize") + + var chan = range.first + + while (chan < range.last - binSize) { + val counter = LongCounter() + val countRateCounter = Counter.real() + val countRateDispersionCounter = Counter.real() + + val binLo = chan + val binUp = chan + binSize + + rows.filter { row -> + (row[channel]?.int ?: 0U) in binLo until binUp + }.forEach { row -> + counter.add(row[count]?.long ?: 0L) + countRateCounter.add(row[cr]?.double ?: 0.0) + countRateDispersionCounter.add(row[crError]?.double?.pow(2.0) ?: 0.0) + } + val bin = min(binSize, range.last - chan) + + valueRow( + channel to (chan.toDouble() + bin.toDouble() / 2.0), + count to counter.value, + cr to countRateCounter.value, + crError to sqrt(countRateDispersionCounter.value), + binSizeColumn to bin + ) + chan += binSize + } +} + +/** + * Subtract reference spectrum. + */ +private fun subtractAmplitudeSpectrum( + sp1: Table, sp2: Table, +): Table = RowTable(channel, cr, crError) { + sp1.rows.forEach { row1 -> + val channelValue = row1[channel]?.double + val row2 = sp2.rows.find { it[channel]?.double == channelValue } ?: MapRow(emptyMap()) + + val value = max((row1[cr]?.double ?: 0.0) - (row2[cr]?.double ?: 0.0), 0.0) + val error1 = row1[crError]?.double ?: 0.0 + val error2 = row2[crError]?.double ?: 0.0 + val error = sqrt(error1 * error1 + error2 * error2) + valueRow(channel to channelValue, cr to value, crError to error) + } +} \ No newline at end of file diff --git a/numass-analysis/src/commonMain/kotlin/ru/inr/mass/data/analysis/NumassGenerator.kt b/numass-analysis/src/commonMain/kotlin/ru/inr/mass/data/analysis/NumassGenerator.kt new file mode 100644 index 0000000..f3936aa --- /dev/null +++ b/numass-analysis/src/commonMain/kotlin/ru/inr/mass/data/analysis/NumassGenerator.kt @@ -0,0 +1,141 @@ +package ru.inr.mass.data.analysis +// +//import hep.dataforge.stat.defaultGenerator +//import kotlinx.coroutines.flow.Flow +//import kotlinx.coroutines.flow.takeWhile +//import kotlinx.coroutines.flow.toList +//import kotlinx.datetime.Instant +//import org.apache.commons.math3.distribution.EnumeratedRealDistribution +//import ru.inr.mass.data.analysis.NumassAnalyzer.Companion.CHANNEL_KEY +//import ru.inr.mass.data.analysis.NumassAnalyzer.Companion.COUNT_RATE_KEY +//import ru.inr.mass.data.api.NumassBlock +//import ru.inr.mass.data.api.OrphanNumassEvent +//import ru.inr.mass.data.api.SimpleBlock +//import space.kscience.dataforge.tables.Table +//import space.kscience.kmath.chains.Chain +//import space.kscience.kmath.chains.MarkovChain +//import space.kscience.kmath.chains.StatefulChain +//import space.kscience.kmath.stat.RandomGenerator +//import kotlin.time.Duration.Companion.nanoseconds +// +//private fun RandomGenerator.nextExp(mean: Double): Double { +// return -mean * ln(1.0 - nextDouble()) +//} +// +//private fun RandomGenerator.nextDeltaTime(cr: Double): Long { +// return (nextExp(1.0 / cr) * 1e9).toLong() +//} +// +//public suspend fun Flow.generateBlock(start: Instant, length: Long): NumassBlock { +// return SimpleBlock.produce(start, length.nanoseconds) { +// takeWhile { it.timeOffset < length }.toList() +// } +//} +// +//private class MergingState(private val chains: List>) { +// suspend fun poll(): OrphanNumassEvent { +// val next = chains.minBy { it.value.timeOffset } ?: chains.first() +// val res = next.value +// next.next() +// return res +// } +// +//} +// +///** +// * Merge event chains in ascending time order +// */ +//public fun List>.merge(): Chain { +// return StatefulChain(MergingState(this), OrphanNumassEvent(0.toUShort(), 0L)) { +// poll() +// } +//} +// +///** +// * Apply dead time based on event that caused it +// */ +//public fun Chain.withDeadTime(deadTime: (OrphanNumassEvent) -> Long): Chain { +// return MarkovChain(this.value) { +// val start = this.value +// val dt = deadTime(start) +// do { +// val next = next() +// } while (next.timeOffset - start.timeOffset < dt) +// this.value +// } +//} +// +//public object NumassGenerator { +// +// public val defaultAmplitudeGenerator: RandomGenerator.(OrphanNumassEvent?, Long) -> Short = +// { _, _ -> ((nextDouble() + 2.0) * 100).toShort() } +// +// /** +// * Generate an event chain with fixed count rate +// * @param cr = count rate in Hz +// * @param rnd = random number generator +// * @param amp amplitude generator for the chain. The receiver is rng, first argument is the previous event and second argument +// * is the delay between the next event. The result is the amplitude in channels +// */ +// public fun generateEvents( +// cr: Double, +// rnd: RandomGenerator = defaultGenerator, +// amp: RandomGenerator.(OrphanNumassEvent?, Long) -> Short = defaultAmplitudeGenerator, +// ): Chain = MarkovChain(OrphanNumassEvent(rnd.amp(null, 0), 0)) { event -> +// val deltaT = rnd.nextDeltaTime(cr) +// OrphanNumassEvent(rnd.amp(event, deltaT), event.timeOffset + deltaT) +// } +// +// public fun mergeEventChains(vararg chains: Chain): Chain = +// listOf(*chains).merge() +// +// +// private data class BunchState(var bunchStart: Long = 0, var bunchEnd: Long = 0) +// +// /** +// * The chain of bunched events +// * @param cr count rate of events inside bunch +// * @param bunchRate number of bunches per second +// * @param bunchLength the length of bunch +// */ +// public fun generateBunches( +// cr: Double, +// bunchRate: Double, +// bunchLength: Double, +// rnd: RandomGenerator = defaultGenerator, +// amp: RandomGenerator.(OrphanNumassEvent?, Long) -> Short = defaultAmplitudeGenerator, +// ): Chain { +// return StatefulChain( +// BunchState(0, 0), +// OrphanNumassEvent(rnd.amp(null, 0), 0)) { event -> +// if (event.timeOffset >= bunchEnd) { +// bunchStart = bunchEnd + rnd.nextDeltaTime(bunchRate) +// bunchEnd = bunchStart + (bunchLength * 1e9).toLong() +// OrphanNumassEvent(rnd.amp(null, 0), bunchStart) +// } else { +// val deltaT = rnd.nextDeltaTime(cr) +// OrphanNumassEvent(rnd.amp(event, deltaT), event.timeOffset + deltaT) +// } +// } +// } +// +// /** +// * Generate a chain using provided spectrum for amplitudes +// */ +// public fun generateEvents( +// cr: Double, +// rnd: RandomGenerator = defaultGenerator, +// spectrum: Table, +// ): Chain { +// +// val channels = DoubleArray(spectrum.size()) +// val values = DoubleArray(spectrum.size()) +// for (i in 0 until spectrum.size()) { +// channels[i] = spectrum.get(CHANNEL_KEY, i).double +// values[i] = spectrum.get(COUNT_RATE_KEY, i).double +// } +// val distribution = EnumeratedRealDistribution(channels, values) +// +// return generateEvents(cr, rnd) { _, _ -> distribution.sample().toShort() } +// } +//} \ No newline at end of file diff --git a/numass-analysis/src/commonMain/kotlin/ru/inr/mass/data/analysis/SimpleAnalyzer.kt b/numass-analysis/src/commonMain/kotlin/ru/inr/mass/data/analysis/SimpleAnalyzer.kt new file mode 100644 index 0000000..2b2af92 --- /dev/null +++ b/numass-analysis/src/commonMain/kotlin/ru/inr/mass/data/analysis/SimpleAnalyzer.kt @@ -0,0 +1,72 @@ +/* + * Copyright 2017 Alexander Nozik. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package inr.numass.data.analyzers + +import kotlinx.coroutines.flow.count +import ru.inr.mass.data.analysis.AbstractAnalyzer +import ru.inr.mass.data.analysis.NumassAnalyzerResult +import ru.inr.mass.data.api.NumassBlock +import ru.inr.mass.data.api.SignalProcessor +import space.kscience.dataforge.meta.Meta +import space.kscience.dataforge.meta.descriptors.MetaDescriptor +import space.kscience.dataforge.meta.descriptors.value +import space.kscience.dataforge.meta.double +import space.kscience.dataforge.meta.get +import space.kscience.dataforge.meta.int +import space.kscience.dataforge.values.ValueType +import kotlin.math.sqrt + +/** + * A simple event counter + * Created by darksnake on 07.07.2017. + */ +public class SimpleAnalyzer(processor: SignalProcessor? = null) : AbstractAnalyzer(processor) { + + override val descriptor: MetaDescriptor = MetaDescriptor { + value("deadTime", ValueType.NUMBER) { + info = "Dead time in nanoseconds for correction" + default(0.0) + } + } + + override suspend fun analyze(block: NumassBlock, config: Meta): NumassAnalyzerResult { + val loChannel = config["window.lo"]?.int ?: 0 + val upChannel = config["window.up"]?.int ?: Int.MAX_VALUE + + val count: Int = getEvents(block, config).count() + val length: Double = block.getLength().inWholeNanoseconds.toDouble() / 1e9 + + val deadTime = config["deadTime"]?.double ?: 0.0 + + val countRate = if (deadTime > 0) { + val mu = count.toDouble() / length + mu / (1.0 - deadTime * 1e-9 * mu) + } else { + count.toDouble() / length + } + val countRateError = sqrt(count.toDouble()) / length + + return NumassAnalyzerResult { + this.length = length.toLong() + this.count = count.toLong() + this.countRate = countRate + this.countRateError = countRateError + this.window = loChannel.toUInt().rangeTo(upChannel.toUInt()) + //TODO NumassAnalyzer.timestamp to block.startTime + } + } +} diff --git a/numass-analysis/src/commonMain/kotlin/ru/inr/mass/data/analysis/SmartAnalyzer.kt b/numass-analysis/src/commonMain/kotlin/ru/inr/mass/data/analysis/SmartAnalyzer.kt new file mode 100644 index 0000000..c421108 --- /dev/null +++ b/numass-analysis/src/commonMain/kotlin/ru/inr/mass/data/analysis/SmartAnalyzer.kt @@ -0,0 +1,104 @@ +///* +// * Copyright 2017 Alexander Nozik. +// * +// * Licensed under the Apache License, Version 2.0 (the "License"); +// * you may not use this file except in compliance with the License. +// * You may obtain a copy of the License at +// * +// * http://www.apache.org/licenses/LICENSE-2.0 +// * +// * Unless required by applicable law or agreed to in writing, software +// * distributed under the License is distributed on an "AS IS" BASIS, +// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// * See the License for the specific language governing permissions and +// * limitations under the License. +// */ +// +package ru.inr.mass.data.analysis + +import inr.numass.data.analyzers.SimpleAnalyzer +import kotlinx.coroutines.flow.Flow +import ru.inr.mass.data.api.NumassBlock +import ru.inr.mass.data.api.NumassEvent +import ru.inr.mass.data.api.NumassSet +import ru.inr.mass.data.api.SignalProcessor +import space.kscience.dataforge.meta.Meta +import space.kscience.dataforge.meta.descriptors.MetaDescriptor +import space.kscience.dataforge.meta.get +import space.kscience.dataforge.meta.string +import space.kscience.dataforge.tables.Table +import space.kscience.dataforge.values.Value +import space.kscience.dataforge.values.asValue +import space.kscience.dataforge.values.setValue + +/** + * An analyzer dispatcher which uses different analyzer for different meta + * Created by darksnake on 11.07.2017. + */ +public open class SmartAnalyzer(processor: SignalProcessor? = null) : AbstractAnalyzer(processor) { + private val simpleAnalyzer = SimpleAnalyzer(processor) + private val debunchAnalyzer = DebunchAnalyzer(processor) + private val timeAnalyzer: NumassAnalyzer = TODO()// TimeAnalyzer(processor) + + override val descriptor: MetaDescriptor? = null + + private fun getAnalyzer(config: Meta): NumassAnalyzer = when (val type = config["type"]?.string) { + null -> if (config["t0"] != null) { + timeAnalyzer + } else { + simpleAnalyzer + } + "simple" -> simpleAnalyzer + "time" -> timeAnalyzer + "debunch" -> debunchAnalyzer + else -> throw IllegalArgumentException("Analyzer $type not found") + } + + override suspend fun analyze(block: NumassBlock, config: Meta): NumassAnalyzerResult { + val analyzer = getAnalyzer(config) + val res = analyzer.analyze(block, config) + return NumassAnalyzerResult.read(res.meta).apply { + setValue(T0_KEY, 0.0.asValue()) + } + } + + override fun getEvents(block: NumassBlock, meta: Meta): Flow = + getAnalyzer(meta).getEvents(block, meta) + + + override suspend fun analyzeSet(set: NumassSet, config: Meta): Table { + return getAnalyzer(config).analyzeSet(set, config) +// fun Value.computeExpression(point: NumassPoint): Int { +// return when { +// this.type == ValueType.NUMBER -> this.int +// this.type == ValueType.STRING -> { +// val exprParams = HashMap() +// +// exprParams["U"] = point.voltage +// +// ExpressionUtils.function(this.string, exprParams).toInt() +// } +// else -> error("Can't interpret $type as expression or number") +// } +// } +// +// val lo = config.getValue("window.lo", 0) +// val up = config.getValue("window.up", Int.MAX_VALUE) +// +// val format = getTableFormat(config) +// +// return ListTable.Builder(format) +// .rows(set.points.map { point -> +// val newConfig = config.builder.apply { +// setValue("window.lo", lo.computeExpression(point)) +// setValue("window.up", up.computeExpression(point)) +// } +// analyzeParent(point, newConfig) +// }) +// .build() + } + + public companion object : SmartAnalyzer() { + public const val T0_KEY: String = "t0" + } +} \ No newline at end of file diff --git a/numass-analysis/src/commonMain/kotlin/ru/inr/mass/data/analysis/TimeAnalyzer.kt b/numass-analysis/src/commonMain/kotlin/ru/inr/mass/data/analysis/TimeAnalyzer.kt new file mode 100644 index 0000000..7ac8b4f --- /dev/null +++ b/numass-analysis/src/commonMain/kotlin/ru/inr/mass/data/analysis/TimeAnalyzer.kt @@ -0,0 +1,300 @@ +///* +// * Copyright 2017 Alexander Nozik. +// * +// * Licensed under the Apache License, Version 2.0 (the "License"); +// * you may not use this file except in compliance with the License. +// * You may obtain a copy of the License at +// * +// * http://www.apache.org/licenses/LICENSE-2.0 +// * +// * Unless required by applicable law or agreed to in writing, software +// * distributed under the License is distributed on an "AS IS" BASIS, +// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// * See the License for the specific language governing permissions and +// * limitations under the License. +// */ +// +package ru.inr.mass.data.analysis +// +//import hep.dataforge.description.ValueDef +//import hep.dataforge.description.ValueDefs +//import hep.dataforge.meta.Meta +//import hep.dataforge.tables.Adapters.* +//import hep.dataforge.tables.TableFormat +//import hep.dataforge.tables.TableFormatBuilder +//import hep.dataforge.values.* +//import ru.inr.mass.data.analysis.NumassAnalyzer.Companion.COUNT_KEY +//import ru.inr.mass.data.analysis.NumassAnalyzer.Companion.COUNT_RATE_ERROR_KEY +//import ru.inr.mass.data.analysis.NumassAnalyzer.Companion.COUNT_RATE_KEY +//import ru.inr.mass.data.analysis.NumassAnalyzer.Companion.LENGTH_KEY +//import ru.inr.mass.data.analysis.TimeAnalyzer.AveragingMethod.* +//import inr.numass.data.api.* +//import inr.numass.data.api.NumassPoint.Companion.HV_KEY +//import ru.inr.mass.data.api.NumassBlock +//import ru.inr.mass.data.api.SignalProcessor +//import space.kscience.dataforge.values.ValueType +//import java.util.* +//import java.util.concurrent.atomic.AtomicLong +//import kotlin.collections.List +//import kotlin.collections.asSequence +//import kotlin.collections.count +//import kotlin.collections.first +//import kotlin.collections.map +//import kotlin.collections.set +//import kotlin.collections.sortBy +//import kotlin.collections.sumBy +//import kotlin.collections.sumByDouble +//import kotlin.collections.toMutableList +//import kotlin.math.* +//import kotlin.streams.asSequence +// +// +///** +// * An analyzer which uses time information from events +// * Created by darksnake on 11.07.2017. +// */ +//@ValueDefs( +// ValueDef( +// key = "separateParallelBlocks", +// type = [ValueType.BOOLEAN], +// info = "If true, then parallel blocks will be forced to be evaluated separately" +// ), +// ValueDef( +// key = "chunkSize", +// type = [ValueType.NUMBER], +// def = "-1", +// info = "The number of events in chunk to split the chain into. If negative, no chunks are used" +// ) +//) +//open class TimeAnalyzer(processor: SignalProcessor? = null) : AbstractAnalyzer(processor) { +// +// override fun analyze(block: NumassBlock, config: Meta): Values { +// //In case points inside points +// if (block is ParentBlock && (block.isSequential || config.getBoolean("separateParallelBlocks", false))) { +// return analyzeParent(block, config) +// } +// +// val t0 = getT0(block, config).toLong() +// +// val chunkSize = config.getInt("chunkSize", -1) +// +// val count = super.getEvents(block, config).count() +// val length = block.length.toNanos().toDouble() / 1e9 +// +// val res = when { +// count < 1000 -> ValueMap.ofPairs( +// LENGTH_KEY to length, +// COUNT_KEY to count, +// COUNT_RATE_KEY to count.toDouble() / length, +// COUNT_RATE_ERROR_KEY to sqrt(count.toDouble()) / length +// ) +// chunkSize > 0 -> getEventsWithDelay(block, config) +// .chunked(chunkSize) { analyzeSequence(it.asSequence(), t0) } +// .toList() +// .mean(config.getEnum("mean", WEIGHTED)) +// else -> analyzeSequence(getEventsWithDelay(block, config), t0) +// } +// +// return ValueMap.Builder(res) +// .putValue("blockLength", length) +// .putValue(NumassAnalyzer.WINDOW_KEY, config.getRange()) +// .putValue(NumassAnalyzer.TIME_KEY, block.startTime) +// .putValue(T0_KEY, t0.toDouble() / 1000.0) +// .build() +// } +// +// +// private fun analyzeSequence(sequence: Sequence>, t0: Long): Values { +// val totalN = AtomicLong(0) +// val totalT = AtomicLong(0) +// sequence.filter { pair -> pair.second >= t0 } +// .forEach { pair -> +// totalN.incrementAndGet() +// //TODO add progress listener here +// totalT.addAndGet(pair.second) +// } +// +// if (totalN.toInt() == 0) { +// error("Zero number of intervals") +// } +// +// val countRate = +// 1e6 * totalN.get() / (totalT.get() / 1000 - t0 * totalN.get() / 1000)//1e9 / (totalT.get() / totalN.get() - t0); +// val countRateError = countRate / sqrt(totalN.get().toDouble()) +// val length = totalT.get() / 1e9 +// val count = (length * countRate).toLong() +// +// return ValueMap.ofPairs( +// LENGTH_KEY to length, +// COUNT_KEY to count, +// COUNT_RATE_KEY to countRate, +// COUNT_RATE_ERROR_KEY to countRateError +// ) +// +// } +// +// override fun analyzeParent(point: ParentBlock, config: Meta): Values { +// //Average count rates, do not sum events +// val res = point.blocks.map { it -> analyze(it, config) } +// +// val map = HashMap(res.mean(config.getEnum("mean", WEIGHTED)).asMap()) +// if (point is NumassPoint) { +// map[HV_KEY] = Value.of(point.voltage) +// } +// return ValueMap(map) +// } +// +// enum class AveragingMethod { +// ARITHMETIC, +// WEIGHTED, +// GEOMETRIC +// } +// +// /** +// * Combine multiple blocks from the same point into one +// * +// * @return +// */ +// private fun List.mean(method: AveragingMethod): Values { +// +// if (this.isEmpty()) { +// return ValueMap.Builder() +// .putValue(LENGTH_KEY, 0) +// .putValue(COUNT_KEY, 0) +// .putValue(COUNT_RATE_KEY, 0) +// .putValue(COUNT_RATE_ERROR_KEY, 0) +// .build() +// } +// +// val totalTime = sumByDouble { it.getDouble(LENGTH_KEY) } +// +// val (countRate, countRateDispersion) = when (method) { +// ARITHMETIC -> Pair( +// sumByDouble { it.getDouble(COUNT_RATE_KEY) } / size, +// sumByDouble { it.getDouble(COUNT_RATE_ERROR_KEY).pow(2.0) } / size / size +// ) +// WEIGHTED -> Pair( +// sumByDouble { it.getDouble(COUNT_RATE_KEY) * it.getDouble(LENGTH_KEY) } / totalTime, +// sumByDouble { (it.getDouble(COUNT_RATE_ERROR_KEY) * it.getDouble(LENGTH_KEY) / totalTime).pow(2.0) } +// ) +// GEOMETRIC -> { +// val mean = exp(sumByDouble { ln(it.getDouble(COUNT_RATE_KEY)) } / size) +// val variance = (mean / size).pow(2.0) * sumByDouble { +// (it.getDouble(COUNT_RATE_ERROR_KEY) / it.getDouble( +// COUNT_RATE_KEY +// )).pow(2.0) +// } +// Pair(mean, variance) +// } +// } +// +// return ValueMap.Builder(first()) +// .putValue(LENGTH_KEY, totalTime) +// .putValue(COUNT_KEY, sumBy { it.getInt(COUNT_KEY) }) +// .putValue(COUNT_RATE_KEY, countRate) +// .putValue(COUNT_RATE_ERROR_KEY, sqrt(countRateDispersion)) +// .build() +// } +// +// @ValueDefs( +// ValueDef(key = "t0", type = arrayOf(ValueType.NUMBER), info = "Constant t0 cut"), +// ValueDef( +// key = "t0.crFraction", +// type = arrayOf(ValueType.NUMBER), +// info = "The relative fraction of events that should be removed by time cut" +// ), +// ValueDef(key = "t0.min", type = arrayOf(ValueType.NUMBER), def = "0", info = "Minimal t0") +// ) +// protected fun getT0(block: NumassBlock, meta: Meta): Int { +// return if (meta.hasValue("t0")) { +// meta.getInt("t0") +// } else if (meta.hasMeta("t0")) { +// val fraction = meta.getDouble("t0.crFraction") +// val cr = estimateCountRate(block) +// if (cr < meta.getDouble("t0.minCR", 0.0)) { +// 0 +// } else { +// max(-1e9 / cr * ln(1.0 - fraction), meta.getDouble("t0.min", 0.0)).toInt() +// } +// } else { +// 0 +// } +// +// } +// +// private fun estimateCountRate(block: NumassBlock): Double { +// return block.events.count().toDouble() / block.length.toMillis() * 1000 +// } +// +// fun zipEvents(block: NumassBlock, config: Meta): Sequence> { +// return getAllEvents(block).asSequence().zipWithNext() +// } +// +// /** +// * The chain of event with delays in nanos +// * +// * @param block +// * @param config +// * @return +// */ +// fun getEventsWithDelay(block: NumassBlock, config: Meta): Sequence> { +// val inverted = config.getBoolean("inverted", true) +// //range is included in super.getEvents +// val events = super.getEvents(block, config).toMutableList() +// +// if (config.getBoolean("sortEvents", false) || (block is ParentBlock && !block.isSequential)) { +// //sort in place if needed +// events.sortBy { it.timeOffset } +// } +// +// return events.asSequence().zipWithNext { prev, next -> +// val delay = max(next.timeOffset - prev.timeOffset, 0) +// if (inverted) { +// Pair(next, delay) +// } else { +// Pair(prev, delay) +// } +// } +// } +// +// /** +// * The filtered stream of events +// * +// * @param block +// * @param meta +// * @return +// */ +// override fun getEvents(block: NumassBlock, meta: Meta): List { +// val t0 = getT0(block, meta).toLong() +// return getEventsWithDelay(block, meta) +// .filter { pair -> pair.second >= t0 } +// .map { it.first }.toList() +// } +// +// public override fun getTableFormat(config: Meta): TableFormat { +// return TableFormatBuilder() +// .addNumber(HV_KEY, X_VALUE_KEY) +// .addNumber(LENGTH_KEY) +// .addNumber(COUNT_KEY) +// .addNumber(COUNT_RATE_KEY, Y_VALUE_KEY) +// .addNumber(COUNT_RATE_ERROR_KEY, Y_ERROR_KEY) +// .addColumn(NumassAnalyzer.WINDOW_KEY) +// .addTime() +// .addNumber(T0_KEY) +// .build() +// } +// +// companion object { +// const val T0_KEY = "t0" +// +// val NAME_LIST = arrayOf( +// LENGTH_KEY, +// COUNT_KEY, +// COUNT_RATE_KEY, +// COUNT_RATE_ERROR_KEY, +// NumassAnalyzer.WINDOW_KEY, +// NumassAnalyzer.TIME_KEY, +// T0_KEY +// ) +// } +//} diff --git a/numass-analysis/src/commonMain/kotlin/ru/inr/mass/data/analysis/Values.kt b/numass-analysis/src/commonMain/kotlin/ru/inr/mass/data/analysis/Values.kt new file mode 100644 index 0000000..9529c62 --- /dev/null +++ b/numass-analysis/src/commonMain/kotlin/ru/inr/mass/data/analysis/Values.kt @@ -0,0 +1,5 @@ +package ru.inr.mass.data.analysis + +import space.kscience.dataforge.values.Value + +public typealias Values = Map \ No newline at end of file diff --git a/numass-data-model/build.gradle.kts b/numass-data-model/build.gradle.kts index f1c4d0f..351bae7 100644 --- a/numass-data-model/build.gradle.kts +++ b/numass-data-model/build.gradle.kts @@ -1,11 +1,8 @@ plugins { - kotlin("multiplatform") - id("ru.mipt.npm.gradle.common") + id("ru.mipt.npm.gradle.mpp") + `maven-publish` } -kscience { - publish() -} val dataforgeVersion: String by rootProject.extra diff --git a/numass-data-model/src/commonMain/kotlin/ru/inr/mass/data/api/MetaBlock.kt b/numass-data-model/src/commonMain/kotlin/ru/inr/mass/data/api/MetaBlock.kt index d0d555f..4dd37b7 100644 --- a/numass-data-model/src/commonMain/kotlin/ru/inr/mass/data/api/MetaBlock.kt +++ b/numass-data-model/src/commonMain/kotlin/ru/inr/mass/data/api/MetaBlock.kt @@ -3,6 +3,7 @@ package ru.inr.mass.data.api import kotlinx.coroutines.flow.* import kotlinx.datetime.Instant import kotlin.time.Duration +import kotlin.time.Duration.Companion.nanoseconds import kotlin.time.DurationUnit public interface ParentBlock : NumassBlock { @@ -19,7 +20,7 @@ public interface ParentBlock : NumassBlock { * A block constructed from a set of other blocks. Internal blocks are not necessary subsequent. Blocks are automatically sorted. * Created by darksnake on 16.07.2017. */ -public class MetaBlock(private val blocks: List) : ParentBlock { +public open class MetaBlock(protected val blocks: List) : ParentBlock { override fun flowBlocks(): Flow = blocks.asFlow() @@ -27,7 +28,7 @@ public class MetaBlock(private val blocks: List) : ParentBlock { get() = blocks.first().startTime override suspend fun getLength(): Duration = - Duration.nanoseconds(blocks.sumOf { it.getLength().toDouble(DurationUnit.NANOSECONDS) }) + blocks.sumOf { it.getLength().toDouble(DurationUnit.NANOSECONDS) }.nanoseconds override val events: Flow get() = flow { @@ -37,5 +38,8 @@ public class MetaBlock(private val blocks: List) : ParentBlock { override val frames: Flow get() = blocks.sortedBy { it.startTime }.asFlow().flatMapConcat { it.frames } - + override val eventsCount: Long + get() = blocks.sumOf { it.eventsCount } + override val framesCount: Long + get() = blocks.sumOf { it.framesCount } } diff --git a/numass-data-model/src/commonMain/kotlin/ru/inr/mass/data/api/NumassBlock.kt b/numass-data-model/src/commonMain/kotlin/ru/inr/mass/data/api/NumassBlock.kt index f67fe17..c4b2622 100644 --- a/numass-data-model/src/commonMain/kotlin/ru/inr/mass/data/api/NumassBlock.kt +++ b/numass-data-model/src/commonMain/kotlin/ru/inr/mass/data/api/NumassBlock.kt @@ -25,10 +25,10 @@ import kotlinx.datetime.plus import kotlin.time.Duration public open class OrphanNumassEvent( - public val amplitude: Short, + public val amplitude: UShort, public val timeOffset: Long, ) : Comparable { - public operator fun component1(): Short = amplitude + public operator fun component1(): UShort = amplitude public operator fun component2(): Long = timeOffset override fun compareTo(other: OrphanNumassEvent): Int { @@ -46,7 +46,7 @@ public open class OrphanNumassEvent( * */ public class NumassEvent( - amplitude: Short, + amplitude: UShort, timeOffset: Long, public val owner: NumassBlock, ) : OrphanNumassEvent(amplitude, timeOffset) @@ -74,11 +74,15 @@ public interface NumassBlock { */ public suspend fun getLength(): Duration + public val eventsCount: Long + /** * Stream of isolated events. Could be empty */ public val events: Flow + public val framesCount: Long + /** * Stream of frames. Could be empty */ @@ -109,6 +113,9 @@ public class SimpleBlock( override val events: Flow get() = eventList.asFlow() + override val eventsCount: Long get() = eventList.size.toLong() + override val framesCount: Long get() = 0L + public companion object { } diff --git a/numass-data-model/src/commonMain/kotlin/ru/inr/mass/data/api/NumassPoint.kt b/numass-data-model/src/commonMain/kotlin/ru/inr/mass/data/api/NumassPoint.kt index cfb155c..2626055 100644 --- a/numass-data-model/src/commonMain/kotlin/ru/inr/mass/data/api/NumassPoint.kt +++ b/numass-data-model/src/commonMain/kotlin/ru/inr/mass/data/api/NumassPoint.kt @@ -16,18 +16,20 @@ package ru.inr.mass.data.api +import kotlinx.coroutines.FlowPreview import kotlinx.coroutines.flow.* import space.kscience.dataforge.meta.Meta import space.kscience.dataforge.meta.double import space.kscience.dataforge.meta.get import space.kscience.dataforge.meta.int import kotlin.time.Duration +import kotlin.time.Duration.Companion.nanoseconds import kotlin.time.DurationUnit -import kotlin.time.nanoseconds /** * Created by darksnake on 06-Jul-17. */ +@OptIn(FlowPreview::class) public interface NumassPoint : ParentBlock { public val meta: Meta @@ -57,8 +59,8 @@ public interface NumassPoint : ParentBlock { /** * Get the length key of meta or calculate length as a sum of block lengths. The latter could be a bit slow */ - override suspend fun getLength(): Duration = - flowBlocks().filter { it.channel == 0 }.toList().sumOf { it.getLength().toDouble(DurationUnit.NANOSECONDS) }.nanoseconds + override suspend fun getLength(): Duration = flowBlocks().filter { it.channel == 0 }.toList() + .sumOf { it.getLength().toLong(DurationUnit.NANOSECONDS) }.nanoseconds /** * Get all events it all blocks as a single sequence diff --git a/numass-data-model/src/commonMain/kotlin/ru/inr/mass/data/api/NumassSet.kt b/numass-data-model/src/commonMain/kotlin/ru/inr/mass/data/api/NumassSet.kt index 0f3e8f2..15d9bbc 100644 --- a/numass-data-model/src/commonMain/kotlin/ru/inr/mass/data/api/NumassSet.kt +++ b/numass-data-model/src/commonMain/kotlin/ru/inr/mass/data/api/NumassSet.kt @@ -10,7 +10,6 @@ import space.kscience.dataforge.meta.Meta import space.kscience.dataforge.meta.get import space.kscience.dataforge.meta.long import space.kscience.dataforge.names.Name -import space.kscience.dataforge.names.toName import space.kscience.dataforge.provider.Provider /** @@ -35,18 +34,14 @@ public interface NumassSet : Iterable, Provider { //suspend fun getHvData(): Table? - override fun iterator(): Iterator { - return points.iterator() - } + override fun iterator(): Iterator = points.iterator() override val defaultTarget: String get() = NUMASS_POINT_TARGET - override fun content(target: String): Map { - return if (target == NUMASS_POINT_TARGET) { - points.associateBy { "point[${it.voltage}]".toName() } - } else { - super.content(target) - } + override fun content(target: String): Map = if (target == NUMASS_POINT_TARGET) { + points.associateBy { Name.parse("point[${it.voltage}]") } + } else { + super.content(target) } public companion object { diff --git a/numass-data-model/src/commonMain/kotlin/ru/inr/mass/data/api/SimpleNumassPoint.kt b/numass-data-model/src/commonMain/kotlin/ru/inr/mass/data/api/SimpleNumassPoint.kt deleted file mode 100644 index 955a443..0000000 --- a/numass-data-model/src/commonMain/kotlin/ru/inr/mass/data/api/SimpleNumassPoint.kt +++ /dev/null @@ -1,26 +0,0 @@ -package ru.inr.mass.data.api - -import kotlinx.coroutines.flow.Flow -import kotlinx.coroutines.flow.asFlow -import kotlinx.datetime.Instant -import space.kscience.dataforge.meta.Meta - -/** - * A simple static implementation of NumassPoint - * Created by darksnake on 08.07.2017. - */ -public class SimpleNumassPoint( - private val blocks: List, - override val meta: Meta, - override val startTime: Instant = Instant.DISTANT_PAST, - override val sequential: Boolean = true, -) : NumassPoint { - - init { - check(blocks.isNotEmpty()) { "No blocks in a point" } - } - - override fun flowBlocks(): Flow = blocks.asFlow() - - override fun toString(): String = "SimpleNumassPoint(index = ${index}, hv = $voltage)" -} diff --git a/numass-data-proto/build.gradle.kts b/numass-data-proto/build.gradle.kts index 740f2df..742406f 100644 --- a/numass-data-proto/build.gradle.kts +++ b/numass-data-proto/build.gradle.kts @@ -2,10 +2,7 @@ plugins { kotlin("jvm") id("ru.mipt.npm.gradle.common") id("com.squareup.wire") version "3.5.0" -} - -kscience{ - publish() + `maven-publish` } val dataforgeVersion: String by rootProject.extra diff --git a/numass-data-proto/src/main/kotlin/ru/inr/mass/data/proto/ProtoNumassPoint.kt b/numass-data-proto/src/main/kotlin/ru/inr/mass/data/proto/ProtoNumassPoint.kt index efd2278..850cd67 100644 --- a/numass-data-proto/src/main/kotlin/ru/inr/mass/data/proto/ProtoNumassPoint.kt +++ b/numass-data-proto/src/main/kotlin/ru/inr/mass/data/proto/ProtoNumassPoint.kt @@ -32,6 +32,8 @@ import java.io.ByteArrayOutputStream import java.io.InputStream import java.util.zip.Inflater import kotlin.time.Duration +import kotlin.time.Duration.Companion.milliseconds +import kotlin.time.Duration.Companion.nanoseconds /** * Protobuf based numass point @@ -42,7 +44,7 @@ internal class ProtoNumassPoint( private val protoBuilder: () -> Point, ) : NumassPoint { - val point by lazy(protoBuilder) + val point: Point by lazy(protoBuilder) override fun flowBlocks() = point.channels.flatMap { channel -> channel.blocks @@ -65,9 +67,23 @@ internal class ProtoNumassPoint( } ?: Instant.DISTANT_PAST override suspend fun getLength(): Duration = meta["acquisition_time"].double?.let { - Duration.milliseconds(it * 1000) + (it * 1000).milliseconds } ?: super.getLength() + override val eventsCount: Long + get() = point.channels.sumOf { channel -> + channel.blocks.sumOf { block -> + block.events?.amplitudes?.size ?: 0 + }.toLong() + } + + override val framesCount: Long + get() = point.channels.sumOf { channel -> + channel.blocks.sumOf { block -> + block.frames.size ?: 0 + }.toLong() + } + override fun toString(): String = "ProtoNumassPoint(index = ${index}, hv = $voltage)" public companion object { @@ -128,15 +144,15 @@ public class ProtoNumassBlock( } override suspend fun getLength(): Duration = when { - block.length > 0 -> Duration.nanoseconds(block.length) - parent?.meta["acquisition_time"] != null -> - Duration.milliseconds((parent?.meta["acquisition_time"].double ?: 0.0 * 1000)) + block.length > 0 -> block.length.nanoseconds + parent?.meta?.get("acquisition_time") != null -> + (parent.meta["acquisition_time"].double ?: (0.0 * 1000)).milliseconds else -> { LoggerFactory.getLogger(javaClass) .error("No length information on block. Trying to infer from first and last events") val times = runBlocking { events.map { it.timeOffset }.toList() } val nanos = (times.maxOrNull()!! - times.minOrNull()!!) - Duration.nanoseconds(nanos) + nanos.nanoseconds } } @@ -152,7 +168,7 @@ public class ProtoNumassBlock( } amplitudes.zip(times) { amp, time -> - NumassEvent(amp.toShort(), time, this) + NumassEvent(amp.toUShort(), time, this) }.asFlow() } else { @@ -170,11 +186,14 @@ public class ProtoNumassBlock( override val frames: Flow get() { - val tickSize = Duration.nanoseconds(block.bin_size) + val tickSize = block.bin_size.nanoseconds return block.frames.asFlow().map { frame -> val time = startTime.plus(frame.time, DateTimeUnit.NANOSECOND) val frameData = frame.data_ NumassFrame(time, tickSize, frameData.toShortArray()) } } + + override val eventsCount: Long get() = block.frames.size.toLong() + override val framesCount: Long get() = block.events?.amplitudes?.size?.toLong() ?: 0L } \ No newline at end of file diff --git a/numass-data-proto/src/main/kotlin/ru/inr/mass/data/proto/TaggedNumassEnvelopeFormat.kt b/numass-data-proto/src/main/kotlin/ru/inr/mass/data/proto/TaggedNumassEnvelopeFormat.kt index 1251067..3621747 100644 --- a/numass-data-proto/src/main/kotlin/ru/inr/mass/data/proto/TaggedNumassEnvelopeFormat.kt +++ b/numass-data-proto/src/main/kotlin/ru/inr/mass/data/proto/TaggedNumassEnvelopeFormat.kt @@ -24,7 +24,6 @@ import space.kscience.dataforge.meta.get import space.kscience.dataforge.meta.string import space.kscience.dataforge.names.Name import space.kscience.dataforge.names.plus -import space.kscience.dataforge.names.toName import java.util.* @@ -113,7 +112,7 @@ internal class TaggedNumassEnvelopeFormat(private val io: IOPlugin) : EnvelopeFo override fun invoke(meta: Meta, context: Context): EnvelopeFormat { val io = context.io - val metaFormatName = meta["name"].string?.toName() ?: JsonMetaFormat.name + val metaFormatName = meta["name"].string?.let(Name::parse) ?: JsonMetaFormat.name //Check if appropriate factory exists io.metaFormatFactories.find { it.name == metaFormatName } ?: error("Meta format could not be resolved") diff --git a/numass-data-proto/src/test/kotlin/ru/inr/mass/data/proto/TestNumassDirectory.kt b/numass-data-proto/src/test/kotlin/ru/inr/mass/data/proto/TestNumassDirectory.kt index 224652d..5cad5c6 100644 --- a/numass-data-proto/src/test/kotlin/ru/inr/mass/data/proto/TestNumassDirectory.kt +++ b/numass-data-proto/src/test/kotlin/ru/inr/mass/data/proto/TestNumassDirectory.kt @@ -4,7 +4,6 @@ import org.junit.jupiter.api.Test import space.kscience.dataforge.context.Context import space.kscience.dataforge.meta.get import space.kscience.dataforge.meta.string -import space.kscience.dataforge.meta.value import space.kscience.dataforge.values.ListValue import java.nio.file.Path import kotlin.test.assertEquals @@ -19,7 +18,7 @@ class TestNumassDirectory { val dataPath = Path.of("src/test/resources", "testData/set_4") val testSet = context.readNumassDirectory(dataPath) assertEquals("2018-04-13T22:01:46", testSet.meta["end_time"].string) - assertEquals(ListValue.EMPTY, testSet.meta["comments"].value) + assertEquals(ListValue.EMPTY, testSet.meta["comments"]?.value) assertEquals(31, testSet.points.size) val point22 = testSet.points.find { it.index == 22 }!! point22.flowBlocks() diff --git a/numass-workspace/build.gradle.kts b/numass-workspace/build.gradle.kts index 10782a5..07f0fec 100644 --- a/numass-workspace/build.gradle.kts +++ b/numass-workspace/build.gradle.kts @@ -16,6 +16,7 @@ val kmathVersion: String by rootProject.extra dependencies { implementation(project(":numass-data-proto")) implementation(project(":numass-model")) + implementation(project(":numass-analysis")) implementation("space.kscience:dataforge-workspace:$dataforgeVersion") implementation("space.kscience:plotlykt-core:$plotlyVersion") implementation("space.kscience:kmath-histograms:$kmathVersion") diff --git a/numass-workspace/src/main/kotlin/ru/inr/mass/scripts/run_2020_12.kt b/numass-workspace/src/main/kotlin/ru/inr/mass/scripts/run_2020_12.kt index c1c5f1c..9802148 100644 --- a/numass-workspace/src/main/kotlin/ru/inr/mass/scripts/run_2020_12.kt +++ b/numass-workspace/src/main/kotlin/ru/inr/mass/scripts/run_2020_12.kt @@ -5,13 +5,13 @@ import ru.inr.mass.data.proto.NumassDirectorySet import ru.inr.mass.workspace.readNumassRepository import space.kscience.dataforge.data.DataTree import space.kscience.dataforge.data.filter -import space.kscience.dataforge.meta.get import space.kscience.dataforge.meta.string suspend fun main() { val repo: DataTree = readNumassRepository("D:\\Work\\Numass\\data\\2018_04") val filtered = repo.filter { _, data -> - data.meta["operator"].string?.startsWith("Vas") ?: false + val operator by data.meta.string() + operator?.startsWith("Vas") ?: false } filtered.flow().collect { diff --git a/numass-workspace/src/main/kotlin/ru/inr/mass/workspace/NumassPlugin.kt b/numass-workspace/src/main/kotlin/ru/inr/mass/workspace/NumassPlugin.kt new file mode 100644 index 0000000..7e6fc4d --- /dev/null +++ b/numass-workspace/src/main/kotlin/ru/inr/mass/workspace/NumassPlugin.kt @@ -0,0 +1,70 @@ +package ru.inr.mass.workspace + +import ru.inr.mass.data.analysis.SmartAnalyzer +import ru.inr.mass.data.api.NumassSet +import ru.inr.mass.data.proto.NumassProtoPlugin +import space.kscience.dataforge.context.Context +import space.kscience.dataforge.context.PluginFactory +import space.kscience.dataforge.context.PluginTag +import space.kscience.dataforge.data.select +import space.kscience.dataforge.meta.Meta +import space.kscience.dataforge.meta.boolean +import space.kscience.dataforge.meta.descriptors.MetaDescriptor +import space.kscience.dataforge.meta.descriptors.value +import space.kscience.dataforge.meta.get +import space.kscience.dataforge.meta.toMutableMeta +import space.kscience.dataforge.names.Name +import space.kscience.dataforge.tables.Table +import space.kscience.dataforge.values.Value +import space.kscience.dataforge.values.ValueType +import space.kscience.dataforge.workspace.WorkspacePlugin +import space.kscience.dataforge.workspace.pipeFrom +import space.kscience.dataforge.workspace.task +import kotlin.reflect.KClass + +class NumassPlugin : WorkspacePlugin() { + override val tag: PluginTag get() = Companion.tag + + val numassProtoPlugin by require(NumassProtoPlugin) + + val select by task( + descriptor = MetaDescriptor { + info = "Select data from workspace data pool" + value("forward", ValueType.BOOLEAN) { + info = "Select only forward or only backward sets" + } + } + ) { + val forward = meta["forward"]?.boolean + val filtered = workspace.data.select { _, meta -> + when (forward) { + true -> meta["iteration_info.reverse"]?.boolean?.not() ?: false + false -> meta["iteration_info.reverse"]?.boolean ?: false + else -> true + } + } + + emit(Name.EMPTY, filtered) + } + + val analyze by task>( + MetaDescriptor { + info = "Count the number of events for each voltage and produce a table with the results" + } + ) { + pipeFrom(select) { set, name, meta -> + val res = SmartAnalyzer.analyzeSet(set, meta["analyzer"] ?: Meta.EMPTY) + val outputMeta = meta.toMutableMeta().apply { + "data" put set.meta + } + // context.output.render(res, stage = "numass.analyze", name = name, meta = outputMeta) + res + } + } + + companion object : PluginFactory { + override val tag: PluginTag = PluginTag("numass", "ru.mipt.npm") + override val type: KClass = NumassPlugin::class + override fun invoke(meta: Meta, context: Context): NumassPlugin = NumassPlugin() + } +} \ No newline at end of file diff --git a/numass-workspace/src/main/kotlin/ru/inr/mass/workspace/workspace.kt b/numass-workspace/src/main/kotlin/ru/inr/mass/workspace/workspace.kt index 2da8f3e..7d1785f 100644 --- a/numass-workspace/src/main/kotlin/ru/inr/mass/workspace/workspace.kt +++ b/numass-workspace/src/main/kotlin/ru/inr/mass/workspace/workspace.kt @@ -5,7 +5,6 @@ import space.kscience.dataforge.workspace.Workspace val NUMASS = Workspace { context{ - name("NUMASS") plugin(NumassProtoPlugin) } } \ No newline at end of file diff --git a/settings.gradle.kts b/settings.gradle.kts index 26c9cf7..3df4d32 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -2,13 +2,12 @@ pluginManagement { repositories { maven("https://repo.kotlin.link") mavenCentral() - jcenter() gradlePluginPortal() mavenLocal() } - val toolsVersion = "0.9.9" - val kotlinVersion = "1.5.0" + val toolsVersion = "0.10.7" + val kotlinVersion = "1.6.0" plugins { id("ru.mipt.npm.gradle.project") version toolsVersion @@ -30,7 +29,10 @@ pluginManagement { } } -include("numass-data-model") -include("numass-data-proto") -include("numass-workspace") -include("numass-model") +include( + ":numass-data-model", + ":numass-analysis", + ":numass-data-proto", + ":numass-workspace", + ":numass-model" +)