Quick-patch analyzers. They need more work

build.gradle.kts

@@ -12,10 +12,10 @@ allprojects {
     version = "0.1.0-dev-1"
 }
 
-val dataforgeVersion by extra("0.4.0")
+val dataforgeVersion by extra("0.5.2-dev-2")
 val kmathVersion by extra("0.3.0-dev-15")
 
 ksciencePublish{
-    vcs("https://mipt-npm.jetbrains.space/p/numass/code/numass/")
+    git("https://mipt-npm.jetbrains.space/p/numass/code/numass/")
     space("https://maven.pkg.jetbrains.space/mipt-npm/p/numass/maven")
 }

numass-analysis/build.gradle.kts

@@ -0,0 +1,21 @@
+plugins {
+    id("ru.mipt.npm.gradle.mpp")
+    `maven-publish`
+}
+
+
+val dataforgeVersion: String by rootProject.extra
+
+kotlin.sourceSets {
+    commonMain {
+        dependencies {
+            api(project(":numass-data-model"))
+            api("space.kscience:tables-kt:0.1.1-dev-2")
+            api("space.kscience:kmath-complex:0.3.0-dev-17")
+            api("space.kscience:kmath-stat:0.3.0-dev-17")
+            api("space.kscience:kmath-histograms:0.3.0-dev-17")
+        }
+    }
+}
+
+

numass-analysis/src/commonMain/kotlin/ru/inr/mass/data/analysis/AbstractAnalyzer.kt

@@ -0,0 +1,112 @@
+/*
+ * Copyright  2017 Alexander Nozik.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package ru.inr.mass.data.analysis
+
+import kotlinx.coroutines.flow.*
+import ru.inr.mass.data.api.NumassBlock
+import ru.inr.mass.data.api.NumassEvent
+import ru.inr.mass.data.api.NumassSet
+import ru.inr.mass.data.api.SignalProcessor
+import space.kscience.dataforge.meta.Meta
+import space.kscience.dataforge.meta.get
+import space.kscience.dataforge.meta.int
+import space.kscience.dataforge.tables.RowTable
+import space.kscience.dataforge.tables.Table
+import space.kscience.dataforge.values.Value
+
+/**
+ * Created by darksnake on 11.07.2017.
+ */
+public abstract class AbstractAnalyzer(
+    private val processor: SignalProcessor? = null,
+) : NumassAnalyzer {
+
+    /**
+     * Return unsorted stream of events including events from frames.
+     * In theory, events after processing could be unsorted due to mixture of frames and events.
+     * In practice usually block have either frame or events, but not both.
+     *
+     * @param block
+     * @return
+     */
+    override fun getEvents(block: NumassBlock, meta: Meta): Flow<NumassEvent> {
+        val range = meta.getRange()
+        return getAllEvents(block).filter { event ->
+            event.amplitude.toInt() in range
+        }
+    }
+
+    protected fun Meta.getRange(): IntRange {
+        val loChannel = get("window.lo")?.int ?: 0
+        val upChannel = get("window.up")?.int ?: Int.MAX_VALUE
+        return loChannel until upChannel
+    }
+
+    protected fun getAllEvents(block: NumassBlock): Flow<NumassEvent> {
+        return when {
+            block.framesCount == 0L -> block.events
+            processor == null -> throw IllegalArgumentException("Signal processor needed to analyze frames")
+            else -> flow {
+                emitAll(block.events)
+                emitAll(block.frames.flatMapConcat { processor.analyze(it) })
+            }
+        }
+    }
+
+//    /**
+//     * Get table format for summary table
+//     *
+//     * @param config
+//     * @return
+//     */
+//    protected open fun getTableFormat(config: Meta): ValueTableHeader {
+//        return TableFormatBuilder()
+//            .addNumber(HV_KEY, X_VALUE_KEY)
+//            .addNumber(NumassAnalyzer.LENGTH_KEY)
+//            .addNumber(NumassAnalyzer.COUNT_KEY)
+//            .addNumber(NumassAnalyzer.COUNT_RATE_KEY, Y_VALUE_KEY)
+//            .addNumber(NumassAnalyzer.COUNT_RATE_ERROR_KEY, Y_ERROR_KEY)
+//            .addColumn(NumassAnalyzer.WINDOW_KEY)
+//            .addTime()
+//            .build()
+//    }
+
+    override suspend fun analyzeSet(set: NumassSet, config: Meta): Table<Value> = RowTable(
+        NumassAnalyzer.length,
+        NumassAnalyzer.count,
+        NumassAnalyzer.cr,
+        NumassAnalyzer.crError,
+//        NumassAnalyzer.window,
+//        NumassAnalyzer.timestamp
+    ) {
+
+        set.points.forEach { point ->
+            analyzeParent(point, config)
+        }
+    }
+
+    public companion object {
+//        public val NAME_LIST: List<String> = listOf(
+//            NumassAnalyzer.LENGTH_KEY,
+//            NumassAnalyzer.COUNT_KEY,
+//            NumassAnalyzer.COUNT_RATE_KEY,
+//            NumassAnalyzer.COUNT_RATE_ERROR_KEY,
+//            NumassAnalyzer.WINDOW_KEY,
+//            NumassAnalyzer.TIME_KEY
+//        )
+    }
+}

numass-analysis/src/commonMain/kotlin/ru/inr/mass/data/analysis/DebunchAnalyzer.kt

@@ -0,0 +1,35 @@
+/*
+ * Copyright  2017 Alexander Nozik.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package ru.inr.mass.data.analysis
+
+import ru.inr.mass.data.api.NumassBlock
+import ru.inr.mass.data.api.SignalProcessor
+import space.kscience.dataforge.meta.Meta
+import space.kscience.dataforge.meta.descriptors.MetaDescriptor
+
+/**
+ * Block analyzer that can perform debunching
+ * Created by darksnake on 11.07.2017.
+ */
+public class DebunchAnalyzer(processor: SignalProcessor? = null) : AbstractAnalyzer(processor) {
+
+    override suspend fun analyze(block: NumassBlock, config: Meta): NumassAnalyzerResult {
+        TODO()
+    }
+
+    override val descriptor: MetaDescriptor? = null
+}

numass-analysis/src/commonMain/kotlin/ru/inr/mass/data/analysis/NumassAnalyzer.kt

@@ -0,0 +1,268 @@
+/*
+ * Copyright  2017 Alexander Nozik.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package ru.inr.mass.data.analysis
+
+import kotlinx.coroutines.flow.Flow
+import kotlinx.coroutines.flow.collect
+import ru.inr.mass.data.analysis.NumassAnalyzer.Companion.MAX_CHANNEL
+import ru.inr.mass.data.analysis.NumassAnalyzer.Companion.channel
+import ru.inr.mass.data.analysis.NumassAnalyzer.Companion.count
+import ru.inr.mass.data.analysis.NumassAnalyzer.Companion.cr
+import ru.inr.mass.data.analysis.NumassAnalyzer.Companion.crError
+import ru.inr.mass.data.api.*
+import ru.inr.mass.data.api.NumassPoint.Companion.HV_KEY
+import ru.inr.mass.data.api.NumassPoint.Companion.LENGTH_KEY
+import space.kscience.dataforge.meta.*
+import space.kscience.dataforge.meta.descriptors.Described
+import space.kscience.dataforge.names.asName
+import space.kscience.dataforge.tables.*
+import space.kscience.dataforge.values.*
+import space.kscience.kmath.histogram.Counter
+import space.kscience.kmath.histogram.LongCounter
+import kotlin.math.max
+import kotlin.math.min
+import kotlin.math.pow
+import kotlin.math.sqrt
+
+public class NumassAnalyzerResult : Scheme() {
+    public var count: Long? by long(NumassAnalyzer.count.name.asName())
+    public var countRate: Double? by double(NumassAnalyzer.cr.name.asName())
+    public var countRateError: Double? by double(NumassAnalyzer.crError.name.asName())
+    public var length: Long? by long(NumassAnalyzer.length.name.asName())
+
+    public var voltage: Double? by double(HV_KEY.asName())
+
+    public var window: UIntRange?
+        get() = meta["window"]?.value?.list?.let {
+            it[0].int.toUInt().rangeTo(it[1].int.toUInt())
+        }
+        set(value) {
+            meta["window"] = value?.let { ListValue(it.first.toInt(), it.first.toInt()) }
+        }
+
+    public companion object : SchemeSpec<NumassAnalyzerResult>(::NumassAnalyzerResult)
+}
+
+
+/**
+ * A general raw data analysis utility. Could have different implementations
+ * Created by darksnake on 06-Jul-17.
+ */
+public interface NumassAnalyzer : Described {
+
+    /**
+     * Perform analysis on block. The values for count rate, its error and point length in nanos must
+     * exist, but occasionally additional values could also be presented.
+     *
+     * @param block
+     * @return
+     */
+    public suspend fun analyze(block: NumassBlock, config: Meta = Meta.EMPTY): NumassAnalyzerResult
+
+    /**
+     * Analysis result for point including hv information
+     * @param point
+     * @param config
+     * @return
+     */
+    public suspend fun analyzeParent(point: ParentBlock, config: Meta = Meta.EMPTY): NumassAnalyzerResult {
+//        //Add properties to config
+//        val newConfig = config.builder.apply {
+//            if (point is NumassPoint) {
+//                setValue("voltage", point.voltage)
+//                setValue("index", point.index)
+//            }
+//            setValue("channel", point.channel)
+//        }
+        val res = analyze(point, config)
+        if (point is NumassPoint) {
+            res.voltage = point.voltage
+        }
+
+        return res
+    }
+
+    /**
+     * Return unsorted stream of events including events from frames
+     *
+     * @param block
+     * @return
+     */
+    public fun getEvents(block: NumassBlock, meta: Meta = Meta.EMPTY): Flow<NumassEvent>
+
+    /**
+     * Analyze the whole set. And return results as a table
+     *
+     * @param set
+     * @param config
+     * @return
+     */
+    public suspend fun analyzeSet(set: NumassSet, config: Meta): Table<Value>
+
+    /**
+     * Get the approximate number of events in block. Not all analyzers support precise event counting
+     *
+     * @param block
+     * @param config
+     * @return
+     */
+    public suspend fun getCount(block: NumassBlock, config: Meta): Long =
+        analyze(block, config).getValue(count.name)?.long ?: 0L
+
+    /**
+     * Get approximate effective point length in nanos. It is not necessary corresponds to real point length.
+     *
+     * @param block
+     * @param config
+     * @return
+     */
+    public suspend fun getLength(block: NumassBlock, config: Meta = Meta.EMPTY): Long =
+        analyze(block, config).getValue(LENGTH_KEY)?.long ?: 0L
+
+    public companion object {
+
+        public val channel: ColumnHeader<Value> by ColumnHeader.value(ValueType.NUMBER)
+        public val count: ColumnHeader<Value> by ColumnHeader.value(ValueType.NUMBER)
+        public val length: ColumnHeader<Value> by ColumnHeader.value(ValueType.NUMBER)
+        public val cr: ColumnHeader<Value> by ColumnHeader.value(ValueType.NUMBER)
+        public val crError: ColumnHeader<Value> by ColumnHeader.value(ValueType.NUMBER)
+        public val window: ColumnHeader<Value> by ColumnHeader.value(ValueType.LIST)
+        public val timestamp: ColumnHeader<Value> by ColumnHeader.value(ValueType.NUMBER)
+//
+//        val AMPLITUDE_ADAPTER: ValuesAdapter = Adapters.buildXYAdapter(CHANNEL_KEY, COUNT_RATE_KEY)
+
+        public val MAX_CHANNEL: UInt = 10000U
+    }
+}
+
+public suspend fun NumassAnalyzer.getAmplitudeSpectrum(
+    block: NumassBlock,
+    range: UIntRange = 0U..MAX_CHANNEL,
+    config: Meta = Meta.EMPTY,
+): Table<Value> {
+    val seconds = block.getLength().inWholeMilliseconds.toDouble() / 1000.0
+    return getEvents(block, config).getAmplitudeSpectrum(seconds, range)
+}
+
+/**
+ * Calculate number of counts in the given channel
+ *
+ * @param spectrum
+ * @param loChannel
+ * @param upChannel
+ * @return
+ */
+internal fun Table<Value>.countInWindow(loChannel: Short, upChannel: Short): Long = rows.filter { row ->
+    row[channel]?.int in loChannel until upChannel
+}.sumOf { it[count]?.long ?: 0L }
+
+/**
+ * Calculate the amplitude spectrum for a given block. The s
+ *
+ * @param this@getAmplitudeSpectrum
+ * @param length length in seconds, used for count rate calculation
+ * @param config
+ * @return
+ */
+private suspend fun Flow<NumassEvent>.getAmplitudeSpectrum(
+    length: Double,
+    range: UIntRange = 0U..MAX_CHANNEL,
+): Table<Value> {
+
+    //optimized for fastest computation
+    val spectrum: MutableMap<UInt, LongCounter> = HashMap()
+    collect { event ->
+        val channel = event.amplitude
+        spectrum.getOrPut(channel.toUInt()) {
+            LongCounter()
+        }.add(1L)
+    }
+
+    return RowTable<Value>(channel, count, cr, crError) {
+        range.forEach { ch ->
+            val countValue: Long = spectrum[ch]?.value ?: 0L
+            valueRow(
+                channel to ch,
+                count to countValue,
+                cr to (countValue.toDouble() / length),
+                crError to sqrt(countValue.toDouble()) / length
+            )
+        }
+    }
+}
+
+/**
+ * Apply window and binning to a spectrum. Empty bins are filled with zeroes
+ */
+private fun Table<Value>.withBinning(
+    binSize: UInt, range: UIntRange = 0U..MAX_CHANNEL,
+): Table<Value> = RowTable<Value>(channel, count, cr, crError) {
+//    var chan = loChannel
+//        ?: this.getColumn(NumassAnalyzer.CHANNEL_KEY).stream().mapToInt { it.int }.min().orElse(0)
+//
+//    val top = upChannel
+//        ?: this.getColumn(NumassAnalyzer.CHANNEL_KEY).stream().mapToInt { it.int }.max().orElse(1)
+
+    val binSizeColumn = newColumn<Value>("binSize")
+
+    var chan = range.first
+
+    while (chan < range.last - binSize) {
+        val counter = LongCounter()
+        val countRateCounter = Counter.real()
+        val countRateDispersionCounter = Counter.real()
+
+        val binLo = chan
+        val binUp = chan + binSize
+
+        rows.filter { row ->
+            (row[channel]?.int ?: 0U) in binLo until binUp
+        }.forEach { row ->
+            counter.add(row[count]?.long ?: 0L)
+            countRateCounter.add(row[cr]?.double ?: 0.0)
+            countRateDispersionCounter.add(row[crError]?.double?.pow(2.0) ?: 0.0)
+        }
+        val bin = min(binSize, range.last - chan)
+
+        valueRow(
+            channel to (chan.toDouble() + bin.toDouble() / 2.0),
+            count to counter.value,
+            cr to countRateCounter.value,
+            crError to sqrt(countRateDispersionCounter.value),
+            binSizeColumn to bin
+        )
+        chan += binSize
+    }
+}
+
+/**
+ * Subtract reference spectrum.
+ */
+private fun subtractAmplitudeSpectrum(
+    sp1: Table<Value>, sp2: Table<Value>,
+): Table<Value> = RowTable<Value>(channel, cr, crError) {
+    sp1.rows.forEach { row1 ->
+        val channelValue = row1[channel]?.double
+        val row2 = sp2.rows.find { it[channel]?.double == channelValue } ?: MapRow(emptyMap())
+
+        val value = max((row1[cr]?.double ?: 0.0) - (row2[cr]?.double ?: 0.0), 0.0)
+        val error1 = row1[crError]?.double ?: 0.0
+        val error2 = row2[crError]?.double ?: 0.0
+        val error = sqrt(error1 * error1 + error2 * error2)
+        valueRow(channel to channelValue, cr to value, crError to error)
+    }
+}

numass-analysis/src/commonMain/kotlin/ru/inr/mass/data/analysis/NumassGenerator.kt

@@ -0,0 +1,141 @@
+package ru.inr.mass.data.analysis
+//
+//import hep.dataforge.stat.defaultGenerator
+//import kotlinx.coroutines.flow.Flow
+//import kotlinx.coroutines.flow.takeWhile
+//import kotlinx.coroutines.flow.toList
+//import kotlinx.datetime.Instant
+//import org.apache.commons.math3.distribution.EnumeratedRealDistribution
+//import ru.inr.mass.data.analysis.NumassAnalyzer.Companion.CHANNEL_KEY
+//import ru.inr.mass.data.analysis.NumassAnalyzer.Companion.COUNT_RATE_KEY
+//import ru.inr.mass.data.api.NumassBlock
+//import ru.inr.mass.data.api.OrphanNumassEvent
+//import ru.inr.mass.data.api.SimpleBlock
+//import space.kscience.dataforge.tables.Table
+//import space.kscience.kmath.chains.Chain
+//import space.kscience.kmath.chains.MarkovChain
+//import space.kscience.kmath.chains.StatefulChain
+//import space.kscience.kmath.stat.RandomGenerator
+//import kotlin.time.Duration.Companion.nanoseconds
+//
+//private fun RandomGenerator.nextExp(mean: Double): Double {
+//    return -mean * ln(1.0 - nextDouble())
+//}
+//
+//private fun RandomGenerator.nextDeltaTime(cr: Double): Long {
+//    return (nextExp(1.0 / cr) * 1e9).toLong()
+//}
+//
+//public suspend fun Flow<OrphanNumassEvent>.generateBlock(start: Instant, length: Long): NumassBlock {
+//    return SimpleBlock.produce(start, length.nanoseconds) {
+//        takeWhile { it.timeOffset < length }.toList()
+//    }
+//}
+//
+//private class MergingState(private val chains: List<Chain<OrphanNumassEvent>>) {
+//    suspend fun poll(): OrphanNumassEvent {
+//        val next = chains.minBy { it.value.timeOffset } ?: chains.first()
+//        val res = next.value
+//        next.next()
+//        return res
+//    }
+//
+//}
+//
+///**
+// * Merge event chains in ascending time order
+// */
+//public fun List<Chain<OrphanNumassEvent>>.merge(): Chain<OrphanNumassEvent> {
+//    return StatefulChain(MergingState(this), OrphanNumassEvent(0.toUShort(), 0L)) {
+//        poll()
+//    }
+//}
+//
+///**
+// * Apply dead time based on event that caused it
+// */
+//public fun Chain<OrphanNumassEvent>.withDeadTime(deadTime: (OrphanNumassEvent) -> Long): Chain<OrphanNumassEvent> {
+//    return MarkovChain(this.value) {
+//        val start = this.value
+//        val dt = deadTime(start)
+//        do {
+//            val next = next()
+//        } while (next.timeOffset - start.timeOffset < dt)
+//        this.value
+//    }
+//}
+//
+//public object NumassGenerator {
+//
+//    public val defaultAmplitudeGenerator: RandomGenerator.(OrphanNumassEvent?, Long) -> Short =
+//        { _, _ -> ((nextDouble() + 2.0) * 100).toShort() }
+//
+//    /**
+//     * Generate an event chain with fixed count rate
+//     * @param cr = count rate in Hz
+//     * @param rnd = random number generator
+//     * @param amp amplitude generator for the chain. The receiver is rng, first argument is the previous event and second argument
+//     * is the delay between the next event. The result is the amplitude in channels
+//     */
+//    public fun generateEvents(
+//        cr: Double,
+//        rnd: RandomGenerator = defaultGenerator,
+//        amp: RandomGenerator.(OrphanNumassEvent?, Long) -> Short = defaultAmplitudeGenerator,
+//    ): Chain<OrphanNumassEvent> = MarkovChain(OrphanNumassEvent(rnd.amp(null, 0), 0)) { event ->
+//        val deltaT = rnd.nextDeltaTime(cr)
+//        OrphanNumassEvent(rnd.amp(event, deltaT), event.timeOffset + deltaT)
+//    }
+//
+//    public fun mergeEventChains(vararg chains: Chain<OrphanNumassEvent>): Chain<OrphanNumassEvent> =
+//        listOf(*chains).merge()
+//
+//
+//    private data class BunchState(var bunchStart: Long = 0, var bunchEnd: Long = 0)
+//
+//    /**
+//     * The chain of bunched events
+//     * @param cr count rate of events inside bunch
+//     * @param bunchRate number of bunches per second
+//     * @param bunchLength the length of bunch
+//     */
+//    public fun generateBunches(
+//        cr: Double,
+//        bunchRate: Double,
+//        bunchLength: Double,
+//        rnd: RandomGenerator = defaultGenerator,
+//        amp: RandomGenerator.(OrphanNumassEvent?, Long) -> Short = defaultAmplitudeGenerator,
+//    ): Chain<OrphanNumassEvent> {
+//        return StatefulChain(
+//            BunchState(0, 0),
+//            OrphanNumassEvent(rnd.amp(null, 0), 0)) { event ->
+//            if (event.timeOffset >= bunchEnd) {
+//                bunchStart = bunchEnd + rnd.nextDeltaTime(bunchRate)
+//                bunchEnd = bunchStart + (bunchLength * 1e9).toLong()
+//                OrphanNumassEvent(rnd.amp(null, 0), bunchStart)
+//            } else {
+//                val deltaT = rnd.nextDeltaTime(cr)
+//                OrphanNumassEvent(rnd.amp(event, deltaT), event.timeOffset + deltaT)
+//            }
+//        }
+//    }
+//
+//    /**
+//     * Generate a chain using provided spectrum for amplitudes
+//     */
+//    public fun generateEvents(
+//        cr: Double,
+//        rnd: RandomGenerator = defaultGenerator,
+//        spectrum: Table,
+//    ): Chain<OrphanNumassEvent> {
+//
+//        val channels = DoubleArray(spectrum.size())
+//        val values = DoubleArray(spectrum.size())
+//        for (i in 0 until spectrum.size()) {
+//            channels[i] = spectrum.get(CHANNEL_KEY, i).double
+//            values[i] = spectrum.get(COUNT_RATE_KEY, i).double
+//        }
+//        val distribution = EnumeratedRealDistribution(channels, values)
+//
+//        return generateEvents(cr, rnd) { _, _ -> distribution.sample().toShort() }
+//    }
+//}

numass-analysis/src/commonMain/kotlin/ru/inr/mass/data/analysis/SimpleAnalyzer.kt

@@ -0,0 +1,72 @@
+/*
+ * Copyright  2017 Alexander Nozik.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package inr.numass.data.analyzers
+
+import kotlinx.coroutines.flow.count
+import ru.inr.mass.data.analysis.AbstractAnalyzer
+import ru.inr.mass.data.analysis.NumassAnalyzerResult
+import ru.inr.mass.data.api.NumassBlock
+import ru.inr.mass.data.api.SignalProcessor
+import space.kscience.dataforge.meta.Meta
+import space.kscience.dataforge.meta.descriptors.MetaDescriptor
+import space.kscience.dataforge.meta.descriptors.value
+import space.kscience.dataforge.meta.double
+import space.kscience.dataforge.meta.get
+import space.kscience.dataforge.meta.int
+import space.kscience.dataforge.values.ValueType
+import kotlin.math.sqrt
+
+/**
+ * A simple event counter
+ * Created by darksnake on 07.07.2017.
+ */
+public class SimpleAnalyzer(processor: SignalProcessor? = null) : AbstractAnalyzer(processor) {
+
+    override val descriptor: MetaDescriptor = MetaDescriptor {
+        value("deadTime", ValueType.NUMBER) {
+            info = "Dead time in nanoseconds for correction"
+            default(0.0)
+        }
+    }
+
+    override suspend fun analyze(block: NumassBlock, config: Meta): NumassAnalyzerResult {
+        val loChannel = config["window.lo"]?.int ?: 0
+        val upChannel = config["window.up"]?.int ?: Int.MAX_VALUE
+
+        val count: Int = getEvents(block, config).count()
+        val length: Double = block.getLength().inWholeNanoseconds.toDouble() / 1e9
+
+        val deadTime = config["deadTime"]?.double ?: 0.0
+
+        val countRate = if (deadTime > 0) {
+            val mu = count.toDouble() / length
+            mu / (1.0 - deadTime * 1e-9 * mu)
+        } else {
+            count.toDouble() / length
+        }
+        val countRateError = sqrt(count.toDouble()) / length
+
+        return NumassAnalyzerResult {
+            this.length = length.toLong()
+            this.count = count.toLong()
+            this.countRate = countRate
+            this.countRateError = countRateError
+            this.window = loChannel.toUInt().rangeTo(upChannel.toUInt())
+            //TODO NumassAnalyzer.timestamp to block.startTime
+        }
+    }
+}

numass-analysis/src/commonMain/kotlin/ru/inr/mass/data/analysis/SmartAnalyzer.kt

@@ -0,0 +1,104 @@
+///*
+// * Copyright  2017 Alexander Nozik.
+// *
+// * Licensed under the Apache License, Version 2.0 (the "License");
+// * you may not use this file except in compliance with the License.
+// * You may obtain a copy of the License at
+// *
+// *      http://www.apache.org/licenses/LICENSE-2.0
+// *
+// *  Unless required by applicable law or agreed to in writing, software
+// *  distributed under the License is distributed on an "AS IS" BASIS,
+// *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// *  See the License for the specific language governing permissions and
+// *  limitations under the License.
+// */
+//
+package ru.inr.mass.data.analysis
+
+import inr.numass.data.analyzers.SimpleAnalyzer
+import kotlinx.coroutines.flow.Flow
+import ru.inr.mass.data.api.NumassBlock
+import ru.inr.mass.data.api.NumassEvent
+import ru.inr.mass.data.api.NumassSet
+import ru.inr.mass.data.api.SignalProcessor
+import space.kscience.dataforge.meta.Meta
+import space.kscience.dataforge.meta.descriptors.MetaDescriptor
+import space.kscience.dataforge.meta.get
+import space.kscience.dataforge.meta.string
+import space.kscience.dataforge.tables.Table
+import space.kscience.dataforge.values.Value
+import space.kscience.dataforge.values.asValue
+import space.kscience.dataforge.values.setValue
+
+/**
+ * An analyzer dispatcher which uses different analyzer for different meta
+ * Created by darksnake on 11.07.2017.
+ */
+public open class SmartAnalyzer(processor: SignalProcessor? = null) : AbstractAnalyzer(processor) {
+    private val simpleAnalyzer = SimpleAnalyzer(processor)
+    private val debunchAnalyzer = DebunchAnalyzer(processor)
+    private val timeAnalyzer: NumassAnalyzer = TODO()// TimeAnalyzer(processor)
+
+    override val descriptor: MetaDescriptor? = null
+
+    private fun getAnalyzer(config: Meta): NumassAnalyzer = when (val type = config["type"]?.string) {
+        null -> if (config["t0"] != null) {
+            timeAnalyzer
+        } else {
+            simpleAnalyzer
+        }
+        "simple" -> simpleAnalyzer
+        "time" -> timeAnalyzer
+        "debunch" -> debunchAnalyzer
+        else -> throw IllegalArgumentException("Analyzer $type not found")
+    }
+
+    override suspend fun analyze(block: NumassBlock, config: Meta): NumassAnalyzerResult {
+        val analyzer = getAnalyzer(config)
+        val res = analyzer.analyze(block, config)
+        return NumassAnalyzerResult.read(res.meta).apply {
+            setValue(T0_KEY, 0.0.asValue())
+        }
+    }
+
+    override fun getEvents(block: NumassBlock, meta: Meta): Flow<NumassEvent> =
+        getAnalyzer(meta).getEvents(block, meta)
+
+
+    override suspend fun analyzeSet(set: NumassSet, config: Meta): Table<Value> {
+        return getAnalyzer(config).analyzeSet(set, config)
+//        fun Value.computeExpression(point: NumassPoint): Int {
+//            return when {
+//                this.type == ValueType.NUMBER -> this.int
+//                this.type == ValueType.STRING -> {
+//                    val exprParams = HashMap<String, Any>()
+//
+//                    exprParams["U"] = point.voltage
+//
+//                    ExpressionUtils.function(this.string, exprParams).toInt()
+//                }
+//                else -> error("Can't interpret $type as expression or number")
+//            }
+//        }
+//
+//        val lo = config.getValue("window.lo", 0)
+//        val up = config.getValue("window.up", Int.MAX_VALUE)
+//
+//        val format = getTableFormat(config)
+//
+//        return ListTable.Builder(format)
+//            .rows(set.points.map { point ->
+//                val newConfig = config.builder.apply {
+//                    setValue("window.lo", lo.computeExpression(point))
+//                    setValue("window.up", up.computeExpression(point))
+//                }
+//                analyzeParent(point, newConfig)
+//            })
+//            .build()
+    }
+
+    public companion object : SmartAnalyzer() {
+        public const val T0_KEY: String = "t0"
+    }
+}

numass-analysis/src/commonMain/kotlin/ru/inr/mass/data/analysis/TimeAnalyzer.kt

@@ -0,0 +1,300 @@
+///*
+// * Copyright  2017 Alexander Nozik.
+// *
+// * Licensed under the Apache License, Version 2.0 (the "License");
+// * you may not use this file except in compliance with the License.
+// * You may obtain a copy of the License at
+// *
+// *      http://www.apache.org/licenses/LICENSE-2.0
+// *
+// *  Unless required by applicable law or agreed to in writing, software
+// *  distributed under the License is distributed on an "AS IS" BASIS,
+// *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// *  See the License for the specific language governing permissions and
+// *  limitations under the License.
+// */
+//
+package ru.inr.mass.data.analysis
+//
+//import hep.dataforge.description.ValueDef
+//import hep.dataforge.description.ValueDefs
+//import hep.dataforge.meta.Meta
+//import hep.dataforge.tables.Adapters.*
+//import hep.dataforge.tables.TableFormat
+//import hep.dataforge.tables.TableFormatBuilder
+//import hep.dataforge.values.*
+//import ru.inr.mass.data.analysis.NumassAnalyzer.Companion.COUNT_KEY
+//import ru.inr.mass.data.analysis.NumassAnalyzer.Companion.COUNT_RATE_ERROR_KEY
+//import ru.inr.mass.data.analysis.NumassAnalyzer.Companion.COUNT_RATE_KEY
+//import ru.inr.mass.data.analysis.NumassAnalyzer.Companion.LENGTH_KEY
+//import ru.inr.mass.data.analysis.TimeAnalyzer.AveragingMethod.*
+//import inr.numass.data.api.*
+//import inr.numass.data.api.NumassPoint.Companion.HV_KEY
+//import ru.inr.mass.data.api.NumassBlock
+//import ru.inr.mass.data.api.SignalProcessor
+//import space.kscience.dataforge.values.ValueType
+//import java.util.*
+//import java.util.concurrent.atomic.AtomicLong
+//import kotlin.collections.List
+//import kotlin.collections.asSequence
+//import kotlin.collections.count
+//import kotlin.collections.first
+//import kotlin.collections.map
+//import kotlin.collections.set
+//import kotlin.collections.sortBy
+//import kotlin.collections.sumBy
+//import kotlin.collections.sumByDouble
+//import kotlin.collections.toMutableList
+//import kotlin.math.*
+//import kotlin.streams.asSequence
+//
+//
+///**
+// * An analyzer which uses time information from events
+// * Created by darksnake on 11.07.2017.
+// */
+//@ValueDefs(
+//    ValueDef(
+//        key = "separateParallelBlocks",
+//        type = [ValueType.BOOLEAN],
+//        info = "If true, then parallel blocks will be forced to be evaluated separately"
+//    ),
+//    ValueDef(
+//        key = "chunkSize",
+//        type = [ValueType.NUMBER],
+//        def = "-1",
+//        info = "The number of events in chunk to split the chain into. If negative, no chunks are used"
+//    )
+//)
+//open class TimeAnalyzer(processor: SignalProcessor? = null) : AbstractAnalyzer(processor) {
+//
+//    override fun analyze(block: NumassBlock, config: Meta): Values {
+//        //In case points inside points
+//        if (block is ParentBlock && (block.isSequential || config.getBoolean("separateParallelBlocks", false))) {
+//            return analyzeParent(block, config)
+//        }
+//
+//        val t0 = getT0(block, config).toLong()
+//
+//        val chunkSize = config.getInt("chunkSize", -1)
+//
+//        val count = super.getEvents(block, config).count()
+//        val length = block.length.toNanos().toDouble() / 1e9
+//
+//        val res = when {
+//            count < 1000 -> ValueMap.ofPairs(
+//                LENGTH_KEY to length,
+//                COUNT_KEY to count,
+//                COUNT_RATE_KEY to count.toDouble() / length,
+//                COUNT_RATE_ERROR_KEY to sqrt(count.toDouble()) / length
+//            )
+//            chunkSize > 0 -> getEventsWithDelay(block, config)
+//                .chunked(chunkSize) { analyzeSequence(it.asSequence(), t0) }
+//                .toList()
+//                .mean(config.getEnum("mean", WEIGHTED))
+//            else -> analyzeSequence(getEventsWithDelay(block, config), t0)
+//        }
+//
+//        return ValueMap.Builder(res)
+//            .putValue("blockLength", length)
+//            .putValue(NumassAnalyzer.WINDOW_KEY, config.getRange())
+//            .putValue(NumassAnalyzer.TIME_KEY, block.startTime)
+//            .putValue(T0_KEY, t0.toDouble() / 1000.0)
+//            .build()
+//    }
+//
+//
+//    private fun analyzeSequence(sequence: Sequence<Pair<NumassEvent, Long>>, t0: Long): Values {
+//        val totalN = AtomicLong(0)
+//        val totalT = AtomicLong(0)
+//        sequence.filter { pair -> pair.second >= t0 }
+//            .forEach { pair ->
+//                totalN.incrementAndGet()
+//                //TODO add progress listener here
+//                totalT.addAndGet(pair.second)
+//            }
+//
+//        if (totalN.toInt() == 0) {
+//            error("Zero number of intervals")
+//        }
+//
+//        val countRate =
+//            1e6 * totalN.get() / (totalT.get() / 1000 - t0 * totalN.get() / 1000)//1e9 / (totalT.get() / totalN.get() - t0);
+//        val countRateError = countRate / sqrt(totalN.get().toDouble())
+//        val length = totalT.get() / 1e9
+//        val count = (length * countRate).toLong()
+//
+//        return ValueMap.ofPairs(
+//            LENGTH_KEY to length,
+//            COUNT_KEY to count,
+//            COUNT_RATE_KEY to countRate,
+//            COUNT_RATE_ERROR_KEY to countRateError
+//        )
+//
+//    }
+//
+//    override fun analyzeParent(point: ParentBlock, config: Meta): Values {
+//        //Average count rates, do not sum events
+//        val res = point.blocks.map { it -> analyze(it, config) }
+//
+//        val map = HashMap(res.mean(config.getEnum("mean", WEIGHTED)).asMap())
+//        if (point is NumassPoint) {
+//            map[HV_KEY] = Value.of(point.voltage)
+//        }
+//        return ValueMap(map)
+//    }
+//
+//    enum class AveragingMethod {
+//        ARITHMETIC,
+//        WEIGHTED,
+//        GEOMETRIC
+//    }
+//
+//    /**
+//     * Combine multiple blocks from the same point into one
+//     *
+//     * @return
+//     */
+//    private fun List<Values>.mean(method: AveragingMethod): Values {
+//
+//        if (this.isEmpty()) {
+//            return ValueMap.Builder()
+//                .putValue(LENGTH_KEY, 0)
+//                .putValue(COUNT_KEY, 0)
+//                .putValue(COUNT_RATE_KEY, 0)
+//                .putValue(COUNT_RATE_ERROR_KEY, 0)
+//                .build()
+//        }
+//
+//        val totalTime = sumByDouble { it.getDouble(LENGTH_KEY) }
+//
+//        val (countRate, countRateDispersion) = when (method) {
+//            ARITHMETIC -> Pair(
+//                sumByDouble { it.getDouble(COUNT_RATE_KEY) } / size,
+//                sumByDouble { it.getDouble(COUNT_RATE_ERROR_KEY).pow(2.0) } / size / size
+//            )
+//            WEIGHTED -> Pair(
+//                sumByDouble { it.getDouble(COUNT_RATE_KEY) * it.getDouble(LENGTH_KEY) } / totalTime,
+//                sumByDouble { (it.getDouble(COUNT_RATE_ERROR_KEY) * it.getDouble(LENGTH_KEY) / totalTime).pow(2.0) }
+//            )
+//            GEOMETRIC -> {
+//                val mean = exp(sumByDouble { ln(it.getDouble(COUNT_RATE_KEY)) } / size)
+//                val variance = (mean / size).pow(2.0) * sumByDouble {
+//                    (it.getDouble(COUNT_RATE_ERROR_KEY) / it.getDouble(
+//                        COUNT_RATE_KEY
+//                    )).pow(2.0)
+//                }
+//                Pair(mean, variance)
+//            }
+//        }
+//
+//        return ValueMap.Builder(first())
+//            .putValue(LENGTH_KEY, totalTime)
+//            .putValue(COUNT_KEY, sumBy { it.getInt(COUNT_KEY) })
+//            .putValue(COUNT_RATE_KEY, countRate)
+//            .putValue(COUNT_RATE_ERROR_KEY, sqrt(countRateDispersion))
+//            .build()
+//    }
+//
+//    @ValueDefs(
+//        ValueDef(key = "t0", type = arrayOf(ValueType.NUMBER), info = "Constant t0 cut"),
+//        ValueDef(
+//            key = "t0.crFraction",
+//            type = arrayOf(ValueType.NUMBER),
+//            info = "The relative fraction of events that should be removed by time cut"
+//        ),
+//        ValueDef(key = "t0.min", type = arrayOf(ValueType.NUMBER), def = "0", info = "Minimal t0")
+//    )
+//    protected fun getT0(block: NumassBlock, meta: Meta): Int {
+//        return if (meta.hasValue("t0")) {
+//            meta.getInt("t0")
+//        } else if (meta.hasMeta("t0")) {
+//            val fraction = meta.getDouble("t0.crFraction")
+//            val cr = estimateCountRate(block)
+//            if (cr < meta.getDouble("t0.minCR", 0.0)) {
+//                0
+//            } else {
+//                max(-1e9 / cr * ln(1.0 - fraction), meta.getDouble("t0.min", 0.0)).toInt()
+//            }
+//        } else {
+//            0
+//        }
+//
+//    }
+//
+//    private fun estimateCountRate(block: NumassBlock): Double {
+//        return block.events.count().toDouble() / block.length.toMillis() * 1000
+//    }
+//
+//    fun zipEvents(block: NumassBlock, config: Meta): Sequence<Pair<NumassEvent, NumassEvent>> {
+//        return getAllEvents(block).asSequence().zipWithNext()
+//    }
+//
+//    /**
+//     * The chain of event with delays in nanos
+//     *
+//     * @param block
+//     * @param config
+//     * @return
+//     */
+//    fun getEventsWithDelay(block: NumassBlock, config: Meta): Sequence<Pair<NumassEvent, Long>> {
+//        val inverted = config.getBoolean("inverted", true)
+//        //range is included in super.getEvents
+//        val events = super.getEvents(block, config).toMutableList()
+//
+//        if (config.getBoolean("sortEvents", false) || (block is ParentBlock && !block.isSequential)) {
+//            //sort in place if needed
+//            events.sortBy { it.timeOffset }
+//        }
+//
+//        return events.asSequence().zipWithNext { prev, next ->
+//            val delay = max(next.timeOffset - prev.timeOffset, 0)
+//            if (inverted) {
+//                Pair(next, delay)
+//            } else {
+//                Pair(prev, delay)
+//            }
+//        }
+//    }
+//
+//    /**
+//     * The filtered stream of events
+//     *
+//     * @param block
+//     * @param meta
+//     * @return
+//     */
+//    override fun getEvents(block: NumassBlock, meta: Meta): List<NumassEvent> {
+//        val t0 = getT0(block, meta).toLong()
+//        return getEventsWithDelay(block, meta)
+//            .filter { pair -> pair.second >= t0 }
+//            .map { it.first }.toList()
+//    }
+//
+//    public override fun getTableFormat(config: Meta): TableFormat {
+//        return TableFormatBuilder()
+//            .addNumber(HV_KEY, X_VALUE_KEY)
+//            .addNumber(LENGTH_KEY)
+//            .addNumber(COUNT_KEY)
+//            .addNumber(COUNT_RATE_KEY, Y_VALUE_KEY)
+//            .addNumber(COUNT_RATE_ERROR_KEY, Y_ERROR_KEY)
+//            .addColumn(NumassAnalyzer.WINDOW_KEY)
+//            .addTime()
+//            .addNumber(T0_KEY)
+//            .build()
+//    }
+//
+//    companion object {
+//        const val T0_KEY = "t0"
+//
+//        val NAME_LIST = arrayOf(
+//            LENGTH_KEY,
+//            COUNT_KEY,
+//            COUNT_RATE_KEY,
+//            COUNT_RATE_ERROR_KEY,
+//            NumassAnalyzer.WINDOW_KEY,
+//            NumassAnalyzer.TIME_KEY,
+//            T0_KEY
+//        )
+//    }
+//}

numass-analysis/src/commonMain/kotlin/ru/inr/mass/data/analysis/Values.kt

@@ -0,0 +1,5 @@
+package ru.inr.mass.data.analysis
+
+import space.kscience.dataforge.values.Value
+
+public typealias Values = Map<String, Value>

numass-data-model/build.gradle.kts

@@ -1,11 +1,8 @@
 plugins {
-    kotlin("multiplatform")
+    id("ru.mipt.npm.gradle.mpp")
-    id("ru.mipt.npm.gradle.common")
+    `maven-publish`
 }
 
-kscience {
-    publish()
-}
 
 val dataforgeVersion: String by rootProject.extra
 

numass-data-model/src/commonMain/kotlin/ru/inr/mass/data/api/MetaBlock.kt

@@ -3,6 +3,7 @@ package ru.inr.mass.data.api
 import kotlinx.coroutines.flow.*
 import kotlinx.datetime.Instant
 import kotlin.time.Duration
+import kotlin.time.Duration.Companion.nanoseconds
 import kotlin.time.DurationUnit
 
 public interface ParentBlock : NumassBlock {
@@ -19,7 +20,7 @@ public interface ParentBlock : NumassBlock {
  * A block constructed from a set of other blocks. Internal blocks are not necessary subsequent. Blocks are automatically sorted.
  * Created by darksnake on 16.07.2017.
  */
-public class MetaBlock(private val blocks: List<NumassBlock>) : ParentBlock {
+public open class MetaBlock(protected val blocks: List<NumassBlock>) : ParentBlock {
 
     override fun flowBlocks(): Flow<NumassBlock> = blocks.asFlow()
 
@@ -27,7 +28,7 @@ public class MetaBlock(private val blocks: List<NumassBlock>) : ParentBlock {
         get() = blocks.first().startTime
 
     override suspend fun getLength(): Duration =
-        Duration.nanoseconds(blocks.sumOf { it.getLength().toDouble(DurationUnit.NANOSECONDS) })
+        blocks.sumOf { it.getLength().toDouble(DurationUnit.NANOSECONDS) }.nanoseconds
 
     override val events: Flow<NumassEvent>
         get() = flow {
@@ -37,5 +38,8 @@ public class MetaBlock(private val blocks: List<NumassBlock>) : ParentBlock {
     override val frames: Flow<NumassFrame>
         get() = blocks.sortedBy { it.startTime }.asFlow().flatMapConcat { it.frames }
 
-
+    override val eventsCount: Long
+        get() = blocks.sumOf { it.eventsCount }
+    override val framesCount: Long
+        get() = blocks.sumOf { it.framesCount }
 }

numass-data-model/src/commonMain/kotlin/ru/inr/mass/data/api/NumassBlock.kt

@@ -25,10 +25,10 @@ import kotlinx.datetime.plus
 import kotlin.time.Duration
 
 public open class OrphanNumassEvent(
-    public val amplitude: Short,
+    public val amplitude: UShort,
     public val timeOffset: Long,
 ) : Comparable<OrphanNumassEvent> {
-    public operator fun component1(): Short = amplitude
+    public operator fun component1(): UShort = amplitude
     public operator fun component2(): Long = timeOffset
 
     override fun compareTo(other: OrphanNumassEvent): Int {
@@ -46,7 +46,7 @@ public open class OrphanNumassEvent(
  *
  */
 public class NumassEvent(
-    amplitude: Short,
+    amplitude: UShort,
     timeOffset: Long,
     public val owner: NumassBlock,
 ) : OrphanNumassEvent(amplitude, timeOffset)
@@ -74,11 +74,15 @@ public interface NumassBlock {
      */
     public suspend fun getLength(): Duration
 
+    public val eventsCount: Long
+
     /**
      * Stream of isolated events. Could be empty
      */
     public val events: Flow<NumassEvent>
 
+    public val framesCount: Long
+
     /**
      * Stream of frames. Could be empty
      */
@@ -109,6 +113,9 @@ public class SimpleBlock(
 
     override val events: Flow<NumassEvent> get() = eventList.asFlow()
 
+    override val eventsCount: Long get() = eventList.size.toLong()
+    override val framesCount: Long get() = 0L
+
     public companion object {
 
     }

numass-data-model/src/commonMain/kotlin/ru/inr/mass/data/api/NumassPoint.kt

@@ -16,18 +16,20 @@
 
 package ru.inr.mass.data.api
 
+import kotlinx.coroutines.FlowPreview
 import kotlinx.coroutines.flow.*
 import space.kscience.dataforge.meta.Meta
 import space.kscience.dataforge.meta.double
 import space.kscience.dataforge.meta.get
 import space.kscience.dataforge.meta.int
 import kotlin.time.Duration
+import kotlin.time.Duration.Companion.nanoseconds
 import kotlin.time.DurationUnit
-import kotlin.time.nanoseconds
 
 /**
  * Created by darksnake on 06-Jul-17.
  */
+@OptIn(FlowPreview::class)
 public interface NumassPoint : ParentBlock {
 
     public val meta: Meta
@@ -57,8 +59,8 @@ public interface NumassPoint : ParentBlock {
     /**
      * Get the length key of meta or calculate length as a sum of block lengths. The latter could be a bit slow
      */
-    override suspend fun getLength(): Duration =
+    override suspend fun getLength(): Duration = flowBlocks().filter { it.channel == 0 }.toList()
-        flowBlocks().filter { it.channel == 0 }.toList().sumOf { it.getLength().toDouble(DurationUnit.NANOSECONDS) }.nanoseconds
+        .sumOf { it.getLength().toLong(DurationUnit.NANOSECONDS) }.nanoseconds
 
     /**
      * Get all events it all blocks as a single sequence

numass-data-model/src/commonMain/kotlin/ru/inr/mass/data/api/NumassSet.kt

@@ -10,7 +10,6 @@ import space.kscience.dataforge.meta.Meta
 import space.kscience.dataforge.meta.get
 import space.kscience.dataforge.meta.long
 import space.kscience.dataforge.names.Name
-import space.kscience.dataforge.names.toName
 import space.kscience.dataforge.provider.Provider
 
 /**
@@ -35,18 +34,14 @@ public interface NumassSet : Iterable<NumassPoint>, Provider {
 
     //suspend fun getHvData(): Table?
 
-    override fun iterator(): Iterator<NumassPoint> {
+    override fun iterator(): Iterator<NumassPoint> = points.iterator()
-        return points.iterator()
-    }
 
     override val defaultTarget: String get() = NUMASS_POINT_TARGET
 
-    override fun content(target: String): Map<Name, Any> {
+    override fun content(target: String): Map<Name, Any> = if (target == NUMASS_POINT_TARGET) {
-        return if (target == NUMASS_POINT_TARGET) {
+        points.associateBy { Name.parse("point[${it.voltage}]") }
-            points.associateBy { "point[${it.voltage}]".toName() }
+    } else {
-        } else {
+        super.content(target)
-            super.content(target)
-        }
     }
 
     public companion object {

numass-data-model/src/commonMain/kotlin/ru/inr/mass/data/api/SimpleNumassPoint.kt

@@ -1,26 +0,0 @@
-package ru.inr.mass.data.api
-
-import kotlinx.coroutines.flow.Flow
-import kotlinx.coroutines.flow.asFlow
-import kotlinx.datetime.Instant
-import space.kscience.dataforge.meta.Meta
-
-/**
- * A simple static implementation of NumassPoint
- * Created by darksnake on 08.07.2017.
- */
-public class SimpleNumassPoint(
-    private val blocks: List<NumassBlock>,
-    override val meta: Meta,
-    override val startTime: Instant = Instant.DISTANT_PAST,
-    override val sequential: Boolean = true,
-) : NumassPoint {
-
-    init {
-        check(blocks.isNotEmpty()) { "No blocks in a point" }
-    }
-
-    override fun flowBlocks(): Flow<NumassBlock> = blocks.asFlow()
-
-    override fun toString(): String = "SimpleNumassPoint(index = ${index}, hv = $voltage)"
-}

numass-data-proto/build.gradle.kts

@@ -2,10 +2,7 @@ plugins {
     kotlin("jvm")
     id("ru.mipt.npm.gradle.common")
     id("com.squareup.wire") version "3.5.0"
-}
+    `maven-publish`
-
-kscience{
-    publish()
 }
 
 val dataforgeVersion: String by rootProject.extra

numass-data-proto/src/main/kotlin/ru/inr/mass/data/proto/ProtoNumassPoint.kt

@@ -32,6 +32,8 @@ import java.io.ByteArrayOutputStream
 import java.io.InputStream
 import java.util.zip.Inflater
 import kotlin.time.Duration
+import kotlin.time.Duration.Companion.milliseconds
+import kotlin.time.Duration.Companion.nanoseconds
 
 /**
  * Protobuf based numass point
@@ -42,7 +44,7 @@ internal class ProtoNumassPoint(
     private val protoBuilder: () -> Point,
 ) : NumassPoint {
 
-    val point by lazy(protoBuilder)
+    val point: Point by lazy(protoBuilder)
 
     override fun flowBlocks() = point.channels.flatMap { channel ->
         channel.blocks
@@ -65,9 +67,23 @@ internal class ProtoNumassPoint(
         } ?: Instant.DISTANT_PAST
 
     override suspend fun getLength(): Duration = meta["acquisition_time"].double?.let {
-        Duration.milliseconds(it * 1000)
+        (it * 1000).milliseconds
     } ?: super.getLength()
 
+    override val eventsCount: Long
+        get() = point.channels.sumOf { channel ->
+            channel.blocks.sumOf { block ->
+                block.events?.amplitudes?.size ?: 0
+            }.toLong()
+        }
+
+    override val framesCount: Long
+        get() = point.channels.sumOf { channel ->
+            channel.blocks.sumOf { block ->
+                block.frames.size ?: 0
+            }.toLong()
+        }
+
     override fun toString(): String = "ProtoNumassPoint(index = ${index}, hv = $voltage)"
 
     public companion object {
@@ -128,15 +144,15 @@ public class ProtoNumassBlock(
         }
 
     override suspend fun getLength(): Duration = when {
-        block.length > 0 -> Duration.nanoseconds(block.length)
+        block.length > 0 -> block.length.nanoseconds
-        parent?.meta["acquisition_time"] != null ->
+        parent?.meta?.get("acquisition_time") != null ->
-            Duration.milliseconds((parent?.meta["acquisition_time"].double ?: 0.0 * 1000))
+            (parent.meta["acquisition_time"].double ?: (0.0 * 1000)).milliseconds
         else -> {
             LoggerFactory.getLogger(javaClass)
                 .error("No length information on block. Trying to infer from first and last events")
             val times = runBlocking { events.map { it.timeOffset }.toList() }
             val nanos = (times.maxOrNull()!! - times.minOrNull()!!)
-            Duration.nanoseconds(nanos)
+            nanos.nanoseconds
         }
     }
 
@@ -152,7 +168,7 @@ public class ProtoNumassBlock(
             }
 
             amplitudes.zip(times) { amp, time ->
-                NumassEvent(amp.toShort(), time, this)
+                NumassEvent(amp.toUShort(), time, this)
             }.asFlow()
 
         } else {
@@ -170,11 +186,14 @@ public class ProtoNumassBlock(
 
     override val frames: Flow<NumassFrame>
         get() {
-            val tickSize = Duration.nanoseconds(block.bin_size)
+            val tickSize = block.bin_size.nanoseconds
             return block.frames.asFlow().map { frame ->
                 val time = startTime.plus(frame.time, DateTimeUnit.NANOSECOND)
                 val frameData = frame.data_
                 NumassFrame(time, tickSize, frameData.toShortArray())
             }
         }
+
+    override val eventsCount: Long get() = block.frames.size.toLong()
+    override val framesCount: Long get() = block.events?.amplitudes?.size?.toLong() ?: 0L
 }

numass-data-proto/src/main/kotlin/ru/inr/mass/data/proto/TaggedNumassEnvelopeFormat.kt

@@ -24,7 +24,6 @@ import space.kscience.dataforge.meta.get
 import space.kscience.dataforge.meta.string
 import space.kscience.dataforge.names.Name
 import space.kscience.dataforge.names.plus
-import space.kscience.dataforge.names.toName
 import java.util.*
 
 
@@ -113,7 +112,7 @@ internal class TaggedNumassEnvelopeFormat(private val io: IOPlugin) : EnvelopeFo
         override fun invoke(meta: Meta, context: Context): EnvelopeFormat {
             val io = context.io
 
-            val metaFormatName = meta["name"].string?.toName() ?: JsonMetaFormat.name
+            val metaFormatName = meta["name"].string?.let(Name::parse) ?: JsonMetaFormat.name
             //Check if appropriate factory exists
             io.metaFormatFactories.find { it.name == metaFormatName } ?: error("Meta format could not be resolved")
 

numass-data-proto/src/test/kotlin/ru/inr/mass/data/proto/TestNumassDirectory.kt

@@ -4,7 +4,6 @@ import org.junit.jupiter.api.Test
 import space.kscience.dataforge.context.Context
 import space.kscience.dataforge.meta.get
 import space.kscience.dataforge.meta.string
-import space.kscience.dataforge.meta.value
 import space.kscience.dataforge.values.ListValue
 import java.nio.file.Path
 import kotlin.test.assertEquals
@@ -19,7 +18,7 @@ class TestNumassDirectory {
         val dataPath = Path.of("src/test/resources", "testData/set_4")
         val testSet = context.readNumassDirectory(dataPath)
         assertEquals("2018-04-13T22:01:46", testSet.meta["end_time"].string)
-        assertEquals(ListValue.EMPTY, testSet.meta["comments"].value)
+        assertEquals(ListValue.EMPTY, testSet.meta["comments"]?.value)
         assertEquals(31, testSet.points.size)
         val point22 = testSet.points.find { it.index == 22 }!!
         point22.flowBlocks()

numass-workspace/build.gradle.kts

@@ -16,6 +16,7 @@ val kmathVersion: String by rootProject.extra
 dependencies {
     implementation(project(":numass-data-proto"))
     implementation(project(":numass-model"))
+    implementation(project(":numass-analysis"))
     implementation("space.kscience:dataforge-workspace:$dataforgeVersion")
     implementation("space.kscience:plotlykt-core:$plotlyVersion")
     implementation("space.kscience:kmath-histograms:$kmathVersion")

numass-workspace/src/main/kotlin/ru/inr/mass/scripts/run_2020_12.kt

@@ -5,13 +5,13 @@ import ru.inr.mass.data.proto.NumassDirectorySet
 import ru.inr.mass.workspace.readNumassRepository
 import space.kscience.dataforge.data.DataTree
 import space.kscience.dataforge.data.filter
-import space.kscience.dataforge.meta.get
 import space.kscience.dataforge.meta.string
 
 suspend fun main() {
     val repo: DataTree<NumassDirectorySet> = readNumassRepository("D:\\Work\\Numass\\data\\2018_04")
     val filtered = repo.filter { _, data ->
-        data.meta["operator"].string?.startsWith("Vas") ?: false
+        val operator by data.meta.string()
+        operator?.startsWith("Vas") ?: false
     }
 
     filtered.flow().collect {

numass-workspace/src/main/kotlin/ru/inr/mass/workspace/NumassPlugin.kt

@@ -0,0 +1,70 @@
+package ru.inr.mass.workspace
+
+import ru.inr.mass.data.analysis.SmartAnalyzer
+import ru.inr.mass.data.api.NumassSet
+import ru.inr.mass.data.proto.NumassProtoPlugin
+import space.kscience.dataforge.context.Context
+import space.kscience.dataforge.context.PluginFactory
+import space.kscience.dataforge.context.PluginTag
+import space.kscience.dataforge.data.select
+import space.kscience.dataforge.meta.Meta
+import space.kscience.dataforge.meta.boolean
+import space.kscience.dataforge.meta.descriptors.MetaDescriptor
+import space.kscience.dataforge.meta.descriptors.value
+import space.kscience.dataforge.meta.get
+import space.kscience.dataforge.meta.toMutableMeta
+import space.kscience.dataforge.names.Name
+import space.kscience.dataforge.tables.Table
+import space.kscience.dataforge.values.Value
+import space.kscience.dataforge.values.ValueType
+import space.kscience.dataforge.workspace.WorkspacePlugin
+import space.kscience.dataforge.workspace.pipeFrom
+import space.kscience.dataforge.workspace.task
+import kotlin.reflect.KClass
+
+class NumassPlugin : WorkspacePlugin() {
+    override val tag: PluginTag get() = Companion.tag
+
+    val numassProtoPlugin by require(NumassProtoPlugin)
+
+    val select by task<NumassSet>(
+        descriptor = MetaDescriptor {
+            info = "Select data from workspace data pool"
+            value("forward", ValueType.BOOLEAN) {
+                info = "Select only forward or only backward sets"
+            }
+        }
+    ) {
+        val forward = meta["forward"]?.boolean
+        val filtered = workspace.data.select<NumassSet> { _, meta ->
+            when (forward) {
+                true -> meta["iteration_info.reverse"]?.boolean?.not() ?: false
+                false -> meta["iteration_info.reverse"]?.boolean ?: false
+                else -> true
+            }
+        }
+
+        emit(Name.EMPTY, filtered)
+    }
+
+    val analyze by task<Table<Value>>(
+        MetaDescriptor {
+            info = "Count the number of events for each voltage and produce a table with the results"
+        }
+    ) {
+        pipeFrom(select) { set, name, meta ->
+            val res = SmartAnalyzer.analyzeSet(set, meta["analyzer"] ?: Meta.EMPTY)
+            val outputMeta = meta.toMutableMeta().apply {
+                "data" put set.meta
+            }
+           // context.output.render(res, stage = "numass.analyze", name = name, meta = outputMeta)
+            res
+        }
+    }
+
+    companion object : PluginFactory<NumassPlugin> {
+        override val tag: PluginTag = PluginTag("numass", "ru.mipt.npm")
+        override val type: KClass<out NumassPlugin> = NumassPlugin::class
+        override fun invoke(meta: Meta, context: Context): NumassPlugin = NumassPlugin()
+    }
+}

numass-workspace/src/main/kotlin/ru/inr/mass/workspace/workspace.kt

@@ -5,7 +5,6 @@ import space.kscience.dataforge.workspace.Workspace
 
 val NUMASS = Workspace {
     context{
-        name("NUMASS")
         plugin(NumassProtoPlugin)
     }
 }

settings.gradle.kts

@@ -2,13 +2,12 @@ pluginManagement {
     repositories {
         maven("https://repo.kotlin.link")
         mavenCentral()
-        jcenter()
         gradlePluginPortal()
         mavenLocal()
     }
 
-    val toolsVersion = "0.9.9"
+    val toolsVersion = "0.10.7"
-    val kotlinVersion = "1.5.0"
+    val kotlinVersion = "1.6.0"
 
     plugins {
         id("ru.mipt.npm.gradle.project") version toolsVersion
@@ -30,7 +29,10 @@ pluginManagement {
     }
 }
 
-include("numass-data-model")
+include(
-include("numass-data-proto")
+    ":numass-data-model",
-include("numass-workspace")
+    ":numass-analysis",
-include("numass-model")
+    ":numass-data-proto",
+    ":numass-workspace",
+    ":numass-model"
+)