Numass protobuf update. Plots fix

numass-core/src/main/java/inr/numass/data/analyzers/AbstractAnalyzer.java

@@ -40,7 +40,7 @@ public abstract class AbstractAnalyzer implements NumassAnalyzer {
         int loChannel = config.getInt("window.lo", 0);
         int upChannel = config.getInt("window.up", Integer.MAX_VALUE);
         if (block.getFrames().count() == 0) {
-            return block.getEvents().filter(it -> it.getChanel() >= loChannel && it.getChanel() <= upChannel);
+            return block.getEvents().filter(it -> it.getChanel() >= loChannel && it.getChanel() < upChannel);
         } else if (getProcessor() == null) {
             throw new IllegalArgumentException("Signal processor needed to analyze frames");
         } else {

numass-core/src/main/java/inr/numass/data/analyzers/TimeAnalyzer.java

@@ -9,10 +9,9 @@ import inr.numass.data.api.NumassPoint;
 import inr.numass.data.api.SignalProcessor;
 import org.jetbrains.annotations.Nullable;
 
-import java.time.Duration;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.atomic.AtomicReference;
-import java.util.stream.DoubleStream;
+import java.util.stream.LongStream;
 
 /**
  * An analyzer which uses time information from events
@@ -32,22 +31,22 @@ public class TimeAnalyzer extends AbstractAnalyzer {
         int loChannel = config.getInt("window.lo", 0);
         int upChannel = config.getInt("window.up", Integer.MAX_VALUE);
 
-        double t0 = config.getDouble("t0");
+        long t0 = config.getValue("t0").longValue();
 
         AtomicLong totalN = new AtomicLong(0);
-        AtomicReference<Double> totalT = new AtomicReference<>(0d);
+        AtomicLong totalT = new AtomicLong(0);
 
         timeChain(block, config).forEach(delay -> {
             if (delay >= t0) {
                 totalN.incrementAndGet();
                 //TODO add progress listener here
-                totalT.accumulateAndGet(delay, (d1, d2) -> d1 + d2);
+                totalT.addAndGet(delay);
             }
         });
 
-        double countRate = 1d / (totalT.get() / totalN.get() - t0);
+        double countRate = 1e6 * totalN.get() / (totalT.get()/1000 - t0 * totalN.get()/1000);//1e9 / (totalT.get() / totalN.get() - t0);
-        double countRateError = countRate/Math.sqrt(totalN.get());
+        double countRateError = countRate / Math.sqrt(totalN.get());
-        long count = (long) (countRate * totalT.get());
+        long count = (long) (totalT.get() * (countRate / 1e9));
         double length = totalT.get();
 
         if (block instanceof NumassPoint) {
@@ -76,22 +75,31 @@ public class TimeAnalyzer extends AbstractAnalyzer {
         }
     }
 
-    public DoubleStream timeChain(NumassBlock block, Meta config) {
+    /**
+     * The chain of event times in nanos
+     *
+     * @param block
+     * @param config
+     * @return
+     */
+    public LongStream timeChain(NumassBlock block, Meta config) {
         AtomicReference<NumassEvent> lastEvent = new AtomicReference<>(null);
-        return getEventStream(block, config).mapToDouble(event -> {
+        return getEventStream(block, config)
-            if (lastEvent.get() == null) {
+                .sorted()
-                lastEvent.set(event);
+                .mapToLong(event -> {
-                return 0d;
+                    if (lastEvent.get() == null) {
-            } else {
+                        lastEvent.set(event);
-                double res = Duration.between(lastEvent.get().getTime(),event.getTime()).toNanos();//event.getTimeOffset() - lastEvent.get().getTimeOffset();
+                        return 0;
-                if (res > 0) {
+                    } else {
-                    lastEvent.set(event);
+                        long res = event.getTimeOffset() - lastEvent.get().getTimeOffset();
-                    return res;
+                        if (res >= 0) {
-                } else {
+                            lastEvent.set(event);
-                    lastEvent.set(null);
+                            return res;
-                    return 0d;
+                        } else {
-                }
+                            lastEvent.set(null);
-            }
+                            return 0;
-        });
+                        }
+                    }
+                });
     }
 }

numass-core/src/main/java/inr/numass/data/api/NumassEvent.java

@@ -27,11 +27,11 @@ import java.time.Instant;
  */
 public class NumassEvent implements Comparable<NumassEvent>, Serializable {
     // channel
-    protected final short chanel;
+    private final short chanel;
     //The time of the block start
-    protected final Instant blockTime;
+    private final Instant blockTime;
     //time in nanoseconds relative to block start
-    protected final long timeOffset;
+    private final long timeOffset;
 
     public NumassEvent(short chanel, Instant blockTime, long offset) {
         this.chanel = chanel;
@@ -51,6 +51,7 @@ public class NumassEvent implements Comparable<NumassEvent>, Serializable {
     }
 
     /**
+     * time in nanoseconds relative to block start
      * @return the time
      */
     public long getTimeOffset() {

numass-core/src/main/java/inr/numass/data/storage/ClassicNumassPoint.java

@@ -38,7 +38,7 @@ public class ClassicNumassPoint implements NumassPoint {
         } else {
             length = envelope.meta().getValue("acquisition_time").longValue();
         }
-        return Stream.of(new ClassicBlock(getStartTime(), Duration.ofSeconds(length), 0));
+        return Stream.of(new ClassicBlock(getStartTime(), Duration.ofSeconds(length)));
     }
 
     @Override
@@ -64,12 +64,12 @@ public class ClassicNumassPoint implements NumassPoint {
     private class ClassicBlock implements NumassBlock, Iterable<NumassEvent> {
         private final Instant startTime;
         private final Duration length;
-        private final long blockOffset;
+//        private final long blockOffset;
 
-        public ClassicBlock(Instant startTime, Duration length, long blockOffset) {
+        public ClassicBlock(Instant startTime, Duration length) {
             this.startTime = startTime;
             this.length = length;
-            this.blockOffset = blockOffset;
+//            this.blockOffset = blockOffset;
         }
 
         @Override

numass-core/src/main/java/inr/numass/data/storage/ProtoNumassPoint.java

@@ -2,6 +2,7 @@ package inr.numass.data.storage;
 
 import hep.dataforge.io.envelopes.Envelope;
 import hep.dataforge.meta.Meta;
+import hep.dataforge.storage.filestorage.FileEnvelope;
 import inr.numass.data.NumassProto;
 import inr.numass.data.api.NumassBlock;
 import inr.numass.data.api.NumassEvent;
@@ -11,6 +12,7 @@ import inr.numass.data.api.NumassPoint;
 import java.io.IOException;
 import java.io.InputStream;
 import java.nio.ByteBuffer;
+import java.nio.file.Path;
 import java.time.Duration;
 import java.time.Instant;
 import java.util.stream.IntStream;
@@ -21,6 +23,11 @@ import java.util.stream.Stream;
  * Created by darksnake on 09.07.2017.
  */
 public class ProtoNumassPoint implements NumassPoint {
+    public static ProtoNumassPoint readFile(Path path) {
+        return new ProtoNumassPoint(FileEnvelope.open(path, true));
+    }
+
+
     private final Envelope envelope;
 
     public ProtoNumassPoint(Envelope envelope) {
@@ -88,7 +95,7 @@ public class ProtoNumassPoint implements NumassPoint {
 
         @Override
         public Stream<NumassFrame> getFrames() {
-            Duration tickSize = Duration.ofNanos((long) (1e9 / meta().getInt("sample_freq")));
+            Duration tickSize = Duration.ofNanos((long) (1e9 / meta().getInt("params.sample_freq")));
             return block.getFramesList().stream().map(frame -> {
                 Instant time = getStartTime().plusNanos(frame.getTime());
                 ByteBuffer data = frame.getData().asReadOnlyByteBuffer();

numass-main/src/main/groovy/inr/numass/data/PointAnalyzer.groovy

@@ -4,6 +4,7 @@ import groovy.transform.CompileStatic
 import hep.dataforge.grind.Grind
 import hep.dataforge.maths.histogram.Histogram
 import hep.dataforge.maths.histogram.UnivariateHistogram
+import hep.dataforge.values.Values
 import inr.numass.data.analyzers.TimeAnalyzer
 import inr.numass.data.api.NumassBlock
 
@@ -17,15 +18,19 @@ class PointAnalyzer {
 
     static TimeAnalyzer analyzer = new TimeAnalyzer();
 
-    static Histogram histogram(NumassBlock point, int loChannel = 0, int upChannel = 4000, double binSize = 1e-6d, int binNum = 500) {
+    static Histogram histogram(NumassBlock point, int loChannel = 0, int upChannel = 10000, double binSize = 0.5, int binNum = 500) {
         return UnivariateHistogram.buildUniform(0d, binSize * binNum, binSize)
-                .fill(analyzer.timeChain(point, Grind.buildMeta("window.lo": loChannel, "window.up": upChannel)))
+                .fill(analyzer.timeChain(point, Grind.buildMeta("window.lo": loChannel, "window.up": upChannel)).mapToDouble {it / 1000 as double})
     }
 
-    static Histogram histogram(DoubleStream stream, double binSize = 1e-6d, int binNum = 500) {
+    static Histogram histogram(DoubleStream stream, double binSize = 0.5, int binNum = 500) {
         return UnivariateHistogram.buildUniform(0d, binSize * binNum, binSize).fill(stream)
     }
 
+    static Values analyze(Map values = Collections.emptyMap(), NumassBlock block, Closure metaClosure = null) {
+        return analyzer.analyze(block, Grind.buildMeta(values, metaClosure))
+    }
+
     static class Result {
         double cr;
         double crErr;

numass-main/src/main/groovy/inr/numass/scripts/times/TestPointAnalyzer.groovy

@@ -8,6 +8,8 @@ import hep.dataforge.plots.fx.FXPlotManager
 import hep.dataforge.tables.ValueMap
 import inr.numass.NumassPlugin
 import inr.numass.data.PointAnalyzer
+import inr.numass.data.api.NumassPoint
+import inr.numass.data.api.NumassSet
 import inr.numass.data.storage.NumassStorage
 import inr.numass.data.storage.NumassStorageFactory
 
@@ -20,59 +22,63 @@ Context ctx = Global.instance()
 ctx.pluginManager().load(FXPlotManager)
 ctx.pluginManager().load(NumassPlugin.class)
 
-GrindShell shell = new GrindShell(ctx)
+new GrindShell(ctx).eval {
-
-shell.eval {
     PlotHelper plot = plots
-    File rootDir = new File("D:\\Work\\Numass\\data\\2017_05\\Fill_1")
+    File rootDir = new File("D:\\Work\\Numass\\data\\2017_05\\Fill_3")
 
     NumassStorage storage = NumassStorageFactory.buildLocal(rootDir);
 
-    def set = "set_1"
-    def hv = 18400;
 
-    def loChannel = 400;
+    def set = "set_1"
+    def hv = 14500;
+    def loader = storage.provide("loader::$set", NumassSet.class).get();
+    def point = loader.provide("$hv", NumassPoint.class).get()
+
+    def loChannel = 500;
     def upChannel = 2000;
 
-    def point = storage.provide("loader::$set/rawPoint::$hv", RawNMPoint.class).get();
+    def histogram = PointAnalyzer.histogram(point, loChannel, upChannel, 0.7, 1000).asTable();
-
-    def histogram = PointAnalyzer.histogram(point, loChannel, upChannel).asTable();
 
     println "finished histogram calculation..."
 
     plot.configure("histogram") {
+        xAxis(axisTitle: "delay", axisUnits: "us")
         yAxis(type: "log")
     }
 
-    plot.plot(name: hv, frame: "histogram", showLine: true, showSymbol: false, showErrors: false, connectionType: "step", histogram, {
+    plot.plot(name: "test", frame: "histogram", showLine: true, showSymbol: false, showErrors: false, connectionType: "step", histogram, {
         adapter("x.value": "x", "y.value": "count")
     })
 
-    def trueCR = PointAnalyzer.analyzePoint(point, 30e-6, loChannel, upChannel).cr
+    def trueCR = PointAnalyzer.analyze(point, t0: 30e3, "window.lo": loChannel, "window.up": upChannel).getDouble("cr")
 
     println "The expected count rate for 30 us delay is $trueCR"
 
-    def t0 = (1..150).collect { 5.5e-6 + 2e-7 * it }
+    def t0 = (1..150).collect { 500 * it }
+
+//    point.blocks.eachWithIndex { block, index ->
+//        def statPlotPoints = t0.collect {
+//            def result = PointAnalyzer.analyze(block, t0: it, "window.lo": loChannel, "window.up": upChannel)
+//            ValueMap.fromMap("x": it / 1000, "y": result.getDouble("cr"), "y.err": result.getDouble("cr.err"));
+//        }
+//        plot.plot(name: index, frame: "stat-method", showLine: true, statPlotPoints)
+//    }
 
     def statPlotPoints = t0.collect {
-        def result = PointAnalyzer.analyzePoint(point, it, loChannel, upChannel)
+        def result = PointAnalyzer.analyze(point, t0: it, "window.lo": loChannel, "window.up": upChannel)
-        ValueMap.fromMap("x.value": it, "y.value": result.cr, "y.err": result.crErr);
+        ValueMap.fromMap("x": it / 1000, "y": result.getDouble("cr"), "y.err": result.getDouble("cr.err"));
     }
-    //def cr = t0.collect { PointAnalyzer.analyzePoint(point, it).cr }
+    plot.plot(name: "total", frame: "stat-method", showLine: true, statPlotPoints)
 
-    plot.plot(name: hv, frame: "stat-method", statPlotPoints)
+//    def delta = 5e-6
-
+//    def discrepancyPlotPoints = (1..20).collect { delta * it }.collect {
-    def delta = 5e-6
+//        def t1 = it
-    def discrepancyPlotPoints = (1..20).collect { delta * it }.collect {
+//        def t2 = it + delta
-        def t1 = it
+//        def result = PointAnalyzer.count(point, t1, t2, loChannel, upChannel) - (Math.exp(-trueCR * t1) - Math.exp(-trueCR * t2)) * point.length * trueCR
-        def t2 = it + delta
+//        ValueMap.fromMap("x.value": it + delta / 2, "y.value": result);
-        def result = PointAnalyzer.count(point, t1, t2, loChannel, upChannel) - (Math.exp(- trueCR * t1) - Math.exp(- trueCR * t2)) * point.length * trueCR
+//    }
-        ValueMap.fromMap("x.value": it + delta / 2, "y.value": result);
+//
-    }
+//    plot.plot(name: hv, frame: "discrepancy", discrepancyPlotPoints)
-
-    plot.plot(name: hv, frame: "discrepancy", discrepancyPlotPoints)
-
-//    plot.plot(title: "dead time", from: 5.5e-6, to: 2e-5) { point.cr * 1d / (1d - 6.55e-6 * point.cr) }
 
 
     storage.close()

numass-main/src/main/groovy/inr/numass/scripts/times/TestProto.groovy

@@ -0,0 +1,77 @@
+package inr.numass.scripts.times
+
+import hep.dataforge.context.Context
+import hep.dataforge.context.Global
+import hep.dataforge.grind.GrindShell
+import hep.dataforge.grind.helpers.PlotHelper
+import hep.dataforge.plots.fx.FXPlotManager
+import hep.dataforge.tables.ValueMap
+import inr.numass.NumassPlugin
+import inr.numass.data.PointAnalyzer
+import inr.numass.data.api.NumassPoint
+import inr.numass.data.storage.ProtoNumassPoint
+
+import java.nio.file.Paths
+
+Context ctx = Global.instance()
+ctx.pluginManager().load(FXPlotManager)
+ctx.pluginManager().load(NumassPlugin.class)
+
+new GrindShell(ctx).eval {
+    PlotHelper plot = plots
+    NumassPoint point = ProtoNumassPoint.readFile(Paths.get("D:\\Work\\Numass\\data\\test\\40_kHz_5s.df"))
+
+    def loChannel = 0;
+    def upChannel = 10000;
+
+    def histogram = PointAnalyzer.histogram(point, loChannel, upChannel, 0.2, 1000).asTable();
+
+    println "finished histogram calculation..."
+
+    plot.configure("histogram") {
+        xAxis(axisTitle: "delay", axisUnits: "us")
+        yAxis(type: "log")
+    }
+
+    plot.plot(name: "test", frame: "histogram", showLine: true, showSymbol: false, showErrors: false, connectionType: "step", histogram, {
+        adapter("x.value": "x", "y.value": "count")
+    })
+
+    def trueCR = PointAnalyzer.analyze(point, t0: 30e3, "window.lo": loChannel, "window.up": upChannel).getDouble("cr")
+
+    println "The expected count rate for 30 us delay is $trueCR"
+
+    def t0 = (1..150).collect { 500 * it }
+
+//    point.blocks.eachWithIndex { block, index ->
+//        def statPlotPoints = t0.collect {
+//            def result = PointAnalyzer.analyze(block, t0: it, "window.lo": loChannel, "window.up": upChannel)
+//            ValueMap.fromMap("x": it / 1000, "y": result.getDouble("cr"), "y.err": result.getDouble("cr.err"));
+//        }
+//        plot.plot(name: index, frame: "stat-method", showLine: true, statPlotPoints)
+//    }
+
+    def statPlotPoints = t0.collect {
+        def result = PointAnalyzer.analyze(point, t0: it, "window.lo": loChannel, "window.up": upChannel)
+        ValueMap.fromMap("x": it / 1000, "y": result.getDouble("cr"), "y.err": result.getDouble("cr.err"));
+    }
+    plot.plot(name: "total", frame: "stat-method", showLine: true, thickness: 4, statPlotPoints)
+
+
+
+//    def delta = 5e-6
+//    def discrepancyPlotPoints = (1..20).collect { delta * it }.collect {
+//        def t1 = it
+//        def t2 = it + delta
+//        def result = PointAnalyzer.count(point, t1, t2, loChannel, upChannel) - (Math.exp(-trueCR * t1) - Math.exp(-trueCR * t2)) * point.length * trueCR
+//        ValueMap.fromMap("x.value": it + delta / 2, "y.value": result);
+//    }
+//
+//    plot.plot(name: hv, frame: "discrepancy", discrepancyPlotPoints)
+
+
+}
+
+
+
+

numass-viewer/src/main/kotlin/inr/numass/viewer/MainView.kt

@@ -41,7 +41,7 @@ class MainView : View("Numass data viewer") {
     private val slowControlView: SlowControlView by inject()
 
     private val consoleButton: ToggleButton by fxid()
-    private val processManagerButton: ToggleButton by fxid()
+//    private val processManagerButton: ToggleButton by fxid()
     private val loadDirectoryButton: Button by fxid()
     private val loadRemoteButton: Button by fxid()
     private val storagePathLabel: Label by fxid()
@@ -60,7 +60,7 @@ class MainView : View("Numass data viewer") {
 //        WorkManagerFragment(getWorkManager())
 //    }
 
-    private val storageProperty = SimpleObjectProperty<Storage>();
+    private val storageProperty = SimpleObjectProperty<Storage?>();
 
     init {
         loadDirectoryButton.action {
@@ -143,7 +143,10 @@ class MainView : View("Numass data viewer") {
             }
         }
 
-
+        primaryStage.setOnCloseRequest {
+            log.info("Closing storage")
+            storageProperty.get()?.close()
+        }
     }
 
     private fun loadDirectory(path: URI) {

numass-viewer/src/main/kotlin/inr/numass/viewer/NumassLoaderView.kt

@@ -316,7 +316,7 @@ class NumassLoaderView : View() {
         }
 
         runAsync {
-            detectorPlot.progressProperty().bind(progressProperty());
+            Platform.runLater { detectorPlot.progressProperty().bind(progressProperty()) }
             val totalCount = data.points.count();
             val index = AtomicInteger(0);
             data.points.map { point ->
@@ -328,7 +328,7 @@ class NumassLoaderView : View() {
                 ).apply {
                     configure(plottableConfig)
                 }.also {
-                    updateProgress(index.get() as Long, totalCount);
+                    updateProgress(index.get().toLong(), totalCount);
                 }
             }.collect(Collectors.toList())
         } ui { plots ->

numass-viewer/src/main/kotlin/inr/numass/viewer/Viewer.kt

@@ -21,4 +21,9 @@ class Viewer : App(MainView::class) {
         StorageManager().startGlobal()
         super.start(stage)
     }
+
+    override fun stop() {
+        super.stop()
+        Global.terminate();
+    }
 }