Numass fixes

numass-core/src/main/java/inr/numass/storage/NumassStorageFactory.java

@@ -1,11 +1,14 @@
 package inr.numass.storage;
 
 import hep.dataforge.context.Context;
+import hep.dataforge.context.Global;
 import hep.dataforge.meta.Meta;
 import hep.dataforge.meta.MetaBuilder;
 import hep.dataforge.storage.api.Storage;
 import hep.dataforge.storage.api.StorageType;
 
+import java.io.File;
+
 /**
  * Created by darksnake on 17-May-17.
  */
@@ -19,6 +22,16 @@ public class NumassStorageFactory implements StorageType {
                 .setValue("monitor", monitor);
     }
 
+    /**
+     * Build local storage with Global context. Used for tests.
+     * @param file
+     * @return
+     */
+    public static NumassStorage buildLocal(File file) {
+        return new NumassStorage(Global.instance(),
+                new MetaBuilder("storage").setValue("path", file.toURI()));
+    }
+
     @Override
     public String type() {
         return "numass";

numass-main/src/main/groovy/inr/numass/scripts/Underflow.groovy

@@ -13,13 +13,15 @@ import inr.numass.data.NMPoint
 import inr.numass.data.NumassData
 import inr.numass.data.NumassDataUtils
 import inr.numass.storage.NumassStorage
+import inr.numass.storage.NumassStorageFactory
 import inr.numass.utils.UnderflowCorrection
 
 //File rootDir = new File("D:\\Work\\Numass\\data\\2016_10\\Fill_1")
-File rootDir = new File("D:\\Work\\Numass\\data\\2016_10\\Fill_2_wide")
+//File rootDir = new File("D:\\Work\\Numass\\data\\2016_10\\Fill_2_wide")
 //File rootDir = new File("D:\\Work\\Numass\\data\\2017_01\\Fill_2_wide")
+File rootDir = new File("D:\\Work\\Numass\\data\\2017_05\\Fill_1")
 
-NumassStorage storage = NumassStorage.buildLocalNumassRoot(rootDir, true);
+NumassStorage storage = NumassStorageFactory.buildLocal(rootDir);
 
 Collection<NMPoint> data = NumassDataUtils.joinSpectra(
         StorageUtils.loaderStream(storage)
@@ -53,13 +55,14 @@ data = NumassDataUtils.substractReferencePoint(data, 18600d);
 //    }
 //}
 
-def printPoint(Iterable<NMPoint> data, List us, int binning = 20, normalize = true) {
+def printPoint(Iterable<NMPoint> data, List<Double> us, int binning = 20, normalize = true) {
     List<NMPoint> points = data.findAll { it.voltage in us }.sort { it.voltage }
 
     Map spectra = points.first().getMap(binning, normalize).collectEntries { key, value ->
         [key, [value]]
     };
 
+    print "channel"
     points.eachWithIndex { it, index ->
         print "\t${it.voltage}"
         it.getMap(binning, normalize).each { k, v ->
@@ -85,6 +88,6 @@ printPoint(data, [14000d, 14500d, 15000d, 15500d, 16500d])
 
 println()
 
-Table t = new UnderflowCorrection().fitAllPoints(data, 400, 600, 3100, 20);
+Table t = new UnderflowCorrection().fitAllPoints(data, 350, 550, 3100, 20);
 ColumnedDataWriter.writeTable(System.out, t, "underflow parameters")
 

numass-viewer/src/main/kotlin/inr/numass/viewer/MainView.kt

@@ -31,6 +31,7 @@ import javafx.util.Pair
 import org.controlsfx.control.StatusBar
 import tornadofx.*
 import java.io.File
+import java.net.URI
 import java.util.logging.Level
 
 /**
@@ -79,7 +80,7 @@ class MainView : View("Numass data viewer") {
 
             if (rootDir != null) {
                 NumassProperties.setNumassProperty("numass.storage.root", rootDir.absolutePath)
-                loadDirectory(rootDir.toURI().toString())
+                loadDirectory(rootDir.toURI())
             }
         }
         loadRemoteButton.action { onLoadRemote() }
@@ -144,7 +145,7 @@ class MainView : View("Numass data viewer") {
 
     }
 
-    private fun loadDirectory(path: String) {
+    private fun loadDirectory(path: URI) {
         getWorkManager().startWork("viewer.loadDirectory") { work: Work ->
             work.title = "Load storage ($path)"
             work.progress = -1.0
@@ -234,7 +235,7 @@ class MainView : View("Numass data viewer") {
         val result = dialog.showAndWait()
 
         if (result.isPresent) {
-            val path = result.get().key + "/data/" + result.get().value
+            val path = URI.create(result.get().key + "/data/" + result.get().value)
             loadDirectory(path)
         }
     }

numass-viewer/src/main/kotlin/inr/numass/viewer/NumassDataCache.kt

@@ -0,0 +1,49 @@
+package inr.numass.viewer
+
+import hep.dataforge.data.Data
+import hep.dataforge.meta.Meta
+import hep.dataforge.tables.Table
+import inr.numass.data.NumassData
+import inr.numass.data.NumassPoint
+import java.time.Instant
+import java.util.stream.Collectors
+import java.util.stream.Stream
+
+/**
+ * Cached numass data
+ * Created by darksnake on 23-Jun-17.
+ */
+class NumassDataCache(val data: NumassData) : NumassData {
+    private val cachedDescription: String by lazy { data.description }
+    private val cachedMeta: Meta by lazy { data.meta }
+    private val cachedPoints: List<NumassPoint> by lazy { data.stream().collect(Collectors.toList()) }
+    private val hv: Table by lazy { data.hvData.get() }
+
+    override fun getDescription(): String {
+        return cachedDescription
+    }
+
+    override fun meta(): Meta {
+        return cachedMeta
+    }
+
+    override fun stream(): Stream<NumassPoint> {
+        return cachedPoints.stream();
+    }
+
+    override fun isEmpty(): Boolean {
+        return data.isEmpty
+    }
+
+    override fun startTime(): Instant {
+        return data.startTime()
+    }
+
+    override fun getName(): String {
+        return data.name;
+    }
+
+    override fun getHVData(): Data<Table> {
+        return Data.buildStatic(hv);
+    }
+}

numass-viewer/src/main/kotlin/inr/numass/viewer/NumassLoaderView.kt

@@ -2,7 +2,6 @@ package inr.numass.viewer
 
 import hep.dataforge.context.Context
 import hep.dataforge.context.Global
-import hep.dataforge.data.Data
 import hep.dataforge.fx.work.WorkManager
 import hep.dataforge.io.ColumnedDataWriter
 import hep.dataforge.meta.MetaBuilder
@@ -11,15 +10,17 @@ import hep.dataforge.plots.data.PlotDataUtils
 import hep.dataforge.plots.data.PlottableData
 import hep.dataforge.plots.data.PlottableGroup
 import hep.dataforge.plots.data.TimePlottable
-import hep.dataforge.plots.fx.FXPlotFrame
 import hep.dataforge.plots.fx.PlotContainer
 import hep.dataforge.plots.jfreechart.JFreeChartFrame
 import hep.dataforge.storage.commons.JSONMetaWriter
-import hep.dataforge.tables.*
+import hep.dataforge.tables.DataPoint
+import hep.dataforge.tables.ListTable
+import hep.dataforge.tables.MapPoint
+import hep.dataforge.tables.XYAdapter
 import inr.numass.data.NumassData
 import inr.numass.data.NumassDataUtils
 import inr.numass.data.NumassPoint
-import javafx.application.Platform
+import javafx.beans.property.SimpleObjectProperty
 import javafx.beans.value.ObservableValue
 import javafx.collections.FXCollections
 import javafx.event.ActionEvent
@@ -67,11 +68,37 @@ class NumassLoaderView : View() {
     private val detectorNormalizeSwitch: CheckBox = CheckBox("Normailize")
     private val detectorDataExportButton: Button = Button("Export")
 
-    //plots data
+    val dataProperty = SimpleObjectProperty<NumassData>()
-    var data: NumassData? = null
+    var data: NumassData? by dataProperty
+
     val spectrumData = PlottableData("spectrum")
     val hvPlotData = PlottableGroup<TimePlottable>()
-    private var points = FXCollections.observableArrayList<NumassPoint>()
+    //private var points = FXCollections.observableArrayList<NumassPoint>()
+
+    val detectorPlotFrame = JFreeChartFrame(
+            MetaBuilder("frame")
+                    .setValue("title", "Detector response plot")
+                    .setNode(MetaBuilder("xAxis")
+                            .setValue("axisTitle", "ADC")
+                            .setValue("axisUnits", "channels")
+                            .build())
+                    .setNode(MetaBuilder("yAxis")
+                            .setValue("axisTitle", "count rate")
+                            .setValue("axisUnits", "Hz")
+                            .build())
+                    .setNode(MetaBuilder("legend")
+                            .setValue("show", false))
+                    .build()
+    )
+
+    val plottableConfig = MetaBuilder("plot")
+            .setValue("connectionType", "step")
+            .setValue("thickness", 2)
+            .setValue("showLine", true)
+            .setValue("showSymbol", false)
+            .setValue("showErrors", false)
+            .setValue("JFreeChart.cache", true)
+            .build()
 
 
     init {
@@ -94,6 +121,7 @@ class NumassLoaderView : View() {
         //setup spectrum pane
 
         spectrumExportButton.onAction = EventHandler { this.onSpectrumExportClick(it) }
+
         val spectrumPlotMeta = MetaBuilder("plot")
                 .setValue("xAxis.axisTitle", "U")
                 .setValue("xAxis.axisUnits", "V")
@@ -108,9 +136,30 @@ class NumassLoaderView : View() {
         channelSlider.highValue = 1900.0
         channelSlider.lowValue = 300.0
 
-        val rangeChangeListener = { _: ObservableValue<out Number>, _: Number, _: Number -> setupSpectrumPane(points) }
+        detectorBinningSelector.selectionModel.selectedItemProperty().addListener { observable, oldValue, newValue ->
+            if (data != null) {
+                updateDetectorPane(data!!)
+            }
+        }
 
-        dTimeField.textProperty().addListener { _: ObservableValue<out String>, _: String, _: String -> setupSpectrumPane(points) }
+        detectorNormalizeSwitch.selectedProperty().addListener { observable, oldValue, newValue ->
+            if (data != null) {
+                updateDetectorPane(data!!)
+            }
+        }
+
+
+        dTimeField.textProperty().addListener { _: ObservableValue<out String>, _: String, _: String ->
+            if (data != null) {
+                updateSpectrum(data!!)
+            }
+        }
+
+        val rangeChangeListener = { _: ObservableValue<out Number>, _: Number, _: Number ->
+            if (data != null) {
+                updateSpectrum(data!!)
+            }
+        }
 
         channelSlider.lowValueProperty().addListener(rangeChangeListener)
         channelSlider.highValueProperty().addListener(rangeChangeListener)
@@ -134,6 +183,28 @@ class NumassLoaderView : View() {
                 .setValue("yAxis.axisTitle", "HV")
         hvPlot.plot = JFreeChartFrame(hvPlotMeta)
 
+        dataProperty.addListener { observable, oldValue, newData ->
+            if (newData != null) {
+                getWorkManager().startWork("viewer.numass.load") { work ->
+                    work.title = "Load numass data (" + newData.name + ")"
+
+                    //setup info
+                    updateInfo(newData)
+                    //setup spectrum plot
+                    updateSpectrum(newData)
+                    //setup hv plot
+                    updateHV(newData)
+                    //setup detector data
+                    updateDetectorPane(newData)
+
+                }
+            } else {
+                spectrumData.clear()
+                hvPlotData.forEach { it.clear() }
+            }
+        }
+
+
     }
 
     fun getContext(): Context {
@@ -145,42 +216,18 @@ class NumassLoaderView : View() {
     }
 
     fun loadData(data: NumassData?) {
-        synchronized(this) {
+        this.data = if (data == null) {
-            this.data = data
+            data
-            if (data != null) {
+        } else {
-                getWorkManager().startWork("viewer.numass.load") { work ->
+            NumassDataCache(data)
-                    work.title = "Load numass data (" + data.name + ")"
-                    points.setAll(data.nmPoints)
-
-                    Platform.runLater {
-                        //setup detector data
-                        setupDetectorPane(points)
-                        //setup spectrum plot
-                        setupSpectrumPane(points)
-                    }
-                }
-
-                //setup hv plot
-                val hvData = data.hvData
-                if (hvData != null) {
-                    setupHVPane(hvData)
-                }
-                setupInfo(data)
-
-            } else {
-                log.severe("The data model is null")
-            }
-//            tabPane.selectionModel.select(1)
         }
     }
 
-    private fun setupHVPane(hvData: Data<Table>) {
+    private fun updateHV(data: NumassData) {
+        hvPlotData.forEach { it.clear() }
         runAsync {
-            hvData.get()
+            data.hvData.get()
         } ui {
-            for (pl in hvPlotData) {
-                pl.clear()
-            }
             for (dp in it) {
                 val block = dp.getString("block", "default")
                 if (!hvPlotData.has(block)) {
@@ -190,47 +237,25 @@ class NumassLoaderView : View() {
             }
             hvPlot.plot.addAll(hvPlotData)
         }
+
     }
 
-    /**
-     * setup detector pane
 
-     * @param points
+    private fun updateInfo(data: NumassData) {
-     */
+        val info = data.meta()
-    private fun setupDetectorPane(points: List<NumassPoint>) {
-        val normalize = detectorNormalizeSwitch.isSelected
-        val binning = detectorBinningSelector.value
-        updateDetectorPane(points, binning, normalize)
-        detectorBinningSelector.selectionModel.selectedItemProperty()
-                .addListener { observable: ObservableValue<out Int>, oldValue: Int, newValue: Int ->
-                    val norm = detectorNormalizeSwitch.isSelected
-                    updateDetectorPane(points, newValue, norm)
-                }
-        detectorNormalizeSwitch.selectedProperty().addListener { observable: ObservableValue<out Boolean>, oldValue: Boolean, newValue: Boolean ->
-            val bin = detectorBinningSelector.value
-            updateDetectorPane(points, bin, newValue)
-        }
-        detectorDataExportButton.isDisable = false
-    }
-
-    private fun setupInfo(loader: NumassData) {
-        val info = loader.meta()
         infoTextBox.text = JSONMetaWriter().writeString(info).replace("\\r", "\r\t").replace("\\n", "\n\t")
     }
 
-    private fun setupSpectrumPane(points: List<NumassPoint>) {
+    private fun updateSpectrum(data: NumassData) {
         spectrumPlot.plot.add(spectrumData)
 
         val lowChannel = channelSlider.lowValue.toInt()
         val highChannel = channelSlider.highValue.toInt()
 
-        if (points.isEmpty()) {
+        spectrumData.fillData(data.nmPoints.stream()
-            spectrumData.clear()
+                .map { point: NumassPoint -> getSpectrumPoint(point, lowChannel, highChannel, dTime) }
-        } else {
+                .collect(Collectors.toList<DataPoint>())
-            spectrumData.fillData(points.stream()
+        )
-                    .map { point: NumassPoint -> getSpectrumPoint(point, lowChannel, highChannel, dTime) }
-                    .collect(Collectors.toList<DataPoint>()))
-        }
     }
 
     private val dTime: Double
@@ -253,41 +278,15 @@ class NumassLoaderView : View() {
     /**
      * update detector pane with new data
      */
-    private fun updateDetectorPane(points: List<NumassPoint>, binning: Int, normalize: Boolean) {
+    private fun updateDetectorPane(data: NumassData) {
-        val detectorPlotFrame: FXPlotFrame
+        val points = data.nmPoints;
-        if (detectorPlot.plot == null) {
-            val frameMeta = MetaBuilder("frame")
-                    .setValue("title", "Detector response plot")
-                    .setNode(MetaBuilder("xAxis")
-                            .setValue("axisTitle", "ADC")
-                            .setValue("axisUnits", "channels")
-                            .build())
-                    .setNode(MetaBuilder("yAxis")
-                            .setValue("axisTitle", "count rate")
-                            .setValue("axisUnits", "Hz")
-                            .build())
-                    .setNode(MetaBuilder("legend")
-                            .setValue("show", false))
-                    .build()
-            detectorPlotFrame = JFreeChartFrame(frameMeta)
-        } else {
-            detectorPlotFrame = detectorPlot.plot
-        }
-
         val work = getWorkManager().getWork("viewer.numass.load.detector")
-
-        val plottableConfig = MetaBuilder("plot")
-                .setValue("connectionType", "step")
-                .setValue("thickness", 2)
-                .setValue("showLine", true)
-                .setValue("showSymbol", false)
-                .setValue("showErrors", false)
-                .setValue("JFreeChart.cache", true)
-                .build()
-
         work.maxProgress = points.size.toDouble()
         work.progress = 0.0
 
+        val normalize = detectorNormalizeSwitch.isSelected
+        val binning = detectorBinningSelector.value
+
         runAsync {
             points.map { point ->
                 val seriesName = String.format("%d: %.2f", points.indexOf(point), point.voltage)
@@ -303,46 +302,51 @@ class NumassLoaderView : View() {
 
         detectorPlot.plot = detectorPlotFrame
         work.setProgressToMax()
+        detectorDataExportButton.isDisable = false
+
     }
 
     private fun onSpectrumExportClick(event: ActionEvent) {
-        if (points.isNotEmpty()) {
+        if(data!= null){
-            val fileChooser = FileChooser()
+            val points = data!!.nmPoints
-            fileChooser.title = "Choose text export destination"
+            if (points.isNotEmpty()) {
-            fileChooser.initialFileName = data!!.name + "_spectrum.onComplete"
+                val fileChooser = FileChooser()
-            val destination = fileChooser.showSaveDialog(spectrumPlotPane.scene.window)
+                fileChooser.title = "Choose text export destination"
-            if (destination != null) {
+                fileChooser.initialFileName = data!!.name + "_spectrum.onComplete"
-                val names = arrayOf("Uset", "Uread", "Length", "Total", "Window", "CR", "CRerr", "Timestamp")
+                val destination = fileChooser.showSaveDialog(spectrumPlotPane.scene.window)
-                val loChannel = channelSlider.lowValue.toInt()
+                if (destination != null) {
-                val upChannel = channelSlider.highValue.toInt()
+                    val names = arrayOf("Uset", "Uread", "Length", "Total", "Window", "CR", "CRerr", "Timestamp")
-                val dTime = dTime
+                    val loChannel = channelSlider.lowValue.toInt()
-                val spectrumDataSet = ListTable.Builder(*names)
+                    val upChannel = channelSlider.highValue.toInt()
+                    val dTime = dTime
+                    val spectrumDataSet = ListTable.Builder(*names)
+
+                    for (point in points) {
+                        spectrumDataSet.row(
+                                point.voltage,
+                                point.voltage,
+                                point.length,
+                                point.totalCount,
+                                point.getCountInWindow(loChannel, upChannel),
+                                NumassDataUtils.countRateWithDeadTime(point, loChannel, upChannel, dTime),
+                                NumassDataUtils.countRateWithDeadTimeErr(point, loChannel, upChannel, dTime),
+                                point.startTime
+                        )
+                    }
+
+                    try {
+                        val comment = String.format("Numass data viewer spectrum data export for %s%n"
+                                + "Window: (%d, %d)%n"
+                                + "Dead time per event: %g%n",
+                                data!!.name, loChannel, upChannel, dTime)
+
+                        ColumnedDataWriter
+                                .writeTable(destination, spectrumDataSet.build(), comment, false)
+                    } catch (ex: IOException) {
+                        log.log(Level.SEVERE, "Destination file not found", ex)
+                    }
 
-                for (point in points) {
-                    spectrumDataSet.row(
-                            point.voltage,
-                            point.voltage,
-                            point.length,
-                            point.totalCount,
-                            point.getCountInWindow(loChannel, upChannel),
-                            NumassDataUtils.countRateWithDeadTime(point, loChannel, upChannel, dTime),
-                            NumassDataUtils.countRateWithDeadTimeErr(point, loChannel, upChannel, dTime),
-                            point.startTime
-                    )
                 }
-
-                try {
-                    val comment = String.format("Numass data viewer spectrum data export for %s%n"
-                            + "Window: (%d, %d)%n"
-                            + "Dead time per event: %g%n",
-                            data!!.name, loChannel, upChannel, dTime)
-
-                    ColumnedDataWriter
-                            .writeTable(destination, spectrumDataSet.build(), comment, false)
-                } catch (ex: IOException) {
-                    log.log(Level.SEVERE, "Destination file not found", ex)
-                }
-
             }
         }
     }

numass-viewer/src/main/kotlin/inr/numass/viewer/SlowControlView.kt

@@ -13,8 +13,7 @@ import hep.dataforge.tables.DataPoint
 import hep.dataforge.tables.ListTable
 import hep.dataforge.tables.Table
 import hep.dataforge.tables.XYAdapter
-import tornadofx.View
+import tornadofx.*
-import tornadofx.borderpane
 
 /**
  * Created by darksnake on 18.06.2017.
@@ -50,15 +49,14 @@ class SlowControlView : View("My View") {
     }
 
     private fun getData(loader: PointLoader, query: Meta = Meta.empty()): Table {
-        val index: ValueIndex<DataPoint>
+        val index: ValueIndex<DataPoint> =
-
+                if (query.hasValue("index")) {
-        //use custom index if needed
+                    //use custom index if needed
-        if (query.hasValue("index")) {
+                    loader.getIndex(query.getString("index"))
-            index = loader.getIndex(query.getString("index", ""))
+                } else {
-        } else {
+                    //use loader default one otherwise
-            //use loader default one otherwise
+                    loader.index
-            index = loader.index
+                }
-        }
         try {
             return ListTable(loader.format, index.query(query))
         } catch (e: Exception) {

numass-viewer/src/main/kotlin/inr/numass/viewer/Viewer.kt

@@ -5,6 +5,7 @@ import ch.qos.logback.classic.Logger
 import hep.dataforge.context.Global
 import hep.dataforge.fx.work.WorkManager
 import hep.dataforge.storage.commons.StorageManager
+import javafx.scene.image.Image
 import javafx.stage.Stage
 import org.slf4j.LoggerFactory
 import tornadofx.*
@@ -15,6 +16,7 @@ import tornadofx.*
 class Viewer : App(MainView::class) {
 
     override fun start(stage: Stage) {
+        stage.icons += Image(Global::class.java.getResourceAsStream("/img/df.png"))
         (LoggerFactory.getLogger(Logger.ROOT_LOGGER_NAME) as Logger).level = Level.INFO
         Global.setDefaultContext(Global.instance())
         StorageManager().startGlobal()