[no commit message]

numass-main/src/main/groovy/inr/numass/scripts/FindExIonRatio.groovy

@@ -20,7 +20,7 @@ import hep.dataforge.datafitter.ParamSet
 import hep.dataforge.maths.integration.RiemanIntegrator
 import hep.dataforge.maths.integration.UnivariateIntegrator
 import hep.dataforge.plots.PlotFrame
-import hep.dataforge.plots.data.PlottableFunction
+import hep.dataforge.plots.data.PlottableXYFunction
 import hep.dataforge.plots.jfreechart.JFreeChartFrame
 import org.apache.commons.math3.analysis.UnivariateFunction
 import org.apache.commons.math3.analysis.solvers.BisectionSolver
@@ -43,7 +43,7 @@ ParamSet params = new ParamSet()
 UnivariateFunction scatterFunction = LossCalculator.getSingleScatterFunction(params);
 
 PlotFrame frame = JFreeChartFrame.drawFrame("Differential scatter function", null);
-frame.add(new PlottableFunction("differential", null, scatterFunction, 0, 100, 400));
+frame.add(PlottableXYFunction.plotFunction("differential", scatterFunction, 0, 100, 400));
 
 UnivariateIntegrator integrator = NumassContext.defaultIntegrator;
 
@@ -94,7 +94,7 @@ UnivariateFunction integral = {double u ->
 }
 
 
-frame.add(new PlottableFunction("integral", null, integral, 0, 100, 800));
+frame.add(PlottableXYFunction.plotFunction("integral", integral, 0, 100, 800));
 
 BisectionSolver solver = new BisectionSolver(1e-3);
 

numass-main/src/main/groovy/inr/numass/scripts/SystTransmission.groovy

@@ -36,7 +36,7 @@ import inr.numass.models.ResolutionFunction
 import inr.numass.utils.DataModelUtils;
 import hep.dataforge.plotfit.PlotFitResultAction;
 import hep.dataforge.plots.PlotFrame
-import hep.dataforge.plots.data.PlottableFunction
+import hep.dataforge.plots.data.PlottableXYFunction
 import hep.dataforge.plots.jfreechart.JFreeChartFrame
 import java.io.FileNotFoundException;
 import java.util.Locale;

numass-main/src/main/groovy/inr/numass/scripts/Systematics.groovy

@@ -36,7 +36,7 @@ import inr.numass.models.ResolutionFunction
 import inr.numass.utils.DataModelUtils;
 import hep.dataforge.plotfit.PlotFitResultAction;
 import hep.dataforge.plots.PlotFrame
-import hep.dataforge.plots.data.PlottableFunction
+import hep.dataforge.plots.data.PlottableXYFunction
 import hep.dataforge.plots.jfreechart.JFreeChartFrame
 import java.io.FileNotFoundException;
 import java.util.Locale;

numass-main/src/main/groovy/inr/numass/scripts/TestExperimentalVariableLossSpectrum.groovy

@@ -16,7 +16,7 @@
 package inr.numass.scripts
 
 import hep.dataforge.maths.integration.UnivariateIntegrator
-import hep.dataforge.plots.data.PlottableFunction
+import hep.dataforge.plots.data.PlottableXYFunction
 import hep.dataforge.plots.jfreechart.JFreeChartFrame
 import inr.numass.models.ExperimentalVariableLossSpectrum
 import org.apache.commons.math3.analysis.UnivariateFunction
@@ -48,17 +48,17 @@ JFreeChartFrame frame = JFreeChartFrame.drawFrame("Experimental Loss Test", null
 UnivariateIntegrator integrator = NumassContext.defaultIntegrator
 
 UnivariateFunction exFunc = lsp.excitation(params.getValue("exPos"), params.getValue("exW"));
-frame.add(new PlottableFunction("ex", null, exFunc, 0d, 50d, 500));
+frame.add(PlottableXYFunction.plotFunction("ex", exFunc, 0d, 50d, 500));
 
 println "excitation norm factor " + integrator.integrate(exFunc, 0, 50)
 
 UnivariateFunction ionFunc = lsp.ionization(params.getValue("ionPos"), params.getValue("ionW"));
-frame.add(new PlottableFunction("ion", null, ionFunc, 0d, 50d, 500));
+frame.add(PlottableXYFunction.plotFunction("ion",  ionFunc, 0d, 50d, 500));
 
 println "ionization norm factor " + integrator.integrate(ionFunc, 0, 200)
 
 UnivariateFunction sumFunc = lsp.singleScatterFunction(params);
-frame.add(new PlottableFunction("sum", null, sumFunc, 0d, 50d, 500));
+frame.add(PlottableXYFunction.plotFunction("sum",  sumFunc, 0d, 50d, 500));
 
 println "sum norm factor " + integrator.integrate(sumFunc, 0, 100)
 
@@ -68,4 +68,4 @@ PrintFunction.printFunctionSimple(new PrintWriter(System.out), sumFunc, 0d, 50d,
 JFreeChartFrame integerFrame = JFreeChartFrame.drawFrame("Experimental Loss Test", null);
 
 UnivariateFunction integr = { d-> lsp.value(d,params)}
-integerFrame.add(new PlottableFunction("integr", null, integr, 18950d, 19005d, 500));
+integerFrame.add(PlottableXYFunction.plotFunction("integr", integr, 18950d, 19005d, 500));

numass-main/src/main/groovy/inr/numass/scripts/TheoreticalLossFunction.groovy

@@ -15,7 +15,7 @@
  */
 package inr.numass.scripts
 
-import hep.dataforge.plots.data.PlottableFunction
+import hep.dataforge.plots.data.PlottableXYFunction
 import hep.dataforge.plots.jfreechart.JFreeChartFrame
 import org.apache.commons.math3.analysis.UnivariateFunction
 
@@ -43,4 +43,4 @@ UnivariateFunction excitation = excitationSpectrum(lines,0.08)
 
 JFreeChartFrame frame = JFreeChartFrame.drawFrame("theoretical loss spectrum", null);
 
-frame.add(new PlottableFunction("excitation", null, excitation, 0d, 20d, 500));
+frame.add(PlottableXYFunction.plotFunction("excitation", excitation, 0d, 20d, 500));

numass-main/src/main/java/inr/numass/actions/MergeDataAction.java

@@ -46,7 +46,7 @@ import java.util.Map;
 public class MergeDataAction extends ManyToOneAction<Table, Table> {
 
     public static final String MERGE_NAME = "mergeName";
-    public static String[] parnames = {"Uset", "Uread", "Length", "Total", "Window", "Corrected", "CR", "CRerr"};
+    public static String[] parnames = {"Uset", "Uread", "Length", "Total", "Window", "CR", "CRerr"};
 
     @Override
     @SuppressWarnings("unchecked")
@@ -117,7 +117,7 @@ public class MergeDataAction extends ManyToOneAction<Table, Table> {
 
         long total = dp1.getValue(parnames[3]).intValue() + dp2.getValue(parnames[3]).intValue();
         long wind = dp1.getValue(parnames[4]).intValue() + dp2.getValue(parnames[4]).intValue();
-        double corr = dp1.getValue(parnames[5]).doubleValue() + dp2.getValue(parnames[5]).doubleValue();
+//        double corr = dp1.getValue(parnames[5]).doubleValue() + dp2.getValue(parnames[5]).doubleValue();
 
         double cr1 = dp1.getValue("CR").doubleValue();
         double cr2 = dp2.getValue("CR").doubleValue();
@@ -130,7 +130,7 @@ public class MergeDataAction extends ManyToOneAction<Table, Table> {
         // абсолютные ошибки складываются квадратично
         double crErr = Math.sqrt(err1 * err1 * t1 * t1 + err2 * err2 * t2 * t2) / time;
 
-        MapPoint.Builder map = new MapPoint(parnames, Uset, Uread, time, total, wind, corr, cr, crErr).builder();
+        MapPoint.Builder map = new MapPoint(parnames, Uset, Uread, time, total, wind, cr, crErr).builder();
 
         if (dp1.names().contains("relCR") && dp2.names().contains("relCR")) {
             double relCR = (dp1.getDouble("relCR") + dp2.getDouble("relCR")) / 2;

numass-main/src/main/java/inr/numass/actions/MonitorCorrectAction.java

@@ -114,7 +114,6 @@ public class MonitorCorrectAction extends OneToOneAction<Table, Table> {
 
                 pb.putValue("CR", Value.of(dp.getValue("CR").doubleValue() / corrFactor));
                 pb.putValue("Window", Value.of(dp.getValue("Window").doubleValue() / corrFactor));
-                pb.putValue("Corrected", Value.of(dp.getValue("Corrected").doubleValue() / corrFactor));
                 pb.putValue("CRerr", Value.of(err));
             } else {
                 double corrFactor = dp.getValue("CR").doubleValue() / norm;

numass-main/src/main/java/inr/numass/actions/PrepareDataAction.java

@@ -40,6 +40,11 @@ import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.stream.Collectors;
+import org.apache.commons.math3.analysis.ParametricUnivariateFunction;
+import org.apache.commons.math3.exception.DimensionMismatchException;
+import org.apache.commons.math3.fitting.SimpleCurveFitter;
+import org.apache.commons.math3.fitting.WeightedObservedPoint;
 
 /**
  *
@@ -49,12 +54,17 @@ import java.util.Map;
 @ValueDef(name = "lowerWindow", type = "NUMBER", def = "0", info = "Base for the window lowerWindow bound")
 @ValueDef(name = "lowerWindowSlope", type = "NUMBER", def = "0", info = "Slope for the window lowerWindow bound")
 @ValueDef(name = "upperWindow", type = "NUMBER", info = "Upper bound for window")
-@ValueDef(name = "correction",
-        info = "An expression to correct coun tumber depending on potential U, count rate CR and point length T")
 @ValueDef(name = "deadTime", type = "NUMBER", def = "0", info = "Dead time in us")
+@ValueDef(name = "underflow", type = "BOOLEAN", def = "true",
+        info = "Enables calculation of detector threshold underflow using exponential shape of energy spectrum tail. "
+        + "Not recomended to use with floating window.")
+@ValueDef(name = "underflow.upperBorder", type = "NUMBER", def = "800", info = "Upper chanel for underflow calculation.")
+@ValueDef(name = "underflow.thresholdU", type = "NUMBER", def = "17000", info = "The maximum U for undeflow calculation")
+@ValueDef(name = "underflow.correctionFunction",
+        info = "An expression to correct coun tumber depending on potential U, count rate CR and point length T")
 public class PrepareDataAction extends OneToOneAction<NumassData, Table> {
 
-    public static String[] parnames = {"Uset", "Uread", "Length", "Total", "Window", "Corrected", "CR", "CRerr", "Timestamp"};
+    public static String[] parnames = {"Uset", "Uread", "Length", "Total", "Window", "Corr", "CR", "CRerr", "Timestamp"};
 
     private int getLowerBorder(Meta meta, double Uset) throws ContentException {
         double b = meta.getDouble("lowerWindow", 0);
@@ -85,27 +95,19 @@ public class PrepareDataAction extends OneToOneAction<NumassData, Table> {
             // count in window
             long wind = point.getCountInWindow(a, b);
 
-            // count in window with deadTime applied
-            double corr = point.getCountRate(a, b, deadTime) * point.getLength();// - bkg * (b - a);
-
             // count rate after all corrections
             double cr = point.getCountRate(a, b, deadTime);
             // count rate error after all corrections
             double crErr = point.getCountRateErr(a, b, deadTime);
 
-            if(meta.hasValue("correction")){
+            double correctionFactor = getUnderflowCorrection(log, point, meta, cr);
-                Map<String,Double> exprParams = new HashMap<>();
+
-                exprParams.put("U", point.getUread());
-                exprParams.put("CR", cr);
-                exprParams.put("T", time);
-                double correctionFactor = ExpressionUtils.evaluate(meta.getString("correction"), exprParams);
             cr = cr * correctionFactor;
             crErr = crErr * correctionFactor;
-            }
 
             Instant timestamp = point.getStartTime();
 
-            dataList.add(new MapPoint(parnames, new Object[]{Uset, Uread, time, total, wind, corr, cr, crErr, timestamp}));
+            dataList.add(new MapPoint(parnames, new Object[]{Uset, Uread, time, total, wind, correctionFactor, cr, crErr, timestamp}));
         }
 
         TableFormat format;
@@ -134,4 +136,97 @@ public class PrepareDataAction extends OneToOneAction<NumassData, Table> {
         return data;
     }
 
+    /**
+     * The factor to correct for count below detector threshold
+     *
+     * @param log
+     * @param point
+     * @param meta
+     * @param countRate precalculated count rate in main window
+     * @return
+     */
+    private double getUnderflowCorrection(Reportable log, NMPoint point, Laminate meta, double countRate) {
+        if (meta.hasNode("underflow") || meta.getBoolean("underflow", true)) {
+            if (point.getUset() > meta.getDouble("underflow.thresholdU", 17000)) {
+                if (meta.hasValue("underflow.correctionFunction")) {
+                    Map<String, Double> exprParams = new HashMap<>();
+                    exprParams.put("U", point.getUread());
+                    exprParams.put("CR", countRate);
+                    exprParams.put("T", point.getLength());
+                    return ExpressionUtils.evaluate(meta.getString("underflow.correctionFunction"), exprParams);
+                } else {
+                    return 1;
+                }
+            } else {
+                try {
+                    log.report("Calculating underflow correction coefficient for point {}", point.getUset());
+                    int xLow = meta.getInt("underflow.lowerBorder", meta.getInt("lowerWindow"));
+                    int xHigh = meta.getInt("underflow.upperBorder", 800);
+                    int binning = meta.getInt("underflow.binning", 20);
+                    int upper = meta.getInt("upperWindow", RawNMPoint.MAX_CHANEL - 1);
+                    long norm = point.getCountInWindow(xLow, upper);
+                    double[] fitRes = getUnderflowExpParameters(point, xLow, xHigh, binning);
+                    log.report("Underflow interpolation function: {}*exp(c/{})", fitRes[0], fitRes[1]);
+                    double correction = fitRes[0] * fitRes[1] * (Math.exp(xLow / fitRes[1]) - 1d) / norm + 1d;
+                    log.report("Underflow correction factor: {}", correction);
+                    return correction;
+                } catch (Exception ex) {
+                    log.reportError("Failed to calculate underflow parameters for point {} with message:", point.getUset(), ex.getMessage());
+                    return 1d;
+                }
+            }
+        } else {
+            return 1;
+        }
+    }
+
+    /**
+     * Calculate underflow exponent parameters using (xLow, xHigh) window for
+     * extrapolation
+     *
+     * @param point
+     * @param xLow
+     * @param xHigh
+     * @return
+     */
+    private double[] getUnderflowExpParameters(NMPoint point, int xLow, int xHigh, int binning) {
+        if (xHigh <= xLow) {
+            throw new IllegalArgumentException("Wrong borders for underflow calculation");
+        }
+        List<WeightedObservedPoint> points = point.getMapWithBinning(binning, false)
+                .entrySet().stream()
+                .filter(entry -> entry.getKey() >= xLow && entry.getKey() <= xHigh)
+                .map(p -> new WeightedObservedPoint(1d / p.getValue(), p.getKey(), p.getValue()/ binning))
+                .collect(Collectors.toList());
+        SimpleCurveFitter fitter = SimpleCurveFitter.create(new ExponentFunction(), new double[]{1d, 200d});
+        return fitter.fit(points);
+    }
+
+    /**
+     * Exponential function for fitting
+     */
+    private static class ExponentFunction implements ParametricUnivariateFunction {
+
+        @Override
+        public double value(double x, double... parameters) {
+            if (parameters.length != 2) {
+                throw new DimensionMismatchException(parameters.length, 2);
+            }
+            double a = parameters[0];
+            double x0 = parameters[1];
+            return a * Math.exp(x / x0);
+        }
+
+        @Override
+        public double[] gradient(double x, double... parameters) {
+            if (parameters.length != 2) {
+                throw new DimensionMismatchException(parameters.length, 2);
+            }
+            double a = parameters[0];
+            double x0 = parameters[1];
+            return new double[]{Math.exp(x / x0), -a * x / x0 / x0 * Math.exp(x / x0)};
+        }
+
+    }
+
 }

numass-main/src/main/java/inr/numass/workbench/NumassWorkbenchController.java

@@ -36,6 +36,7 @@ import inr.numass.NumassIO;
 import inr.numass.NumassProperties;
 import java.io.File;
 import java.io.IOException;
+import java.io.PrintStream;
 import java.net.URL;
 import java.text.ParseException;
 import java.util.ArrayList;
@@ -251,7 +252,7 @@ public class NumassWorkbenchController implements Initializable, StagePaneHolder
         List<Configuration> actions = config.getNodes("action").stream()
                 .<Configuration>map(m -> new Configuration(m)).collect(Collectors.toList());
 
-        actionsConfig.setNode("action", actions);
+        actionsConfig.attachNodeItem("action", actions);
 
         int counter = 0;
         for (Configuration action : actions) {
@@ -348,7 +349,9 @@ public class NumassWorkbenchController implements Initializable, StagePaneHolder
             } catch (Exception ex) {
                 GlobalContext.instance().getLogger().error("Exception while executing action chain", ex);
                 Platform.runLater(() -> {
-//                    ex.printStackTrace();
+                    //printing stack trace to the default output
+                    ex.printStackTrace(System.err);
+//                    ex.printStackTrace();//??
                     statusBar.setText("Execution failed");
                     Alert alert = new Alert(Alert.AlertType.ERROR);
                     alert.setTitle("Exception!");

numass-main/src/main/java/inr/numass/workbench/WorkbenchIOManager.java

@@ -54,6 +54,7 @@ public class WorkbenchIOManager implements IOManager {
 
     @Override
     public OutputStream out(Name stage, Name name) {
+        //split output between parent output and holder output
         OutputStream primary = holder.getStagePane(stage.toString()).buildTextOutput(name.toString()).getStream();
         OutputStream secondary = manager.out(stage, name);
         return new TeeOutputStream(primary, secondary);