debunch for prepareDAta

numass-main/src/main/groovy/inr/numass/scripts/CountRateSummary.groovy

@@ -0,0 +1,28 @@
+package inr.numass.scripts
+
+import hep.dataforge.grind.GrindWorkspaceBuilder
+import hep.dataforge.tables.Table
+import inr.numass.workspace.*
+
+/**
+ * Created by darksnake on 26-Dec-16.
+ */
+
+
+GrindWorkspaceBuilder numass = new GrindWorkspaceBuilder().read(new File("D:\\Work\\Numass\\sterile2016_10\\workspace.groovy")).startup {
+    it.loadTask(NumassPrepareTask)
+    it.loadTask(NumassTableFilterTask)
+    it.loadTask(NumassFitScanTask)
+    it.loadTask(NumassSubstractEmptySourceTask)
+    it.loadTask(NumassFitScanSummaryTask)
+    it.loadTask(NumassFitTask)
+    it.loadTask(NumassFitSummaryTask)
+}
+
+numass.runTask("prepare", "fill_1_all").forEachDataWithType(Table) {
+    Table table = it.get();
+    def dp18 = table.find { it["Uset"] == 18000 }
+    def dp17 = table.find { it["Uset"] == 17000 }
+    println "${it.name}\t${dp18["CR"]}\t${dp18["CRerr"]}\t${dp17["CR"]}\t${dp17["CRerr"]}"
+}
+

numass-main/src/main/java/inr/numass/models/sterile/NumassResolution.java

@@ -5,13 +5,14 @@
  */
 package inr.numass.models.sterile;
 
-import hep.dataforge.stat.parametric.AbstractParametricBiFunction;
 import hep.dataforge.meta.Meta;
+import hep.dataforge.stat.parametric.AbstractParametricBiFunction;
 import hep.dataforge.values.NamedValueSet;
 import inr.numass.models.ResolutionFunction;
-import static java.lang.Math.sqrt;
 import org.apache.commons.math3.analysis.BivariateFunction;
 
+import static java.lang.Math.sqrt;
+
 /**
  *
  * @author Alexander Nozik <altavir@gmail.com>
@@ -60,7 +61,7 @@ public class NumassResolution extends AbstractParametricBiFunction {
     @Override
     public double value(double E, double U, NamedValueSet set) {
         assert resA > 0;
-        if (resB == 0) {
+        if (resB <= 0) {
             return this.getValueFast(E, U);
         }
         assert resB > 0;

numass-main/src/main/java/inr/numass/workspace/NumassPrepareTask.java

@@ -21,9 +21,9 @@ import hep.dataforge.workspace.TaskModel;
 import inr.numass.actions.MergeDataAction;
 import inr.numass.actions.MonitorCorrectAction;
 import inr.numass.actions.PrepareDataAction;
-import inr.numass.storage.NumassData;
+import inr.numass.debunch.DebunchReport;
-import inr.numass.storage.NumassDataLoader;
+import inr.numass.debunch.FrameAnalizer;
-import inr.numass.storage.NumassStorage;
+import inr.numass.storage.*;
 
 import java.net.URI;
 
@@ -93,9 +93,9 @@ public class NumassPrepareTask extends AbstractTask<Table> {
         callback.setMaxProgress(StorageUtils.loaderStream(storage).count());
         StorageUtils.loaderStream(storage).forEach(pair -> {
             Loader loader = pair.getValue();
-            if (loader instanceof NumassData) {
+            if (loader instanceof NumassDataLoader) {
                 NumassDataLoader nd = (NumassDataLoader) loader;
-                Data<NumassData> datum = Data.buildStatic(nd);
+                Data<NumassData> datum = buildData(context, nd, meta);
                 if (filter.acceptData(pair.getKey(), datum)) {
                     boolean accept = true;
                     if (forwardOnly || reverseOnly) {
@@ -124,6 +124,30 @@ public class NumassPrepareTask extends AbstractTask<Table> {
         return builder;
     }
 
+    private Data<NumassData> buildData(Context context, NumassDataLoader loader, Meta meta) {
+        if (meta.hasNode("debunch")) {
+            return Data.buildStatic(loader.applyRawTransformation(rp -> debunch(context, rp, meta.getMeta("debunch"))));
+        } else {
+            return Data.buildStatic(loader);
+        }
+    }
+
+    private NMPoint debunch(Context context, RawNMPoint point, Meta meta) {
+        int upper = meta.getInt("upperchanel", RawNMPoint.MAX_CHANEL);
+        int lower = meta.getInt("lowerchanel", 0);
+        double rejectionprob = meta.getDouble("rejectprob", 1e-10);
+        double framelength = meta.getDouble("framelength", 1);
+        double maxCR = meta.getDouble("maxcr", 500d);
+
+        double cr = point.selectChanels(lower, upper).getCR();
+        if (cr < maxCR) {
+            DebunchReport report = new FrameAnalizer(rejectionprob, framelength, lower, upper).debunchPoint(point);
+            return new NMPoint(report.getPoint());
+        } else {
+            return new NMPoint(point);
+        }
+    }
+
     private <T, R> DataNode<R> runAction(GenericAction<T, R> action, ProgressCallback callback, Context context, DataNode<T> data, Meta meta) {
         return action.run(context, data, meta);
     }

numass-storage/src/main/java/inr/numass/storage/NumassDataLoader.java

@@ -200,6 +200,7 @@ public class NumassDataLoader extends AbstractLoader implements ObjectLoader<Env
 
     /**
      * Read raw point. Requires a low of memory.
+     *
      * @param envelope
      * @return
      */
@@ -256,7 +257,7 @@ public class NumassDataLoader extends AbstractLoader implements ObjectLoader<Env
      * @return
      */
     public NMPoint readPoint(Envelope envelope) {
-        return readPoint(envelope, (p) -> new NMPoint(p));
+        return readPoint(envelope, NMPoint::new);
     }
 
     private Map<String, Supplier<Envelope>> getItems() {
@@ -303,11 +304,15 @@ public class NumassDataLoader extends AbstractLoader implements ObjectLoader<Env
 
     @Override
     public List<NMPoint> getNMPoints() {
-        return this.getPoints().stream().parallel().map(env -> readPoint(env)).collect(Collectors.toList());
+        return this.getPoints().stream().parallel().map(this::readPoint).collect(Collectors.toList());
+    }
+
+    public List<NMPoint> getNMPoints(Function<RawNMPoint, NMPoint> transformation) {
+        return this.getPoints().stream().parallel().map(env -> readPoint(env, transformation)).collect(Collectors.toList());
     }
 
     public List<RawNMPoint> getRawPoints() {
-        return this.getPoints().stream().parallel().map(env -> readRawPoint(env)).collect(Collectors.toList());
+        return this.getPoints().stream().parallel().map(this::readRawPoint).collect(Collectors.toList());
     }
 
     private List<Envelope> getPoints() {
@@ -323,11 +328,7 @@ public class NumassDataLoader extends AbstractLoader implements ObjectLoader<Env
         //TODO replace by meta tag in later revisions
         return SetDirectionUtility.isReversed(getPath(), n -> {
             List<Envelope> points = getPoints();
-            if (getPoints().size() >= 2) {
+            return getPoints().size() >= 2 && readTime(points.get(0).meta()).isAfter(readTime(points.get(1).meta()));
-                return readTime(points.get(0).meta()).isAfter(readTime(points.get(1).meta()));
-            } else {
-                return false;
-            }
         });
     }
 
@@ -376,4 +377,44 @@ public class NumassDataLoader extends AbstractLoader implements ObjectLoader<Env
     public void open() throws Exception {
 
     }
+
+    /**
+     * Return new NumassData using given transformation for each point
+     *
+     * @param transform
+     * @return
+     */
+    public NumassData applyRawTransformation(Function<RawNMPoint, NMPoint> transform) {
+        return new NumassData() {
+            @Override
+            public String getDescription() {
+                return NumassDataLoader.this.getDescription();
+            }
+
+            @Override
+            public Meta meta() {
+                return NumassDataLoader.this.meta();
+            }
+
+            @Override
+            public List<NMPoint> getNMPoints() {
+                return NumassDataLoader.this.getNMPoints(transform);
+            }
+
+            @Override
+            public boolean isEmpty() {
+                return NumassDataLoader.this.isEmpty();
+            }
+
+            @Override
+            public Instant startTime() {
+                return NumassDataLoader.this.startTime();
+            }
+
+            @Override
+            public String getName() {
+                return NumassDataLoader.this.getName();
+            }
+        };
+    }
 }