Column table implementation

numass-core/src/main/java/inr/numass/data/analyzers/AbstractAnalyzer.java

@@ -0,0 +1,72 @@
+package inr.numass.data.analyzers;
+
+import hep.dataforge.meta.Meta;
+import hep.dataforge.tables.ListTable;
+import hep.dataforge.tables.Table;
+import hep.dataforge.tables.TableFormat;
+import hep.dataforge.tables.TableFormatBuilder;
+import inr.numass.data.api.*;
+import org.jetbrains.annotations.Nullable;
+
+import java.util.stream.Stream;
+
+import static hep.dataforge.tables.XYAdapter.*;
+import static inr.numass.data.api.NumassPoint.HV_KEY;
+
+/**
+ * Created by darksnake on 11.07.2017.
+ */
+public abstract class AbstractAnalyzer implements NumassAnalyzer {
+    public static String[] NAME_LIST = {"length", "count", COUNT_RATE_KEY, COUNT_RATE_ERROR_KEY, "window", "timestamp"};
+    public static String[] NAME_LIST_WITH_HV = {HV_KEY, "length", "count", COUNT_RATE_KEY, COUNT_RATE_ERROR_KEY, "window", "timestamp"};
+    @Nullable
+    private final SignalProcessor processor;
+
+    public AbstractAnalyzer(@Nullable SignalProcessor processor) {
+        this.processor = processor;
+    }
+
+    public AbstractAnalyzer() {
+        this.processor = null;
+    }
+
+    /**
+     * Return unsorted stream of events including events from frames
+     *
+     * @param block
+     * @return
+     */
+    public Stream<NumassEvent> getEventStream(NumassBlock block, Meta config) {
+        if (getProcessor() == null && block.getFrames().count() > 0) {
+            throw new IllegalArgumentException("Signal processor needed to analyze frames");
+        } else {
+            int loChannel = config.getInt("window.lo", 0);
+            int upChannel = config.getInt("window.up", Integer.MAX_VALUE);
+            return Stream.concat(block.getEvents(), block.getFrames().flatMap(getProcessor()::analyze))
+                    .filter(it -> it.getChanel() >= loChannel && it.getChanel() <= upChannel);
+        }
+    }
+
+
+    @Override
+    public Table analyze(NumassSet set, Meta config) {
+        TableFormat format = new TableFormatBuilder()
+                .addNumber(HV_KEY, X_VALUE_KEY)
+                .addNumber("length")
+                .addNumber("count")
+                .addNumber(COUNT_RATE_KEY, Y_VALUE_KEY)
+                .addNumber(COUNT_RATE_ERROR_KEY, Y_ERROR_KEY)
+                .addColumn("window")
+                .addTime()
+                .build();
+
+        return new ListTable.Builder(format)
+                .rows(set.getPoints().map(point -> analyze(point, config)))
+                .build();
+    }
+
+    @Nullable
+    public SignalProcessor getProcessor() {
+        return processor;
+    }
+}

numass-core/src/main/java/inr/numass/data/analyzers/DebunchAnalyzer.java

@@ -0,0 +1,25 @@
+package inr.numass.data.analyzers;
+
+import hep.dataforge.meta.Meta;
+import hep.dataforge.values.Values;
+import inr.numass.data.api.NumassBlock;
+import inr.numass.data.api.SignalProcessor;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Block analyzer that can perform debunching
+ * Created by darksnake on 11.07.2017.
+ */
+public class DebunchAnalyzer extends AbstractAnalyzer {
+    public DebunchAnalyzer(@Nullable SignalProcessor processor) {
+        super(processor);
+    }
+
+    public DebunchAnalyzer() {
+    }
+
+    @Override
+    public Values analyze(NumassBlock block, Meta config) {
+        throw new UnsupportedOperationException("TODO");
+    }
+}

numass-core/src/main/java/inr/numass/data/analyzers/SimpleAnalyzer.java

@@ -1,59 +1,31 @@
 package inr.numass.data.analyzers;
 
 import hep.dataforge.meta.Meta;
-import hep.dataforge.tables.*;
+import hep.dataforge.tables.ValueMap;
 import hep.dataforge.values.Values;
-import inr.numass.data.api.*;
+import inr.numass.data.api.NumassBlock;
+import inr.numass.data.api.NumassPoint;
+import inr.numass.data.api.SignalProcessor;
 import org.jetbrains.annotations.Nullable;
 
-import java.util.NavigableMap;
-import java.util.TreeMap;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.stream.Stream;
-
-import static hep.dataforge.tables.XYAdapter.*;
-import static inr.numass.data.api.NumassPoint.HV_KEY;
-
 /**
  * A simple event counter
  * Created by darksnake on 07.07.2017.
  */
-public class SimpleAnalyzer implements NumassAnalyzer {
+public class SimpleAnalyzer extends AbstractAnalyzer {
-    public static String[] NAME_LIST = {"length", "count", COUNT_RATE_KEY, COUNT_RATE_ERROR_KEY, "window", "timestamp"};
-    public static String[] NAME_LIST_WITH_HV = {HV_KEY, "length", "count", COUNT_RATE_KEY, COUNT_RATE_ERROR_KEY, "window", "timestamp"};
-
-    @Nullable
-    private final SignalProcessor processor;
 
     public SimpleAnalyzer(@Nullable SignalProcessor processor) {
-        this.processor = processor;
+        super(processor);
     }
 
     public SimpleAnalyzer() {
-        this.processor = null;
-    }
-
-    /**
-     * Return unsorted stream of events including events from frames
-     *
-     * @param block
-     * @return
-     */
-    protected Stream<NumassEvent> getEventStream(NumassBlock block) {
-        if (processor == null && block.getFrames().count() > 0) {
-            throw new IllegalArgumentException("Signal processor needed to analyze frames");
-        } else {
-            return Stream.concat(block.getEvents(), block.getFrames().flatMap(processor::analyze));
-        }
     }
 
     @Override
     public Values analyze(NumassBlock block, Meta config) {
-        int loChannel = config.getInt("energy.lo", 0);
+        int loChannel = config.getInt("window.lo", 0);
-        int upChannel = config.getInt("energy.up", Integer.MAX_VALUE);
+        int upChannel = config.getInt("window.up", Integer.MAX_VALUE);
-        long count = getEventStream(block)
+        long count = getEventStream(block, config).count();
-                .filter(it -> it.getChanel() >= loChannel && it.getChanel() <= upChannel)
-                .count();
         double countRate = (double) count / block.getLength().toMillis() * 1000;
         double countRateError = Math.sqrt((double) count) / block.getLength().toMillis() * 1000;
 
@@ -84,50 +56,4 @@ public class SimpleAnalyzer implements NumassAnalyzer {
     }
 
 
-    @Override
-    public Table analyze(NumassSet set, Meta config) {
-        TableFormat format = new TableFormatBuilder()
-                .addNumber(HV_KEY, X_VALUE_KEY)
-                .addNumber("length")
-                .addNumber("count")
-                .addNumber(COUNT_RATE_KEY, Y_VALUE_KEY)
-                .addNumber(COUNT_RATE_ERROR_KEY, Y_ERROR_KEY)
-                .addColumn("window")
-                .addTime()
-                .build();
-
-        return new ListTable.Builder(format)
-                .rows(set.getPoints().map(point -> analyze(point, config)))
-                .build();
-    }
-
-    @Override
-    public Table getSpectrum(NumassBlock block, Meta config) {
-        TableFormat format = new TableFormatBuilder()
-                .addNumber("channel", X_VALUE_KEY)
-                .addNumber("count")
-                .addNumber(COUNT_RATE_KEY, Y_VALUE_KEY)
-                .addNumber(COUNT_RATE_ERROR_KEY, Y_ERROR_KEY)
-                .updateMeta(metaBuilder -> metaBuilder.setNode("config", config))
-                .build();
-        NavigableMap<Short, AtomicLong> map = new TreeMap<>();
-        getEventStream(block).forEach(event -> {
-            if (map.containsKey(event.getChanel())) {
-                map.get(event.getChanel()).incrementAndGet();
-            } else {
-                map.put(event.getChanel(), new AtomicLong(1));
-            }
-        });
-        return new ListTable.Builder(format)
-                .rows(map.entrySet().stream()
-                        .map(entry ->
-                                new ValueMap(format.namesAsArray(),
-                                        entry.getKey(),
-                                        entry.getValue(),
-                                        entry.getValue().get() / block.getLength().toMillis() * 1000,
-                                        Math.sqrt(entry.getValue().get()) / block.getLength().toMillis() * 1000
-                                )
-                        )
-                ).build();
-    }
 }

numass-core/src/main/java/inr/numass/data/analyzers/SmartAnalyzer.java

@@ -0,0 +1,42 @@
+package inr.numass.data.analyzers;
+
+import hep.dataforge.meta.Meta;
+import hep.dataforge.values.Values;
+import inr.numass.data.api.NumassBlock;
+import inr.numass.data.api.SignalProcessor;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * An analyzer dispatcher which uses different analyzer for different meta
+ * Created by darksnake on 11.07.2017.
+ */
+public class SmartAnalyzer extends AbstractAnalyzer {
+    private SimpleAnalyzer simpleAnalyzer = new SimpleAnalyzer();
+    private DebunchAnalyzer debunchAnalyzer = new DebunchAnalyzer();
+    private TimeAnalyzer timeAnalyzer = new TimeAnalyzer();
+
+    public SmartAnalyzer(@Nullable SignalProcessor processor) {
+        super(processor);
+        this.simpleAnalyzer = new SimpleAnalyzer(processor);
+        this.debunchAnalyzer = new DebunchAnalyzer(processor);
+        this.timeAnalyzer = new TimeAnalyzer(processor);
+    }
+
+    public SmartAnalyzer() {
+    }
+
+    @Override
+    public Values analyze(NumassBlock block, Meta config) {
+        //TODO do something more... smart... using information from point if block is point
+        switch (config.getString("type", "simple")) {
+            case "simple":
+                return simpleAnalyzer.analyze(block, config);
+            case "time":
+                return timeAnalyzer.analyze(block, config);
+            case "debunch":
+                return debunchAnalyzer.analyze(block, config);
+            default:
+                throw new IllegalArgumentException("Analyzer not found");
+        }
+    }
+}

numass-core/src/main/java/inr/numass/data/analyzers/TimeAnalyzer.java

@@ -0,0 +1,25 @@
+package inr.numass.data.analyzers;
+
+import hep.dataforge.meta.Meta;
+import hep.dataforge.values.Values;
+import inr.numass.data.api.NumassBlock;
+import inr.numass.data.api.SignalProcessor;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Created by darksnake on 11.07.2017.
+ */
+public class TimeAnalyzer extends AbstractAnalyzer {
+
+    public TimeAnalyzer(@Nullable SignalProcessor processor) {
+        super(processor);
+    }
+
+    public TimeAnalyzer() {
+    }
+
+    @Override
+    public Values analyze(NumassBlock block, Meta config) {
+        throw new UnsupportedOperationException("TODO");
+    }
+}

numass-core/src/main/java/inr/numass/data/api/NumassAnalyzer.java

@@ -1,14 +1,52 @@
 package inr.numass.data.api;
 
 import hep.dataforge.meta.Meta;
-import hep.dataforge.tables.Table;
+import hep.dataforge.tables.*;
 import hep.dataforge.values.Values;
+import inr.numass.data.analyzers.SmartAnalyzer;
+
+import java.util.NavigableMap;
+import java.util.TreeMap;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.stream.Stream;
+
+import static hep.dataforge.tables.XYAdapter.*;
 
 /**
  * A general raw data analysis utility. Could have different implementations
  * Created by darksnake on 06-Jul-17.
  */
 public interface NumassAnalyzer {
+
+    static Table getSpectrum(NumassBlock block, Meta config) {
+        TableFormat format = new TableFormatBuilder()
+                .addNumber("channel", X_VALUE_KEY)
+                .addNumber("count")
+                .addNumber(COUNT_RATE_KEY, Y_VALUE_KEY)
+                .addNumber(COUNT_RATE_ERROR_KEY, Y_ERROR_KEY)
+                .updateMeta(metaBuilder -> metaBuilder.setNode("config", config))
+                .build();
+        NavigableMap<Short, AtomicLong> map = new TreeMap<>();
+        new SmartAnalyzer().getEventStream(block, config).forEach(event -> {
+            if (map.containsKey(event.getChanel())) {
+                map.get(event.getChanel()).incrementAndGet();
+            } else {
+                map.put(event.getChanel(), new AtomicLong(1));
+            }
+        });
+        return new ListTable.Builder(format)
+                .rows(map.entrySet().stream()
+                        .map(entry ->
+                                new ValueMap(format.namesAsArray(),
+                                        entry.getKey(),
+                                        entry.getValue(),
+                                        entry.getValue().get() / block.getLength().toMillis() * 1000,
+                                        Math.sqrt(entry.getValue().get()) / block.getLength().toMillis() * 1000
+                                )
+                        )
+                ).build();
+    }
+
     String COUNT_RATE_KEY = "cr";
     String COUNT_RATE_ERROR_KEY = "crErr";
 
@@ -21,21 +59,21 @@ public interface NumassAnalyzer {
      */
     Values analyze(NumassBlock block, Meta config);
 
+    /**
+     * Return unsorted stream of events including events from frames
+     *
+     * @param block
+     * @return
+     */
+    Stream<NumassEvent> getEventStream(NumassBlock block, Meta config);
+
     /**
      * Analyze the whole set. And return results as a table
+     *
      * @param set
      * @param config
      * @return
      */
     Table analyze(NumassSet set, Meta config);
 
-    /**
-     * Generate energy spectrum for the given block
-     * @param block
-     * @param config
-     * @return
-     */
-    Table getSpectrum(NumassBlock block, Meta config);
-
-
 }

numass-core/src/main/java/inr/numass/data/storage/ProtoNumassPoint.java

@@ -17,31 +17,27 @@ import java.util.stream.IntStream;
 import java.util.stream.Stream;
 
 /**
+ * Protobuf based numass point
  * Created by darksnake on 09.07.2017.
  */
 public class ProtoNumassPoint implements NumassPoint {
     private final Envelope envelope;
 
-    NumassProto.Point point;
-
     public ProtoNumassPoint(Envelope envelope) {
         this.envelope = envelope;
     }
 
     private NumassProto.Point getPoint() {
-        if (point == null) {
         try (InputStream stream = envelope.getData().getStream()) {
-                point = NumassProto.Point.parseFrom(stream);
+            return NumassProto.Point.parseFrom(stream);
         } catch (IOException ex) {
-                throw new RuntimeException("Failed to read point via protbuf");
+            throw new RuntimeException("Failed to read point via protobuf");
         }
     }
-        return point;
-    }
 
     @Override
     public Stream<NumassBlock> getBlocks() {
-        return point.getChannelsList().stream().flatMap(channel ->
+        return getPoint().getChannelsList().stream().flatMap(channel ->
                 channel.getBlocksList().stream().map(block -> new ProtoBlock((int) channel.getNum(), block))
         );
     }

numass-main/src/main/groovy/inr/numass/scripts/FindBorder.groovy

@@ -8,7 +8,6 @@ package inr.numass.scripts
 
 import hep.dataforge.grind.GrindMetaBuilder
 import hep.dataforge.meta.Meta
-import inr.numass.actions.FindBorderAction
 import inr.numass.data.storage.NumassDataLoader
 
 File dataDir = new File("D:\\Work\\Numass\\data\\2016_04\\T2_data\\Fill_2_2\\set_6_e26d123e54010000")

numass-main/src/main/java/inr/numass/actions/AnalyzeDataAction.java

@@ -0,0 +1,37 @@
+package inr.numass.actions;
+
+import hep.dataforge.actions.OneToOneAction;
+import hep.dataforge.context.Context;
+import hep.dataforge.description.TypedActionDef;
+import hep.dataforge.description.ValueDef;
+import hep.dataforge.io.ColumnedDataWriter;
+import hep.dataforge.meta.Laminate;
+import hep.dataforge.tables.Table;
+import inr.numass.data.analyzers.SmartAnalyzer;
+import inr.numass.data.api.NumassAnalyzer;
+import inr.numass.data.api.NumassSet;
+
+import java.io.OutputStream;
+
+import static hep.dataforge.values.ValueType.NUMBER;
+import static hep.dataforge.values.ValueType.STRING;
+
+/**
+ * The action performs the readout of data and collection of count rate into a table
+ * Created by darksnake on 11.07.2017.
+ */
+@TypedActionDef(name = "numass.analyze", inputType = NumassSet.class, outputType = Table.class)
+@ValueDef(name = "window.lo", type = {NUMBER, STRING}, def = "0", info = "Lower bound for window")
+@ValueDef(name = "window.up", type = {NUMBER, STRING}, def = "10000", info = "Upper bound for window")
+public class AnalyzeDataAction extends OneToOneAction<NumassSet, Table> {
+    @Override
+    protected Table execute(Context context, String name, NumassSet input, Laminate inputMeta) {
+        //TODO add processor here
+        NumassAnalyzer analyzer = new SmartAnalyzer();
+        Table res = analyzer.analyze(input,inputMeta);
+        OutputStream stream = buildActionOutput(context, name);
+
+        ColumnedDataWriter.writeTable(stream, data, head);
+        return res;
+    }
+}

numass-main/src/main/java/inr/numass/actions/FindBorderAction.java

@@ -1,129 +0,0 @@
-/* 
- * Copyright 2015 Alexander Nozik.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package inr.numass.actions;
-
-import hep.dataforge.actions.OneToOneAction;
-import hep.dataforge.context.Context;
-import hep.dataforge.description.TypedActionDef;
-import hep.dataforge.exceptions.ContentException;
-import hep.dataforge.io.ColumnedDataWriter;
-import hep.dataforge.meta.Laminate;
-import hep.dataforge.tables.ListTable;
-import hep.dataforge.tables.Table;
-import hep.dataforge.tables.ValueMap;
-import hep.dataforge.values.Value;
-import org.apache.commons.math3.analysis.UnivariateFunction;
-
-import java.io.OutputStream;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.stream.Collectors;
-
-/**
- * @author Darksnake
- */
-@TypedActionDef(name = "findBorder", inputType = NMFile.class, outputType = Table.class)
-public class FindBorderAction extends OneToOneAction<NumassData, Table> {
-
-    private final static String[] names = {"U", "80%", "90%", "95%", "99%"};
-    private final static double[] percents = {0.8, 0.9, 0.95, 0.99};
-
-    private UnivariateFunction normCorrection = e -> 1 + 13.265 * Math.exp(-e / 2343.4);
-
-    @Override
-    protected Table execute(Context context, String name, NumassData source, Laminate meta) throws ContentException {
-        report(context, name, "File {} started", source.getName());
-
-        int upperBorder = meta.getInt("upper", 4094);
-        int lowerBorder = meta.getInt("lower", 0);
-        double substractReference = meta.getDouble("reference", 0);
-
-        NumassPoint referencePoint = null;
-        if (substractReference > 0) {
-            referencePoint = source.getByVoltage(substractReference);
-            if (referencePoint == null) {
-                report(context, name, "Reference point {} not found", substractReference);
-            }
-        }
-
-        ListTable.Builder dataBuilder = new ListTable.Builder(names);
-
-        fill(dataBuilder, source, lowerBorder, upperBorder, referencePoint);
-        Table bData = dataBuilder.build();
-
-        OutputStream stream = buildActionOutput(context, name);
-
-        ColumnedDataWriter.writeTable(stream, bData, String.format("%s : lower = %d upper = %d", name, lowerBorder, upperBorder));
-
-        report(context, name, "File {} completed", source.getName());
-        return bData;
-    }
-
-    private double getNorm(Map<Double, Double> spectrum, int lower, int upper) {
-        double res = 0;
-        for (Map.Entry<Double, Double> entry : spectrum.entrySet()) {
-            if ((entry.getKey() >= lower) && (entry.getKey() <= upper)) {
-                res += entry.getValue();
-            }
-        }
-
-        return res;
-    }
-
-    private void fill(ListTable.Builder dataBuilder, NumassData file, int lower, int upper, NumassPoint reference) {
-        for (NumassPoint point : file) {
-            if ((reference != null) && (point.getVoltage() == reference.getVoltage())) {
-                continue;
-            }
-            //создаем основу для будущей точки
-            HashMap<String, Value> map = new HashMap<>();
-            map.put(names[0], Value.of(point.getVoltage()));
-            Map<Double, Double> spectrum;
-            if (reference != null) {
-
-                Map<Double, Double> sp = point.getMap(0, true);
-                Map<Double, Double> referenceSpectrum = reference.getMap(0, true);
-
-                spectrum = sp.entrySet().stream()
-                        .collect(Collectors.toMap(entry -> entry.getKey(), entry -> Math.max(entry.getValue() - referenceSpectrum.get(entry.getKey()), 0)));
-            } else {
-                spectrum = point.getMap(0, true);
-            }
-            double norm = getNorm(spectrum, lower, upper) * normCorrection.value(point.getVoltage());
-            double counter = 0;
-            int chanel = upper;
-            while (chanel > lower) {
-                chanel--;
-                counter += spectrum.get((double) chanel);
-                for (int i = 0; i < percents.length; i++) {
-                    if (counter / norm > percents[i]) {
-                        if (!map.containsKey(names[i + 1])) {
-                            map.put(names[i + 1], Value.of(chanel));
-                        }
-                    }
-                }
-            }
-            for (String n : names) {
-                if (!map.containsKey(n)) {
-                    map.put(n, Value.of(lower));
-                }
-            }
-
-            dataBuilder.row(new ValueMap(map));
-        }
-    }
-
-}

numass-main/src/main/java/inr/numass/actions/JoinNumassDataAction.java

@@ -1,37 +0,0 @@
-package inr.numass.actions;
-
-import hep.dataforge.actions.ManyToOneAction;
-import hep.dataforge.context.Context;
-import hep.dataforge.description.TypedActionDef;
-import hep.dataforge.meta.Laminate;
-
-import java.util.Collection;
-import java.util.Map;
-import java.util.stream.IntStream;
-
-/**
- * Created by darksnake on 04-Nov-16.
- */
-@TypedActionDef(name = "joinData", inputType = NumassData.class, outputType = NumassData.class,
-        info = "Join a number of numass data files into one single file via spectrum summing")
-public class JoinNumassDataAction extends ManyToOneAction<NumassData, NumassData> {
-
-    @Override
-    protected NumassData execute(Context context, String nodeName, Map<String, NumassData> input, Laminate meta) {
-        throw new UnsupportedOperationException("not implemented");
-    }
-
-    private NumassPoint joinPoint(Collection<NumassPoint> points) {
-        return points.stream().reduce((p1, p2) -> {
-            if (p1.getVoltage() != p2.getVoltage()) {
-                throw new RuntimeException("Can't sum points with different Uset");
-            }
-            return new NumassPointImpl(
-                    (p1.getVoltage() + p2.getVoltage()) / 2,
-                    p1.getStartTime(),
-                    p1.getLength() + p2.getLength(),
-                    IntStream.range(0, p1.getSpectrum().length).map(i -> p1.getSpectrum()[i] * p2.getSpectrum()[i]).toArray()
-            );
-        }).get();
-    }
-}

numass-main/src/main/java/inr/numass/actions/MergeDataAction.java

@@ -161,7 +161,7 @@ public class MergeDataAction extends ManyToOneAction<Table, Table> {
             res.add(curPoint);
         });
 
-        return new ListTable(TableFormat.forNames(parnames), res);
+        return new ListTable(MetaTableFormat.forNames(parnames), res);
 
     }
 

numass-main/src/main/java/inr/numass/actions/PrepareDataAction.java

@@ -25,14 +25,10 @@ import hep.dataforge.io.ColumnedDataWriter;
 import hep.dataforge.io.XMLMetaWriter;
 import hep.dataforge.meta.Laminate;
 import hep.dataforge.meta.Meta;
-import hep.dataforge.tables.ListTable;
+import hep.dataforge.tables.*;
-import hep.dataforge.tables.Table;
-import hep.dataforge.tables.TableFormat;
-import hep.dataforge.tables.ValueMap;
 import hep.dataforge.values.Values;
-import inr.numass.data.NumassPoint;
+import inr.numass.data.api.NumassPoint;
 import inr.numass.data.api.NumassSet;
-import inr.numass.data.storage.NumassDataLoader;
 import inr.numass.debunch.DebunchReport;
 import inr.numass.debunch.FrameAnalizer;
 import inr.numass.utils.ExpressionUtils;
@@ -77,7 +73,7 @@ public class PrepareDataAction extends OneToOneAction<NumassSet, Table> {
     protected ListTable execute(Context context, String name, NumassSet dataFile, Laminate meta) {
 //        log.report("File %s started", dataFile.getName());
 
-        int upper = meta.getInt("upperWindow", RawNMPoint.MAX_CHANEL - 1);
+        int upper = meta.getInt("upperWindow", Integer.MAX_VALUE);
 
         List<Correction> corrections = new ArrayList<>();
         if (meta.hasValue("deadTime")) {
@@ -107,16 +103,16 @@ public class PrepareDataAction extends OneToOneAction<NumassSet, Table> {
             utransform = Function.identity();
         }
 
-        if (meta.hasMeta("debunch")) {
+//        if (meta.hasMeta("debunch")) {
-            if (dataFile instanceof NumassDataLoader) {
+//            if (dataFile instanceof NumassDataLoader) {
-                dataFile = ((NumassDataLoader) dataFile).applyRawTransformation(raw -> debunch(context, raw, meta.getMeta("debunch")));
+//                dataFile = ((NumassDataLoader) dataFile).applyRawTransformation(raw -> debunch(context, raw, meta.getMeta("debunch")));
-            } else {
+//            } else {
-                throw new RuntimeException("Debunch not available");
+//                throw new RuntimeException("Debunch not available");
-            }
+//            }
-        }
+//        }
 
         List<Values> dataList = new ArrayList<>();
-        for (NumassPoint point : dataFile) {
+        dataFile.getPoints().forEach( point -> {
 
             long total = point.getTotalCount();
             double uset = utransform.apply(point.getVoltage());
@@ -148,15 +144,15 @@ public class PrepareDataAction extends OneToOneAction<NumassSet, Table> {
             Instant timestamp = point.getStartTime();
 
             dataList.add(new ValueMap(parnames, new Object[]{uset, uread, time, total, wind, correctionFactor, cr, crErr, timestamp}));
-        }
+        });
 
         TableFormat format;
 
         if (!dataList.isEmpty()) {
             //Генерируем автоматический формат по первой строчке
-            format = TableFormat.forPoint(dataList.get(0));
+            format = MetaTableFormat.forPoint(dataList.get(0));
         } else {
-            format = TableFormat.forNames(parnames);
+            format = MetaTableFormat.forNames(parnames);
         }
 
         String head;

numass-main/src/main/java/inr/numass/actions/ReadLegacyDataAction.java

@@ -1,63 +0,0 @@
-/* 
- * Copyright 2015 Alexander Nozik.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package inr.numass.actions;
-
-import hep.dataforge.actions.OneToOneAction;
-import hep.dataforge.context.Context;
-import hep.dataforge.data.binary.Binary;
-import hep.dataforge.description.NodeDef;
-import hep.dataforge.description.TypedActionDef;
-import hep.dataforge.description.ValueDef;
-import hep.dataforge.exceptions.ContentException;
-import hep.dataforge.meta.Laminate;
-
-import static hep.dataforge.values.ValueType.NUMBER;
-import static inr.numass.NumassIO.getNumassData;
-
-/**
- *
- * @author Darksnake
- */
-@TypedActionDef(name = "readData",
-        inputType = Binary.class, outputType = NMFile.class, info = "Read binary numass data file")
-@ValueDef(name = "fileName", info = "The name of the file. By default equals file name.")
-@ValueDef(name = "HVdev", info = "Divider for HV measurements. Should be set to 1.0 for numass data 2014",
-        def = "2.468555393226049", type = {NUMBER})
-@ValueDef(name = "noUset", info = "If 'true', then Uset = Uread")
-@NodeDef(name = "debunch", target = "class::inr.numass.actions.DebunchAction", info = "If given, governs debunching")
-public class ReadLegacyDataAction extends OneToOneAction<Binary, NMFile> {
-
-    @Override
-    protected NMFile execute(Context context, String name, Binary source, Laminate meta) throws ContentException {
-//        log.logString("File '%s' started", source.getName());
-        RawNMFile raw = getNumassData(source, meta);
-//        if (meta.getBoolean("paw", false)) {
-//            raw.generatePAW(buildActionOutput(context, name + ".paw"));
-//        }
-
-        if (meta.hasMeta("debunch")) {
-            DebunchAction debunch = new DebunchAction();
-            Laminate laminate = new Laminate(meta.getMeta("debunch"))
-                    .setDescriptor(debunch.getDescriptor());
-            raw = debunch.execute(context, name, raw, laminate);
-        }
-
-        NMFile result = new NMFile(raw);
-
-        return result;
-    }
-
-}

numass-main/src/main/java/inr/numass/actions/ShowEnergySpectrumAction.java

@@ -19,6 +19,7 @@ import hep.dataforge.plots.data.XYPlottable;
 import hep.dataforge.tables.*;
 import hep.dataforge.values.ValueType;
 import hep.dataforge.values.Values;
+import inr.numass.data.api.NumassSet;
 
 import java.io.OutputStream;
 import java.util.*;
@@ -28,19 +29,14 @@ import java.util.stream.Collectors;
  *
  * @author Alexander Nozik
  */
-@TypedActionDef(inputType = NumassData.class, outputType = Table.class, name = "energySpectrum", info = "Generate output table and optionally plot for detector energy spectra")
+@TypedActionDef(inputType = NumassSet.class, outputType = Table.class, name = "energySpectrum", info = "Generate output table and optionally plot for detector energy spectra")
-public class ShowEnergySpectrumAction extends OneToOneAction<NumassData, Table> {
+public class ShowEnergySpectrumAction extends OneToOneAction<NumassSet, Table> {
 
     @Override
-    protected Table execute(Context context, String name, NumassData input, Laminate inputMeta) {
+    protected Table execute(Context context, String name, NumassSet input, Laminate inputMeta) {
         int binning = inputMeta.getInt("binning", 20);
         boolean normalize = inputMeta.getBoolean("normalize", true);
-        List<NumassPoint> points = input.getNMPoints();
 
-        if (points.isEmpty()) {
-            getLogger(inputMeta).error("Empty data");
-            return null;
-        }
 
         //build header
         List<String> names = new ArrayList<>();

numass-main/src/main/java/inr/numass/actions/SubstractSpectrumAction.java

@@ -34,9 +34,9 @@ public class SubstractSpectrumAction extends OneToOneAction<Table, Table> {
             File referenceFile = context.io().getFile(referencePath);
             Table referenceTable = new ColumnedDataReader(referenceFile).toTable();
             ListTable.Builder builder = new ListTable.Builder(input.getFormat());
-            input.stream().forEach(point -> {
+            input.getRows().forEach(point -> {
                 ValueMap.Builder pointBuilder = new ValueMap.Builder(point);
-                Optional<Values> referencePoint = referenceTable.stream()
+                Optional<Values> referencePoint = referenceTable.getRows()
                         .filter(p -> Math.abs(p.getDouble("Uset") - point.getDouble("Uset")) < 0.1).findFirst();
                 if (referencePoint.isPresent()) {
                     pointBuilder.putValue("CR", Math.max(0, point.getDouble("CR") - referencePoint.get().getDouble("CR")));

numass-main/src/main/java/inr/numass/actions/SummaryAction.java

@@ -26,8 +26,8 @@ import hep.dataforge.meta.Laminate;
 import hep.dataforge.meta.Meta;
 import hep.dataforge.stat.fit.FitState;
 import hep.dataforge.tables.ListTable;
+import hep.dataforge.tables.MetaTableFormat;
 import hep.dataforge.tables.Table;
-import hep.dataforge.tables.TableFormat;
 import hep.dataforge.tables.ValueMap;
 import hep.dataforge.values.Value;
 import hep.dataforge.values.Values;
@@ -75,7 +75,7 @@ public class SummaryAction extends ManyToOneAction<FitState, Table> {
         }
         names[names.length - 1] = "chi2";
 
-        ListTable.Builder res = new ListTable.Builder(TableFormat.forNames(names));
+        ListTable.Builder res = new ListTable.Builder(MetaTableFormat.forNames(names));
 
         double[] weights = new double[parNames.length];
         Arrays.fill(weights, 0);

numass-main/src/main/java/inr/numass/actions/TransformDataAction.java

@@ -0,0 +1,114 @@
+package inr.numass.actions;
+
+import hep.dataforge.actions.OneToOneAction;
+import hep.dataforge.context.Context;
+import hep.dataforge.description.ValueDef;
+import hep.dataforge.meta.Laminate;
+import hep.dataforge.meta.Meta;
+import hep.dataforge.tables.Table;
+import hep.dataforge.values.Values;
+import inr.numass.utils.ExpressionUtils;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Function;
+import java.util.function.UnaryOperator;
+import java.util.stream.Collectors;
+
+import static hep.dataforge.values.ValueType.NUMBER;
+import static hep.dataforge.values.ValueType.STRING;
+import static inr.numass.utils.TritiumUtils.pointExpression;
+
+/**
+ * Apply corrections and transformations to analyzed data
+ * Created by darksnake on 11.07.2017.
+ */
+public class TransformDataAction extends OneToOneAction<Table, Table> {
+    @Override
+    protected Table execute(Context context, String name, Table input, Laminate meta) {
+        UnaryOperator<Values> transformation = UnaryOperator.identity();
+
+
+
+
+
+        List<Correction> corrections = new ArrayList<>();
+        if (meta.optMeta("correction").isPresent()) {
+            corrections.addAll(meta.getMetaList("correction").stream()
+                    .map((Function<Meta, Correction>) this::makeCorrection)
+                    .collect(Collectors.toList()));
+        }
+
+        if (meta.hasValue("correction")) {
+            final String correction = meta.getString("correction");
+            corrections.add((point) -> pointExpression(correction, point));
+        }
+
+        Function<Double, Double> utransform;
+        if (meta.hasValue("utransform")) {
+            String func = meta.getString("utransform");
+            utransform = u -> {
+                Map<String, Object> binding = new HashMap<>();
+                binding.put("U", u);
+                return ExpressionUtils.function(func, binding);
+            };
+        } else {
+            utransform = Function.identity();
+        }
+    }
+
+
+    @ValueDef(name = "value", type = {NUMBER, STRING}, info = "Value or function to multiply count rate")
+    @ValueDef(name = "err", type = {NUMBER, STRING}, info = "error of the value")
+    private Correction makeCorrection(Meta corrMeta) {
+        final String expr = corrMeta.getString("value");
+        final String errExpr = corrMeta.getString("err", "");
+        return new Correction() {
+            @Override
+            public double corr(Values point) {
+                return pointExpression(expr, point);
+            }
+
+            @Override
+            public double corrErr(Values point) {
+                if (errExpr.isEmpty()) {
+                    return 0;
+                } else {
+                    return pointExpression(errExpr, point);
+                }
+            }
+        };
+    }
+
+    private interface Correction {
+        /**
+         * correction coefficient
+         *
+         * @param point
+         * @return
+         */
+        double corr(Values point);
+
+        /**
+         * correction coefficient uncertainty
+         *
+         * @param point
+         * @return
+         */
+        default double corrErr(Values point) {
+            return 0;
+        }
+
+        default double relativeErr(Values point) {
+            double corrErr = corrErr(point);
+            if (corrErr == 0) {
+                return 0;
+            } else {
+                return corrErr / corr(point);
+            }
+        }
+    }
+
+}

numass-main/src/main/java/inr/numass/models/NamedSpectrumCaching.java

@@ -54,7 +54,7 @@ public class NamedSpectrumCaching extends AbstractParametricFunction {
         this.a = a;
         this.b = b;
         this.source = spectrum;
-        spectrumDerivCache = new HashMap<>(source.size());
+        spectrumDerivCache = new HashMap<>(source.getNames().size());
 //        spectrumDerivCache = new CacheElement[source.getDimension()];
     }
 

numass-main/src/main/java/inr/numass/tasks/NumassSubstractEmptySourceTask.java

@@ -83,9 +83,9 @@ public class NumassSubstractEmptySourceTask extends AbstractTask<Table> {
 
     private Table subtract(Table merge, Table empty) {
         ListTable.Builder builder = new ListTable.Builder(merge.getFormat());
-        merge.stream().forEach(point -> {
+        merge.getRows().forEach(point -> {
             ValueMap.Builder pointBuilder = new ValueMap.Builder(point);
-            Optional<Values> referencePoint = empty.stream()
+            Optional<Values> referencePoint = empty.getRows()
                     .filter(p -> Math.abs(p.getDouble("Uset") - point.getDouble("Uset")) < 0.1).findFirst();
             if (referencePoint.isPresent()) {
                 pointBuilder.putValue("CR", Math.max(0, point.getDouble("CR") - referencePoint.get().getDouble("CR")));

numass-main/src/main/java/inr/numass/tasks/NumassTableFilterTask.java

@@ -73,7 +73,7 @@ public class NumassTableFilterTask extends SingleActionTask<Table, Table> {
         for (String field : dp.getNames()) {
             Value val = dp.getValue(field);
             Object obj;
-            switch (val.valueType()) {
+            switch (val.getType()) {
                 case BOOLEAN:
                     obj = val.booleanValue();
                     break;

numass-main/src/main/java/inr/numass/utils/TritiumUtils.java

@@ -15,6 +15,7 @@
  */
 package inr.numass.utils;
 
+import hep.dataforge.values.Values;
 import org.apache.commons.math3.analysis.UnivariateFunction;
 
 import java.util.HashMap;
@@ -68,12 +69,14 @@ public class TritiumUtils {
      * @param point
      * @return
      */
-    public static double pointExpression(String expression, NumassPoint point) {
+    public static double pointExpression(String expression, Values point) {
         Map<String, Object> exprParams = new HashMap<>();
-        exprParams.put("T", point.getLength());
+        //Adding all point values to expression parameters
-        exprParams.put("U", point.getVoltage());
+        point.getNames().forEach(name-> exprParams.put(name,point.getValue(name).value()));
-        exprParams.put("cr", ((double) point.getTotalCount()) / point.getLength());
+        //Adding aliases for commonly used parameters
-        exprParams.put("point", point);
+        exprParams.put("T", point.getDouble("length"));
+        exprParams.put("U", point.getDouble("voltage"));
+
         return ExpressionUtils.function(expression, exprParams);
     }
 }

numass-server/src/main/java/inr/numass/server/HandlerUtils.java

@@ -20,7 +20,7 @@ public class HandlerUtils {
         for (String state : states.getStateNames()) {
             Value val = states.getValue(state);
             String color;
-            switch (val.valueType()) {
+            switch (val.getType()) {
                 case NUMBER:
                     color = "blue";
                     break;