From c898422e5dff71d318445471af98b75649232f1f Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Wed, 12 Jul 2017 17:22:17 +0300 Subject: [PATCH] Column table implementation --- .../data/analyzers/AbstractAnalyzer.java | 72 ++++++++++ .../data/analyzers/DebunchAnalyzer.java | 25 ++++ .../numass/data/analyzers/SimpleAnalyzer.java | 92 ++----------- .../numass/data/analyzers/SmartAnalyzer.java | 42 ++++++ .../numass/data/analyzers/TimeAnalyzer.java | 25 ++++ .../inr/numass/data/api/NumassAnalyzer.java | 58 ++++++-- .../numass/data/storage/ProtoNumassPoint.java | 16 +-- .../inr/numass/scripts/FindBorder.groovy | 1 - .../inr/numass/actions/AnalyzeDataAction.java | 37 +++++ .../inr/numass/actions/FindBorderAction.java | 129 ------------------ .../numass/actions/JoinNumassDataAction.java | 37 ----- .../inr/numass/actions/MergeDataAction.java | 2 +- .../inr/numass/actions/PrepareDataAction.java | 32 ++--- .../numass/actions/ReadLegacyDataAction.java | 63 --------- .../actions/ShowEnergySpectrumAction.java | 12 +- .../actions/SubstractSpectrumAction.java | 4 +- .../inr/numass/actions/SummaryAction.java | 4 +- .../numass/actions/TransformDataAction.java | 114 ++++++++++++++++ .../numass/models/NamedSpectrumCaching.java | 2 +- .../tasks/NumassSubstractEmptySourceTask.java | 4 +- .../numass/tasks/NumassTableFilterTask.java | 2 +- .../java/inr/numass/utils/TritiumUtils.java | 13 +- .../java/inr/numass/server/HandlerUtils.java | 2 +- 23 files changed, 414 insertions(+), 374 deletions(-) create mode 100644 numass-core/src/main/java/inr/numass/data/analyzers/AbstractAnalyzer.java create mode 100644 numass-core/src/main/java/inr/numass/data/analyzers/DebunchAnalyzer.java create mode 100644 numass-core/src/main/java/inr/numass/data/analyzers/SmartAnalyzer.java create mode 100644 numass-core/src/main/java/inr/numass/data/analyzers/TimeAnalyzer.java create mode 100644 numass-main/src/main/java/inr/numass/actions/AnalyzeDataAction.java delete mode 100644 numass-main/src/main/java/inr/numass/actions/FindBorderAction.java delete mode 100644 numass-main/src/main/java/inr/numass/actions/JoinNumassDataAction.java delete mode 100644 numass-main/src/main/java/inr/numass/actions/ReadLegacyDataAction.java create mode 100644 numass-main/src/main/java/inr/numass/actions/TransformDataAction.java diff --git a/numass-core/src/main/java/inr/numass/data/analyzers/AbstractAnalyzer.java b/numass-core/src/main/java/inr/numass/data/analyzers/AbstractAnalyzer.java new file mode 100644 index 00000000..1104b1c2 --- /dev/null +++ b/numass-core/src/main/java/inr/numass/data/analyzers/AbstractAnalyzer.java @@ -0,0 +1,72 @@ +package inr.numass.data.analyzers; + +import hep.dataforge.meta.Meta; +import hep.dataforge.tables.ListTable; +import hep.dataforge.tables.Table; +import hep.dataforge.tables.TableFormat; +import hep.dataforge.tables.TableFormatBuilder; +import inr.numass.data.api.*; +import org.jetbrains.annotations.Nullable; + +import java.util.stream.Stream; + +import static hep.dataforge.tables.XYAdapter.*; +import static inr.numass.data.api.NumassPoint.HV_KEY; + +/** + * Created by darksnake on 11.07.2017. + */ +public abstract class AbstractAnalyzer implements NumassAnalyzer { + public static String[] NAME_LIST = {"length", "count", COUNT_RATE_KEY, COUNT_RATE_ERROR_KEY, "window", "timestamp"}; + public static String[] NAME_LIST_WITH_HV = {HV_KEY, "length", "count", COUNT_RATE_KEY, COUNT_RATE_ERROR_KEY, "window", "timestamp"}; + @Nullable + private final SignalProcessor processor; + + public AbstractAnalyzer(@Nullable SignalProcessor processor) { + this.processor = processor; + } + + public AbstractAnalyzer() { + this.processor = null; + } + + /** + * Return unsorted stream of events including events from frames + * + * @param block + * @return + */ + public Stream getEventStream(NumassBlock block, Meta config) { + if (getProcessor() == null && block.getFrames().count() > 0) { + throw new IllegalArgumentException("Signal processor needed to analyze frames"); + } else { + int loChannel = config.getInt("window.lo", 0); + int upChannel = config.getInt("window.up", Integer.MAX_VALUE); + return Stream.concat(block.getEvents(), block.getFrames().flatMap(getProcessor()::analyze)) + .filter(it -> it.getChanel() >= loChannel && it.getChanel() <= upChannel); + } + } + + + @Override + public Table analyze(NumassSet set, Meta config) { + TableFormat format = new TableFormatBuilder() + .addNumber(HV_KEY, X_VALUE_KEY) + .addNumber("length") + .addNumber("count") + .addNumber(COUNT_RATE_KEY, Y_VALUE_KEY) + .addNumber(COUNT_RATE_ERROR_KEY, Y_ERROR_KEY) + .addColumn("window") + .addTime() + .build(); + + return new ListTable.Builder(format) + .rows(set.getPoints().map(point -> analyze(point, config))) + .build(); + } + + @Nullable + public SignalProcessor getProcessor() { + return processor; + } +} diff --git a/numass-core/src/main/java/inr/numass/data/analyzers/DebunchAnalyzer.java b/numass-core/src/main/java/inr/numass/data/analyzers/DebunchAnalyzer.java new file mode 100644 index 00000000..4b06a456 --- /dev/null +++ b/numass-core/src/main/java/inr/numass/data/analyzers/DebunchAnalyzer.java @@ -0,0 +1,25 @@ +package inr.numass.data.analyzers; + +import hep.dataforge.meta.Meta; +import hep.dataforge.values.Values; +import inr.numass.data.api.NumassBlock; +import inr.numass.data.api.SignalProcessor; +import org.jetbrains.annotations.Nullable; + +/** + * Block analyzer that can perform debunching + * Created by darksnake on 11.07.2017. + */ +public class DebunchAnalyzer extends AbstractAnalyzer { + public DebunchAnalyzer(@Nullable SignalProcessor processor) { + super(processor); + } + + public DebunchAnalyzer() { + } + + @Override + public Values analyze(NumassBlock block, Meta config) { + throw new UnsupportedOperationException("TODO"); + } +} diff --git a/numass-core/src/main/java/inr/numass/data/analyzers/SimpleAnalyzer.java b/numass-core/src/main/java/inr/numass/data/analyzers/SimpleAnalyzer.java index 1e14a069..cf93a5c0 100644 --- a/numass-core/src/main/java/inr/numass/data/analyzers/SimpleAnalyzer.java +++ b/numass-core/src/main/java/inr/numass/data/analyzers/SimpleAnalyzer.java @@ -1,59 +1,31 @@ package inr.numass.data.analyzers; import hep.dataforge.meta.Meta; -import hep.dataforge.tables.*; +import hep.dataforge.tables.ValueMap; import hep.dataforge.values.Values; -import inr.numass.data.api.*; +import inr.numass.data.api.NumassBlock; +import inr.numass.data.api.NumassPoint; +import inr.numass.data.api.SignalProcessor; import org.jetbrains.annotations.Nullable; -import java.util.NavigableMap; -import java.util.TreeMap; -import java.util.concurrent.atomic.AtomicLong; -import java.util.stream.Stream; - -import static hep.dataforge.tables.XYAdapter.*; -import static inr.numass.data.api.NumassPoint.HV_KEY; - /** * A simple event counter * Created by darksnake on 07.07.2017. */ -public class SimpleAnalyzer implements NumassAnalyzer { - public static String[] NAME_LIST = {"length", "count", COUNT_RATE_KEY, COUNT_RATE_ERROR_KEY, "window", "timestamp"}; - public static String[] NAME_LIST_WITH_HV = {HV_KEY, "length", "count", COUNT_RATE_KEY, COUNT_RATE_ERROR_KEY, "window", "timestamp"}; - - @Nullable - private final SignalProcessor processor; +public class SimpleAnalyzer extends AbstractAnalyzer { public SimpleAnalyzer(@Nullable SignalProcessor processor) { - this.processor = processor; + super(processor); } public SimpleAnalyzer() { - this.processor = null; - } - - /** - * Return unsorted stream of events including events from frames - * - * @param block - * @return - */ - protected Stream getEventStream(NumassBlock block) { - if (processor == null && block.getFrames().count() > 0) { - throw new IllegalArgumentException("Signal processor needed to analyze frames"); - } else { - return Stream.concat(block.getEvents(), block.getFrames().flatMap(processor::analyze)); - } } @Override public Values analyze(NumassBlock block, Meta config) { - int loChannel = config.getInt("energy.lo", 0); - int upChannel = config.getInt("energy.up", Integer.MAX_VALUE); - long count = getEventStream(block) - .filter(it -> it.getChanel() >= loChannel && it.getChanel() <= upChannel) - .count(); + int loChannel = config.getInt("window.lo", 0); + int upChannel = config.getInt("window.up", Integer.MAX_VALUE); + long count = getEventStream(block, config).count(); double countRate = (double) count / block.getLength().toMillis() * 1000; double countRateError = Math.sqrt((double) count) / block.getLength().toMillis() * 1000; @@ -84,50 +56,4 @@ public class SimpleAnalyzer implements NumassAnalyzer { } - @Override - public Table analyze(NumassSet set, Meta config) { - TableFormat format = new TableFormatBuilder() - .addNumber(HV_KEY, X_VALUE_KEY) - .addNumber("length") - .addNumber("count") - .addNumber(COUNT_RATE_KEY, Y_VALUE_KEY) - .addNumber(COUNT_RATE_ERROR_KEY, Y_ERROR_KEY) - .addColumn("window") - .addTime() - .build(); - - return new ListTable.Builder(format) - .rows(set.getPoints().map(point -> analyze(point, config))) - .build(); - } - - @Override - public Table getSpectrum(NumassBlock block, Meta config) { - TableFormat format = new TableFormatBuilder() - .addNumber("channel", X_VALUE_KEY) - .addNumber("count") - .addNumber(COUNT_RATE_KEY, Y_VALUE_KEY) - .addNumber(COUNT_RATE_ERROR_KEY, Y_ERROR_KEY) - .updateMeta(metaBuilder -> metaBuilder.setNode("config", config)) - .build(); - NavigableMap map = new TreeMap<>(); - getEventStream(block).forEach(event -> { - if (map.containsKey(event.getChanel())) { - map.get(event.getChanel()).incrementAndGet(); - } else { - map.put(event.getChanel(), new AtomicLong(1)); - } - }); - return new ListTable.Builder(format) - .rows(map.entrySet().stream() - .map(entry -> - new ValueMap(format.namesAsArray(), - entry.getKey(), - entry.getValue(), - entry.getValue().get() / block.getLength().toMillis() * 1000, - Math.sqrt(entry.getValue().get()) / block.getLength().toMillis() * 1000 - ) - ) - ).build(); - } } diff --git a/numass-core/src/main/java/inr/numass/data/analyzers/SmartAnalyzer.java b/numass-core/src/main/java/inr/numass/data/analyzers/SmartAnalyzer.java new file mode 100644 index 00000000..de36fb3c --- /dev/null +++ b/numass-core/src/main/java/inr/numass/data/analyzers/SmartAnalyzer.java @@ -0,0 +1,42 @@ +package inr.numass.data.analyzers; + +import hep.dataforge.meta.Meta; +import hep.dataforge.values.Values; +import inr.numass.data.api.NumassBlock; +import inr.numass.data.api.SignalProcessor; +import org.jetbrains.annotations.Nullable; + +/** + * An analyzer dispatcher which uses different analyzer for different meta + * Created by darksnake on 11.07.2017. + */ +public class SmartAnalyzer extends AbstractAnalyzer { + private SimpleAnalyzer simpleAnalyzer = new SimpleAnalyzer(); + private DebunchAnalyzer debunchAnalyzer = new DebunchAnalyzer(); + private TimeAnalyzer timeAnalyzer = new TimeAnalyzer(); + + public SmartAnalyzer(@Nullable SignalProcessor processor) { + super(processor); + this.simpleAnalyzer = new SimpleAnalyzer(processor); + this.debunchAnalyzer = new DebunchAnalyzer(processor); + this.timeAnalyzer = new TimeAnalyzer(processor); + } + + public SmartAnalyzer() { + } + + @Override + public Values analyze(NumassBlock block, Meta config) { + //TODO do something more... smart... using information from point if block is point + switch (config.getString("type", "simple")) { + case "simple": + return simpleAnalyzer.analyze(block, config); + case "time": + return timeAnalyzer.analyze(block, config); + case "debunch": + return debunchAnalyzer.analyze(block, config); + default: + throw new IllegalArgumentException("Analyzer not found"); + } + } +} diff --git a/numass-core/src/main/java/inr/numass/data/analyzers/TimeAnalyzer.java b/numass-core/src/main/java/inr/numass/data/analyzers/TimeAnalyzer.java new file mode 100644 index 00000000..ce5c9aea --- /dev/null +++ b/numass-core/src/main/java/inr/numass/data/analyzers/TimeAnalyzer.java @@ -0,0 +1,25 @@ +package inr.numass.data.analyzers; + +import hep.dataforge.meta.Meta; +import hep.dataforge.values.Values; +import inr.numass.data.api.NumassBlock; +import inr.numass.data.api.SignalProcessor; +import org.jetbrains.annotations.Nullable; + +/** + * Created by darksnake on 11.07.2017. + */ +public class TimeAnalyzer extends AbstractAnalyzer { + + public TimeAnalyzer(@Nullable SignalProcessor processor) { + super(processor); + } + + public TimeAnalyzer() { + } + + @Override + public Values analyze(NumassBlock block, Meta config) { + throw new UnsupportedOperationException("TODO"); + } +} diff --git a/numass-core/src/main/java/inr/numass/data/api/NumassAnalyzer.java b/numass-core/src/main/java/inr/numass/data/api/NumassAnalyzer.java index 46a98f9d..66596edc 100644 --- a/numass-core/src/main/java/inr/numass/data/api/NumassAnalyzer.java +++ b/numass-core/src/main/java/inr/numass/data/api/NumassAnalyzer.java @@ -1,14 +1,52 @@ package inr.numass.data.api; import hep.dataforge.meta.Meta; -import hep.dataforge.tables.Table; +import hep.dataforge.tables.*; import hep.dataforge.values.Values; +import inr.numass.data.analyzers.SmartAnalyzer; + +import java.util.NavigableMap; +import java.util.TreeMap; +import java.util.concurrent.atomic.AtomicLong; +import java.util.stream.Stream; + +import static hep.dataforge.tables.XYAdapter.*; /** * A general raw data analysis utility. Could have different implementations * Created by darksnake on 06-Jul-17. */ public interface NumassAnalyzer { + + static Table getSpectrum(NumassBlock block, Meta config) { + TableFormat format = new TableFormatBuilder() + .addNumber("channel", X_VALUE_KEY) + .addNumber("count") + .addNumber(COUNT_RATE_KEY, Y_VALUE_KEY) + .addNumber(COUNT_RATE_ERROR_KEY, Y_ERROR_KEY) + .updateMeta(metaBuilder -> metaBuilder.setNode("config", config)) + .build(); + NavigableMap map = new TreeMap<>(); + new SmartAnalyzer().getEventStream(block, config).forEach(event -> { + if (map.containsKey(event.getChanel())) { + map.get(event.getChanel()).incrementAndGet(); + } else { + map.put(event.getChanel(), new AtomicLong(1)); + } + }); + return new ListTable.Builder(format) + .rows(map.entrySet().stream() + .map(entry -> + new ValueMap(format.namesAsArray(), + entry.getKey(), + entry.getValue(), + entry.getValue().get() / block.getLength().toMillis() * 1000, + Math.sqrt(entry.getValue().get()) / block.getLength().toMillis() * 1000 + ) + ) + ).build(); + } + String COUNT_RATE_KEY = "cr"; String COUNT_RATE_ERROR_KEY = "crErr"; @@ -21,21 +59,21 @@ public interface NumassAnalyzer { */ Values analyze(NumassBlock block, Meta config); + /** + * Return unsorted stream of events including events from frames + * + * @param block + * @return + */ + Stream getEventStream(NumassBlock block, Meta config); + /** * Analyze the whole set. And return results as a table + * * @param set * @param config * @return */ Table analyze(NumassSet set, Meta config); - /** - * Generate energy spectrum for the given block - * @param block - * @param config - * @return - */ - Table getSpectrum(NumassBlock block, Meta config); - - } diff --git a/numass-core/src/main/java/inr/numass/data/storage/ProtoNumassPoint.java b/numass-core/src/main/java/inr/numass/data/storage/ProtoNumassPoint.java index 12ac78c2..efd7137b 100644 --- a/numass-core/src/main/java/inr/numass/data/storage/ProtoNumassPoint.java +++ b/numass-core/src/main/java/inr/numass/data/storage/ProtoNumassPoint.java @@ -17,31 +17,27 @@ import java.util.stream.IntStream; import java.util.stream.Stream; /** + * Protobuf based numass point * Created by darksnake on 09.07.2017. */ public class ProtoNumassPoint implements NumassPoint { private final Envelope envelope; - NumassProto.Point point; - public ProtoNumassPoint(Envelope envelope) { this.envelope = envelope; } private NumassProto.Point getPoint() { - if (point == null) { - try (InputStream stream = envelope.getData().getStream()) { - point = NumassProto.Point.parseFrom(stream); - } catch (IOException ex) { - throw new RuntimeException("Failed to read point via protbuf"); - } + try (InputStream stream = envelope.getData().getStream()) { + return NumassProto.Point.parseFrom(stream); + } catch (IOException ex) { + throw new RuntimeException("Failed to read point via protobuf"); } - return point; } @Override public Stream getBlocks() { - return point.getChannelsList().stream().flatMap(channel -> + return getPoint().getChannelsList().stream().flatMap(channel -> channel.getBlocksList().stream().map(block -> new ProtoBlock((int) channel.getNum(), block)) ); } diff --git a/numass-main/src/main/groovy/inr/numass/scripts/FindBorder.groovy b/numass-main/src/main/groovy/inr/numass/scripts/FindBorder.groovy index 632c2559..8155c7bc 100644 --- a/numass-main/src/main/groovy/inr/numass/scripts/FindBorder.groovy +++ b/numass-main/src/main/groovy/inr/numass/scripts/FindBorder.groovy @@ -8,7 +8,6 @@ package inr.numass.scripts import hep.dataforge.grind.GrindMetaBuilder import hep.dataforge.meta.Meta -import inr.numass.actions.FindBorderAction import inr.numass.data.storage.NumassDataLoader File dataDir = new File("D:\\Work\\Numass\\data\\2016_04\\T2_data\\Fill_2_2\\set_6_e26d123e54010000") diff --git a/numass-main/src/main/java/inr/numass/actions/AnalyzeDataAction.java b/numass-main/src/main/java/inr/numass/actions/AnalyzeDataAction.java new file mode 100644 index 00000000..e1ef78a4 --- /dev/null +++ b/numass-main/src/main/java/inr/numass/actions/AnalyzeDataAction.java @@ -0,0 +1,37 @@ +package inr.numass.actions; + +import hep.dataforge.actions.OneToOneAction; +import hep.dataforge.context.Context; +import hep.dataforge.description.TypedActionDef; +import hep.dataforge.description.ValueDef; +import hep.dataforge.io.ColumnedDataWriter; +import hep.dataforge.meta.Laminate; +import hep.dataforge.tables.Table; +import inr.numass.data.analyzers.SmartAnalyzer; +import inr.numass.data.api.NumassAnalyzer; +import inr.numass.data.api.NumassSet; + +import java.io.OutputStream; + +import static hep.dataforge.values.ValueType.NUMBER; +import static hep.dataforge.values.ValueType.STRING; + +/** + * The action performs the readout of data and collection of count rate into a table + * Created by darksnake on 11.07.2017. + */ +@TypedActionDef(name = "numass.analyze", inputType = NumassSet.class, outputType = Table.class) +@ValueDef(name = "window.lo", type = {NUMBER, STRING}, def = "0", info = "Lower bound for window") +@ValueDef(name = "window.up", type = {NUMBER, STRING}, def = "10000", info = "Upper bound for window") +public class AnalyzeDataAction extends OneToOneAction { + @Override + protected Table execute(Context context, String name, NumassSet input, Laminate inputMeta) { + //TODO add processor here + NumassAnalyzer analyzer = new SmartAnalyzer(); + Table res = analyzer.analyze(input,inputMeta); + OutputStream stream = buildActionOutput(context, name); + + ColumnedDataWriter.writeTable(stream, data, head); + return res; + } +} diff --git a/numass-main/src/main/java/inr/numass/actions/FindBorderAction.java b/numass-main/src/main/java/inr/numass/actions/FindBorderAction.java deleted file mode 100644 index 7190a882..00000000 --- a/numass-main/src/main/java/inr/numass/actions/FindBorderAction.java +++ /dev/null @@ -1,129 +0,0 @@ -/* - * Copyright 2015 Alexander Nozik. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package inr.numass.actions; - -import hep.dataforge.actions.OneToOneAction; -import hep.dataforge.context.Context; -import hep.dataforge.description.TypedActionDef; -import hep.dataforge.exceptions.ContentException; -import hep.dataforge.io.ColumnedDataWriter; -import hep.dataforge.meta.Laminate; -import hep.dataforge.tables.ListTable; -import hep.dataforge.tables.Table; -import hep.dataforge.tables.ValueMap; -import hep.dataforge.values.Value; -import org.apache.commons.math3.analysis.UnivariateFunction; - -import java.io.OutputStream; -import java.util.HashMap; -import java.util.Map; -import java.util.stream.Collectors; - -/** - * @author Darksnake - */ -@TypedActionDef(name = "findBorder", inputType = NMFile.class, outputType = Table.class) -public class FindBorderAction extends OneToOneAction { - - private final static String[] names = {"U", "80%", "90%", "95%", "99%"}; - private final static double[] percents = {0.8, 0.9, 0.95, 0.99}; - - private UnivariateFunction normCorrection = e -> 1 + 13.265 * Math.exp(-e / 2343.4); - - @Override - protected Table execute(Context context, String name, NumassData source, Laminate meta) throws ContentException { - report(context, name, "File {} started", source.getName()); - - int upperBorder = meta.getInt("upper", 4094); - int lowerBorder = meta.getInt("lower", 0); - double substractReference = meta.getDouble("reference", 0); - - NumassPoint referencePoint = null; - if (substractReference > 0) { - referencePoint = source.getByVoltage(substractReference); - if (referencePoint == null) { - report(context, name, "Reference point {} not found", substractReference); - } - } - - ListTable.Builder dataBuilder = new ListTable.Builder(names); - - fill(dataBuilder, source, lowerBorder, upperBorder, referencePoint); - Table bData = dataBuilder.build(); - - OutputStream stream = buildActionOutput(context, name); - - ColumnedDataWriter.writeTable(stream, bData, String.format("%s : lower = %d upper = %d", name, lowerBorder, upperBorder)); - - report(context, name, "File {} completed", source.getName()); - return bData; - } - - private double getNorm(Map spectrum, int lower, int upper) { - double res = 0; - for (Map.Entry entry : spectrum.entrySet()) { - if ((entry.getKey() >= lower) && (entry.getKey() <= upper)) { - res += entry.getValue(); - } - } - - return res; - } - - private void fill(ListTable.Builder dataBuilder, NumassData file, int lower, int upper, NumassPoint reference) { - for (NumassPoint point : file) { - if ((reference != null) && (point.getVoltage() == reference.getVoltage())) { - continue; - } - //создаем основу для будущей точки - HashMap map = new HashMap<>(); - map.put(names[0], Value.of(point.getVoltage())); - Map spectrum; - if (reference != null) { - - Map sp = point.getMap(0, true); - Map referenceSpectrum = reference.getMap(0, true); - - spectrum = sp.entrySet().stream() - .collect(Collectors.toMap(entry -> entry.getKey(), entry -> Math.max(entry.getValue() - referenceSpectrum.get(entry.getKey()), 0))); - } else { - spectrum = point.getMap(0, true); - } - double norm = getNorm(spectrum, lower, upper) * normCorrection.value(point.getVoltage()); - double counter = 0; - int chanel = upper; - while (chanel > lower) { - chanel--; - counter += spectrum.get((double) chanel); - for (int i = 0; i < percents.length; i++) { - if (counter / norm > percents[i]) { - if (!map.containsKey(names[i + 1])) { - map.put(names[i + 1], Value.of(chanel)); - } - } - } - } - for (String n : names) { - if (!map.containsKey(n)) { - map.put(n, Value.of(lower)); - } - } - - dataBuilder.row(new ValueMap(map)); - } - } - -} diff --git a/numass-main/src/main/java/inr/numass/actions/JoinNumassDataAction.java b/numass-main/src/main/java/inr/numass/actions/JoinNumassDataAction.java deleted file mode 100644 index fb0eeff3..00000000 --- a/numass-main/src/main/java/inr/numass/actions/JoinNumassDataAction.java +++ /dev/null @@ -1,37 +0,0 @@ -package inr.numass.actions; - -import hep.dataforge.actions.ManyToOneAction; -import hep.dataforge.context.Context; -import hep.dataforge.description.TypedActionDef; -import hep.dataforge.meta.Laminate; - -import java.util.Collection; -import java.util.Map; -import java.util.stream.IntStream; - -/** - * Created by darksnake on 04-Nov-16. - */ -@TypedActionDef(name = "joinData", inputType = NumassData.class, outputType = NumassData.class, - info = "Join a number of numass data files into one single file via spectrum summing") -public class JoinNumassDataAction extends ManyToOneAction { - - @Override - protected NumassData execute(Context context, String nodeName, Map input, Laminate meta) { - throw new UnsupportedOperationException("not implemented"); - } - - private NumassPoint joinPoint(Collection points) { - return points.stream().reduce((p1, p2) -> { - if (p1.getVoltage() != p2.getVoltage()) { - throw new RuntimeException("Can't sum points with different Uset"); - } - return new NumassPointImpl( - (p1.getVoltage() + p2.getVoltage()) / 2, - p1.getStartTime(), - p1.getLength() + p2.getLength(), - IntStream.range(0, p1.getSpectrum().length).map(i -> p1.getSpectrum()[i] * p2.getSpectrum()[i]).toArray() - ); - }).get(); - } -} diff --git a/numass-main/src/main/java/inr/numass/actions/MergeDataAction.java b/numass-main/src/main/java/inr/numass/actions/MergeDataAction.java index a7dc7d67..fe409844 100644 --- a/numass-main/src/main/java/inr/numass/actions/MergeDataAction.java +++ b/numass-main/src/main/java/inr/numass/actions/MergeDataAction.java @@ -161,7 +161,7 @@ public class MergeDataAction extends ManyToOneAction { res.add(curPoint); }); - return new ListTable(TableFormat.forNames(parnames), res); + return new ListTable(MetaTableFormat.forNames(parnames), res); } diff --git a/numass-main/src/main/java/inr/numass/actions/PrepareDataAction.java b/numass-main/src/main/java/inr/numass/actions/PrepareDataAction.java index 504e59af..1e779569 100644 --- a/numass-main/src/main/java/inr/numass/actions/PrepareDataAction.java +++ b/numass-main/src/main/java/inr/numass/actions/PrepareDataAction.java @@ -25,14 +25,10 @@ import hep.dataforge.io.ColumnedDataWriter; import hep.dataforge.io.XMLMetaWriter; import hep.dataforge.meta.Laminate; import hep.dataforge.meta.Meta; -import hep.dataforge.tables.ListTable; -import hep.dataforge.tables.Table; -import hep.dataforge.tables.TableFormat; -import hep.dataforge.tables.ValueMap; +import hep.dataforge.tables.*; import hep.dataforge.values.Values; -import inr.numass.data.NumassPoint; +import inr.numass.data.api.NumassPoint; import inr.numass.data.api.NumassSet; -import inr.numass.data.storage.NumassDataLoader; import inr.numass.debunch.DebunchReport; import inr.numass.debunch.FrameAnalizer; import inr.numass.utils.ExpressionUtils; @@ -77,7 +73,7 @@ public class PrepareDataAction extends OneToOneAction { protected ListTable execute(Context context, String name, NumassSet dataFile, Laminate meta) { // log.report("File %s started", dataFile.getName()); - int upper = meta.getInt("upperWindow", RawNMPoint.MAX_CHANEL - 1); + int upper = meta.getInt("upperWindow", Integer.MAX_VALUE); List corrections = new ArrayList<>(); if (meta.hasValue("deadTime")) { @@ -107,16 +103,16 @@ public class PrepareDataAction extends OneToOneAction { utransform = Function.identity(); } - if (meta.hasMeta("debunch")) { - if (dataFile instanceof NumassDataLoader) { - dataFile = ((NumassDataLoader) dataFile).applyRawTransformation(raw -> debunch(context, raw, meta.getMeta("debunch"))); - } else { - throw new RuntimeException("Debunch not available"); - } - } +// if (meta.hasMeta("debunch")) { +// if (dataFile instanceof NumassDataLoader) { +// dataFile = ((NumassDataLoader) dataFile).applyRawTransformation(raw -> debunch(context, raw, meta.getMeta("debunch"))); +// } else { +// throw new RuntimeException("Debunch not available"); +// } +// } List dataList = new ArrayList<>(); - for (NumassPoint point : dataFile) { + dataFile.getPoints().forEach( point -> { long total = point.getTotalCount(); double uset = utransform.apply(point.getVoltage()); @@ -148,15 +144,15 @@ public class PrepareDataAction extends OneToOneAction { Instant timestamp = point.getStartTime(); dataList.add(new ValueMap(parnames, new Object[]{uset, uread, time, total, wind, correctionFactor, cr, crErr, timestamp})); - } + }); TableFormat format; if (!dataList.isEmpty()) { //Генерируем автоматический формат по первой строчке - format = TableFormat.forPoint(dataList.get(0)); + format = MetaTableFormat.forPoint(dataList.get(0)); } else { - format = TableFormat.forNames(parnames); + format = MetaTableFormat.forNames(parnames); } String head; diff --git a/numass-main/src/main/java/inr/numass/actions/ReadLegacyDataAction.java b/numass-main/src/main/java/inr/numass/actions/ReadLegacyDataAction.java deleted file mode 100644 index ea90b5df..00000000 --- a/numass-main/src/main/java/inr/numass/actions/ReadLegacyDataAction.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright 2015 Alexander Nozik. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package inr.numass.actions; - -import hep.dataforge.actions.OneToOneAction; -import hep.dataforge.context.Context; -import hep.dataforge.data.binary.Binary; -import hep.dataforge.description.NodeDef; -import hep.dataforge.description.TypedActionDef; -import hep.dataforge.description.ValueDef; -import hep.dataforge.exceptions.ContentException; -import hep.dataforge.meta.Laminate; - -import static hep.dataforge.values.ValueType.NUMBER; -import static inr.numass.NumassIO.getNumassData; - -/** - * - * @author Darksnake - */ -@TypedActionDef(name = "readData", - inputType = Binary.class, outputType = NMFile.class, info = "Read binary numass data file") -@ValueDef(name = "fileName", info = "The name of the file. By default equals file name.") -@ValueDef(name = "HVdev", info = "Divider for HV measurements. Should be set to 1.0 for numass data 2014", - def = "2.468555393226049", type = {NUMBER}) -@ValueDef(name = "noUset", info = "If 'true', then Uset = Uread") -@NodeDef(name = "debunch", target = "class::inr.numass.actions.DebunchAction", info = "If given, governs debunching") -public class ReadLegacyDataAction extends OneToOneAction { - - @Override - protected NMFile execute(Context context, String name, Binary source, Laminate meta) throws ContentException { -// log.logString("File '%s' started", source.getName()); - RawNMFile raw = getNumassData(source, meta); -// if (meta.getBoolean("paw", false)) { -// raw.generatePAW(buildActionOutput(context, name + ".paw")); -// } - - if (meta.hasMeta("debunch")) { - DebunchAction debunch = new DebunchAction(); - Laminate laminate = new Laminate(meta.getMeta("debunch")) - .setDescriptor(debunch.getDescriptor()); - raw = debunch.execute(context, name, raw, laminate); - } - - NMFile result = new NMFile(raw); - - return result; - } - -} diff --git a/numass-main/src/main/java/inr/numass/actions/ShowEnergySpectrumAction.java b/numass-main/src/main/java/inr/numass/actions/ShowEnergySpectrumAction.java index ef84e6ee..23c9cf79 100644 --- a/numass-main/src/main/java/inr/numass/actions/ShowEnergySpectrumAction.java +++ b/numass-main/src/main/java/inr/numass/actions/ShowEnergySpectrumAction.java @@ -19,6 +19,7 @@ import hep.dataforge.plots.data.XYPlottable; import hep.dataforge.tables.*; import hep.dataforge.values.ValueType; import hep.dataforge.values.Values; +import inr.numass.data.api.NumassSet; import java.io.OutputStream; import java.util.*; @@ -28,19 +29,14 @@ import java.util.stream.Collectors; * * @author Alexander Nozik */ -@TypedActionDef(inputType = NumassData.class, outputType = Table.class, name = "energySpectrum", info = "Generate output table and optionally plot for detector energy spectra") -public class ShowEnergySpectrumAction extends OneToOneAction { +@TypedActionDef(inputType = NumassSet.class, outputType = Table.class, name = "energySpectrum", info = "Generate output table and optionally plot for detector energy spectra") +public class ShowEnergySpectrumAction extends OneToOneAction { @Override - protected Table execute(Context context, String name, NumassData input, Laminate inputMeta) { + protected Table execute(Context context, String name, NumassSet input, Laminate inputMeta) { int binning = inputMeta.getInt("binning", 20); boolean normalize = inputMeta.getBoolean("normalize", true); - List points = input.getNMPoints(); - if (points.isEmpty()) { - getLogger(inputMeta).error("Empty data"); - return null; - } //build header List names = new ArrayList<>(); diff --git a/numass-main/src/main/java/inr/numass/actions/SubstractSpectrumAction.java b/numass-main/src/main/java/inr/numass/actions/SubstractSpectrumAction.java index 08815bf1..5695ef5f 100644 --- a/numass-main/src/main/java/inr/numass/actions/SubstractSpectrumAction.java +++ b/numass-main/src/main/java/inr/numass/actions/SubstractSpectrumAction.java @@ -34,9 +34,9 @@ public class SubstractSpectrumAction extends OneToOneAction { File referenceFile = context.io().getFile(referencePath); Table referenceTable = new ColumnedDataReader(referenceFile).toTable(); ListTable.Builder builder = new ListTable.Builder(input.getFormat()); - input.stream().forEach(point -> { + input.getRows().forEach(point -> { ValueMap.Builder pointBuilder = new ValueMap.Builder(point); - Optional referencePoint = referenceTable.stream() + Optional referencePoint = referenceTable.getRows() .filter(p -> Math.abs(p.getDouble("Uset") - point.getDouble("Uset")) < 0.1).findFirst(); if (referencePoint.isPresent()) { pointBuilder.putValue("CR", Math.max(0, point.getDouble("CR") - referencePoint.get().getDouble("CR"))); diff --git a/numass-main/src/main/java/inr/numass/actions/SummaryAction.java b/numass-main/src/main/java/inr/numass/actions/SummaryAction.java index eb4cdc55..c49035ee 100644 --- a/numass-main/src/main/java/inr/numass/actions/SummaryAction.java +++ b/numass-main/src/main/java/inr/numass/actions/SummaryAction.java @@ -26,8 +26,8 @@ import hep.dataforge.meta.Laminate; import hep.dataforge.meta.Meta; import hep.dataforge.stat.fit.FitState; import hep.dataforge.tables.ListTable; +import hep.dataforge.tables.MetaTableFormat; import hep.dataforge.tables.Table; -import hep.dataforge.tables.TableFormat; import hep.dataforge.tables.ValueMap; import hep.dataforge.values.Value; import hep.dataforge.values.Values; @@ -75,7 +75,7 @@ public class SummaryAction extends ManyToOneAction { } names[names.length - 1] = "chi2"; - ListTable.Builder res = new ListTable.Builder(TableFormat.forNames(names)); + ListTable.Builder res = new ListTable.Builder(MetaTableFormat.forNames(names)); double[] weights = new double[parNames.length]; Arrays.fill(weights, 0); diff --git a/numass-main/src/main/java/inr/numass/actions/TransformDataAction.java b/numass-main/src/main/java/inr/numass/actions/TransformDataAction.java new file mode 100644 index 00000000..7c43cd68 --- /dev/null +++ b/numass-main/src/main/java/inr/numass/actions/TransformDataAction.java @@ -0,0 +1,114 @@ +package inr.numass.actions; + +import hep.dataforge.actions.OneToOneAction; +import hep.dataforge.context.Context; +import hep.dataforge.description.ValueDef; +import hep.dataforge.meta.Laminate; +import hep.dataforge.meta.Meta; +import hep.dataforge.tables.Table; +import hep.dataforge.values.Values; +import inr.numass.utils.ExpressionUtils; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.function.UnaryOperator; +import java.util.stream.Collectors; + +import static hep.dataforge.values.ValueType.NUMBER; +import static hep.dataforge.values.ValueType.STRING; +import static inr.numass.utils.TritiumUtils.pointExpression; + +/** + * Apply corrections and transformations to analyzed data + * Created by darksnake on 11.07.2017. + */ +public class TransformDataAction extends OneToOneAction { + @Override + protected Table execute(Context context, String name, Table input, Laminate meta) { + UnaryOperator transformation = UnaryOperator.identity(); + + + + + + List corrections = new ArrayList<>(); + if (meta.optMeta("correction").isPresent()) { + corrections.addAll(meta.getMetaList("correction").stream() + .map((Function) this::makeCorrection) + .collect(Collectors.toList())); + } + + if (meta.hasValue("correction")) { + final String correction = meta.getString("correction"); + corrections.add((point) -> pointExpression(correction, point)); + } + + Function utransform; + if (meta.hasValue("utransform")) { + String func = meta.getString("utransform"); + utransform = u -> { + Map binding = new HashMap<>(); + binding.put("U", u); + return ExpressionUtils.function(func, binding); + }; + } else { + utransform = Function.identity(); + } + } + + + @ValueDef(name = "value", type = {NUMBER, STRING}, info = "Value or function to multiply count rate") + @ValueDef(name = "err", type = {NUMBER, STRING}, info = "error of the value") + private Correction makeCorrection(Meta corrMeta) { + final String expr = corrMeta.getString("value"); + final String errExpr = corrMeta.getString("err", ""); + return new Correction() { + @Override + public double corr(Values point) { + return pointExpression(expr, point); + } + + @Override + public double corrErr(Values point) { + if (errExpr.isEmpty()) { + return 0; + } else { + return pointExpression(errExpr, point); + } + } + }; + } + + private interface Correction { + /** + * correction coefficient + * + * @param point + * @return + */ + double corr(Values point); + + /** + * correction coefficient uncertainty + * + * @param point + * @return + */ + default double corrErr(Values point) { + return 0; + } + + default double relativeErr(Values point) { + double corrErr = corrErr(point); + if (corrErr == 0) { + return 0; + } else { + return corrErr / corr(point); + } + } + } + +} diff --git a/numass-main/src/main/java/inr/numass/models/NamedSpectrumCaching.java b/numass-main/src/main/java/inr/numass/models/NamedSpectrumCaching.java index 65d73aa3..6c235a18 100644 --- a/numass-main/src/main/java/inr/numass/models/NamedSpectrumCaching.java +++ b/numass-main/src/main/java/inr/numass/models/NamedSpectrumCaching.java @@ -54,7 +54,7 @@ public class NamedSpectrumCaching extends AbstractParametricFunction { this.a = a; this.b = b; this.source = spectrum; - spectrumDerivCache = new HashMap<>(source.size()); + spectrumDerivCache = new HashMap<>(source.getNames().size()); // spectrumDerivCache = new CacheElement[source.getDimension()]; } diff --git a/numass-main/src/main/java/inr/numass/tasks/NumassSubstractEmptySourceTask.java b/numass-main/src/main/java/inr/numass/tasks/NumassSubstractEmptySourceTask.java index 353d7acf..1220e122 100644 --- a/numass-main/src/main/java/inr/numass/tasks/NumassSubstractEmptySourceTask.java +++ b/numass-main/src/main/java/inr/numass/tasks/NumassSubstractEmptySourceTask.java @@ -83,9 +83,9 @@ public class NumassSubstractEmptySourceTask extends AbstractTask { private Table subtract(Table merge, Table empty) { ListTable.Builder builder = new ListTable.Builder(merge.getFormat()); - merge.stream().forEach(point -> { + merge.getRows().forEach(point -> { ValueMap.Builder pointBuilder = new ValueMap.Builder(point); - Optional referencePoint = empty.stream() + Optional referencePoint = empty.getRows() .filter(p -> Math.abs(p.getDouble("Uset") - point.getDouble("Uset")) < 0.1).findFirst(); if (referencePoint.isPresent()) { pointBuilder.putValue("CR", Math.max(0, point.getDouble("CR") - referencePoint.get().getDouble("CR"))); diff --git a/numass-main/src/main/java/inr/numass/tasks/NumassTableFilterTask.java b/numass-main/src/main/java/inr/numass/tasks/NumassTableFilterTask.java index f01259f9..94aef4f9 100644 --- a/numass-main/src/main/java/inr/numass/tasks/NumassTableFilterTask.java +++ b/numass-main/src/main/java/inr/numass/tasks/NumassTableFilterTask.java @@ -73,7 +73,7 @@ public class NumassTableFilterTask extends SingleActionTask { for (String field : dp.getNames()) { Value val = dp.getValue(field); Object obj; - switch (val.valueType()) { + switch (val.getType()) { case BOOLEAN: obj = val.booleanValue(); break; diff --git a/numass-main/src/main/java/inr/numass/utils/TritiumUtils.java b/numass-main/src/main/java/inr/numass/utils/TritiumUtils.java index 22b07f4c..2ef8ddc1 100644 --- a/numass-main/src/main/java/inr/numass/utils/TritiumUtils.java +++ b/numass-main/src/main/java/inr/numass/utils/TritiumUtils.java @@ -15,6 +15,7 @@ */ package inr.numass.utils; +import hep.dataforge.values.Values; import org.apache.commons.math3.analysis.UnivariateFunction; import java.util.HashMap; @@ -68,12 +69,14 @@ public class TritiumUtils { * @param point * @return */ - public static double pointExpression(String expression, NumassPoint point) { + public static double pointExpression(String expression, Values point) { Map exprParams = new HashMap<>(); - exprParams.put("T", point.getLength()); - exprParams.put("U", point.getVoltage()); - exprParams.put("cr", ((double) point.getTotalCount()) / point.getLength()); - exprParams.put("point", point); + //Adding all point values to expression parameters + point.getNames().forEach(name-> exprParams.put(name,point.getValue(name).value())); + //Adding aliases for commonly used parameters + exprParams.put("T", point.getDouble("length")); + exprParams.put("U", point.getDouble("voltage")); + return ExpressionUtils.function(expression, exprParams); } } diff --git a/numass-server/src/main/java/inr/numass/server/HandlerUtils.java b/numass-server/src/main/java/inr/numass/server/HandlerUtils.java index 1487d979..328a8416 100644 --- a/numass-server/src/main/java/inr/numass/server/HandlerUtils.java +++ b/numass-server/src/main/java/inr/numass/server/HandlerUtils.java @@ -20,7 +20,7 @@ public class HandlerUtils { for (String state : states.getStateNames()) { Value val = states.getValue(state); String color; - switch (val.valueType()) { + switch (val.getType()) { case NUMBER: color = "blue"; break;