From 530bc869d3ed48bb92a445f44d74b7f074d8e37c Mon Sep 17 00:00:00 2001 From: darksnake Date: Mon, 30 Jan 2017 16:59:46 +0300 Subject: [PATCH] Spectrum merger --- .../inr/numass/scripts/Underflow.groovy | 29 +++++++----- .../inr/numass/actions/DebunchAction.java | 3 +- .../inr/numass/actions/FindBorderAction.java | 2 +- .../inr/numass/actions/MergeDataAction.java | 2 +- .../inr/numass/actions/PrepareDataAction.java | 2 +- .../java/inr/numass/actions/SlicedData.java | 20 ++++----- .../inr/numass/utils/UnderflowCorrection.java | 9 ++-- .../main/java/inr/numass/storage/NMFile.java | 42 +++++------------ .../java/inr/numass/storage/NumassData.java | 21 ++++++--- .../inr/numass/storage/NumassDataLoader.java | 9 ++-- .../inr/numass/storage/NumassDataReader.java | 5 +-- .../inr/numass/storage/NumassDataUtils.java | 43 ++++++++++++++++++ .../inr/numass/storage/NumassStorage.java | 1 + .../java/inr/numass/storage/RawNMFile.java | 45 ++++++++++++------- 14 files changed, 142 insertions(+), 91 deletions(-) create mode 100644 numass-storage/src/main/java/inr/numass/storage/NumassDataUtils.java diff --git a/numass-main/src/main/groovy/inr/numass/scripts/Underflow.groovy b/numass-main/src/main/groovy/inr/numass/scripts/Underflow.groovy index fcbc1087..98797e7e 100644 --- a/numass-main/src/main/groovy/inr/numass/scripts/Underflow.groovy +++ b/numass-main/src/main/groovy/inr/numass/scripts/Underflow.groovy @@ -7,20 +7,29 @@ package inr.numass.scripts import hep.dataforge.io.ColumnedDataWriter +import hep.dataforge.storage.commons.StorageUtils import hep.dataforge.tables.Table -import inr.numass.storage.NumassData -import inr.numass.storage.NumassDataLoader +import inr.numass.storage.NMPoint +import inr.numass.storage.NumassDataUtils +import inr.numass.storage.NumassStorage import inr.numass.utils.UnderflowCorrection -//File dataDir = new File("D:\\Work\\Numass\\data\\2016_04\\T2_data\\Fill_2_2\\set_7_b2a3433e54010000") -//File dataDir = new File("D:\\Work\\Numass\\data\\2016_04\\T2_data\\Fill_2_2\\set_6_e26d123e54010000") //File dataDir = new File("D:\\Work\\Numass\\data\\2016_10\\Fill_1\\set_28") -File dataDir = new File("D:\\Work\\Numass\\data\\2016_10\\Fill_2_wide\\set_31") -if(!dataDir.exists()){ - println "dataDir directory does not exist" -} -NumassData data = NumassDataLoader.fromLocalDir(null, dataDir) -//NumassData data = NMFile.readFile(new File("D:\\Work\\Numass\\sterilie2013-2014\\dat\\2013\\SCAN06.DAT" )) +//File dataDir = new File("D:\\Work\\Numass\\data\\2016_10\\Fill_2_wide\\set_31") + +File rootDir = new File("D:\\Work\\Numass\\data\\2016_10\\Fill_2_wide") + +NumassStorage storage = NumassStorage.buildLocalNumassRoot(rootDir, true); + +Iterable data = NumassDataUtils.sumSpectra( + StorageUtils.loaderStream(storage).map { it.value }.filter { it.name.matches("set_.{2,3}") } +) + +//if(!dataDir.exists()){ +// println "dataDir directory does not exist" +//} +//NumassData data = NumassDataLoader.fromLocalDir(null, dataDir) +////NumassData data = NMFile.readFile(new File("D:\\Work\\Numass\\sterilie2013-2014\\dat\\2013\\SCAN06.DAT" )) Table t = new UnderflowCorrection().fitAllPoints(data, 400, 650, 3100, 20); ColumnedDataWriter.writeDataSet(System.out, t, "underflow parameters") diff --git a/numass-main/src/main/java/inr/numass/actions/DebunchAction.java b/numass-main/src/main/java/inr/numass/actions/DebunchAction.java index 9e6202ae..2893e2a8 100644 --- a/numass-main/src/main/java/inr/numass/actions/DebunchAction.java +++ b/numass-main/src/main/java/inr/numass/actions/DebunchAction.java @@ -49,8 +49,7 @@ public class DebunchAction extends OneToOneAction { double framelength = meta.getDouble("framelength", 5); double maxCR = meta.getDouble("maxcr", 100d); - RawNMFile res = new RawNMFile(source.getName()); - res.setHead(source.getHead()); + RawNMFile res = new RawNMFile(source.getName(), source.getHead()); source.getData().stream().map((point) -> { double cr = point.selectChanels(lower, upper).getCR(); if (cr < maxCR) { diff --git a/numass-main/src/main/java/inr/numass/actions/FindBorderAction.java b/numass-main/src/main/java/inr/numass/actions/FindBorderAction.java index 4cfadb33..a62f9e6d 100644 --- a/numass-main/src/main/java/inr/numass/actions/FindBorderAction.java +++ b/numass-main/src/main/java/inr/numass/actions/FindBorderAction.java @@ -86,7 +86,7 @@ public class FindBorderAction extends OneToOneAction { } private void fill(ListTable.Builder dataBuilder, NumassData file, int lower, int upper, NMPoint reference) { - for (NMPoint point : file.getNMPoints()) { + for (NMPoint point : file) { if ((reference != null) && (point.getUset() == reference.getUset())) { continue; } diff --git a/numass-main/src/main/java/inr/numass/actions/MergeDataAction.java b/numass-main/src/main/java/inr/numass/actions/MergeDataAction.java index 3afba48b..3e760db8 100644 --- a/numass-main/src/main/java/inr/numass/actions/MergeDataAction.java +++ b/numass-main/src/main/java/inr/numass/actions/MergeDataAction.java @@ -47,7 +47,7 @@ public class MergeDataAction extends ManyToOneAction { if (meta.hasValue("grouping.byValue")) { groups = super.buildGroups(context, input, actionMeta); } else { - groups = GroupBuilder.byValue(MERGE_NAME, meta.getString(MERGE_NAME, "merge")).group(input); + groups = GroupBuilder.byValue(MERGE_NAME, meta.getString(MERGE_NAME, input.getName())).group(input); } return groups; } diff --git a/numass-main/src/main/java/inr/numass/actions/PrepareDataAction.java b/numass-main/src/main/java/inr/numass/actions/PrepareDataAction.java index 15e494bc..f34279de 100644 --- a/numass-main/src/main/java/inr/numass/actions/PrepareDataAction.java +++ b/numass-main/src/main/java/inr/numass/actions/PrepareDataAction.java @@ -100,7 +100,7 @@ public class PrepareDataAction extends OneToOneAction { } List dataList = new ArrayList<>(); - for (NMPoint point : dataFile.getNMPoints()) { + for (NMPoint point : dataFile) { long total = point.getEventsCount(); double uset = utransform.apply(point.getUset()); diff --git a/numass-main/src/main/java/inr/numass/actions/SlicedData.java b/numass-main/src/main/java/inr/numass/actions/SlicedData.java index ca23d05b..85dc5670 100644 --- a/numass-main/src/main/java/inr/numass/actions/SlicedData.java +++ b/numass-main/src/main/java/inr/numass/actions/SlicedData.java @@ -36,23 +36,21 @@ public class SlicedData extends SimplePointSource { //format = {U,username1,username2, ...} private static final String UNAME = "U"; - - private static TableFormat prepateFormat(Map> intervals){ - ArrayList names = new ArrayList<>(intervals.keySet()); - names.add(0, TNAME); - names.add(0, UNAME); - return TableFormat.forNames(names); - } - - - public SlicedData(NMFile file, Map> intervals, boolean normalize) { + public SlicedData(NMFile file, Map> intervals, boolean normalize) { super(prepateFormat(intervals)); fill(file, intervals, normalize); } + + private static TableFormat prepateFormat(Map> intervals){ + ArrayList names = new ArrayList<>(intervals.keySet()); + names.add(0, TNAME); + names.add(0, UNAME); + return TableFormat.forNames(names); + } private void fill(NMFile file, Map> intervals, boolean normalize){ - for (NMPoint point : file.getNMPoints()) { + for (NMPoint point : file) { //создаем основу для будущей точки HashMap map = new HashMap<>(); diff --git a/numass-main/src/main/java/inr/numass/utils/UnderflowCorrection.java b/numass-main/src/main/java/inr/numass/utils/UnderflowCorrection.java index 76ab74a7..eb8f790d 100644 --- a/numass-main/src/main/java/inr/numass/utils/UnderflowCorrection.java +++ b/numass-main/src/main/java/inr/numass/utils/UnderflowCorrection.java @@ -10,7 +10,6 @@ import hep.dataforge.meta.Meta; import hep.dataforge.tables.ListTable; import hep.dataforge.tables.Table; import inr.numass.storage.NMPoint; -import inr.numass.storage.NumassData; import inr.numass.storage.RawNMPoint; import org.apache.commons.math3.analysis.ParametricUnivariateFunction; import org.apache.commons.math3.exception.DimensionMismatchException; @@ -51,18 +50,18 @@ public class UnderflowCorrection { } } - public Table fitAllPoints(NumassData data, int xLow, int xHigh, int binning) { + public Table fitAllPoints(Iterable data, int xLow, int xHigh, int binning) { ListTable.Builder builder = new ListTable.Builder("U", "amp", "expConst"); - for (NMPoint point : data.getNMPoints()) { + for (NMPoint point : data) { double[] fitRes = getUnderflowExpParameters(point, xLow, xHigh, binning); builder.row(point.getUset(), fitRes[0], fitRes[1]); } return builder.build(); } - public Table fitAllPoints(NumassData data, int xLow, int xHigh, int upper, int binning) { + public Table fitAllPoints(Iterable data, int xLow, int xHigh, int upper, int binning) { ListTable.Builder builder = new ListTable.Builder("U", "amp", "expConst", "correction"); - for (NMPoint point : data.getNMPoints()) { + for (NMPoint point : data) { double norm = ((double) point.getCountInWindow(xLow, upper))/point.getLength(); double[] fitRes = getUnderflowExpParameters(point, xLow, xHigh, binning); builder.row(point.getUset(), fitRes[0], fitRes[1], fitRes[0] * fitRes[1] * (Math.exp(xLow / fitRes[1]) - 1d) / norm + 1d); diff --git a/numass-storage/src/main/java/inr/numass/storage/NMFile.java b/numass-storage/src/main/java/inr/numass/storage/NMFile.java index a26b246f..5a7e5b60 100644 --- a/numass-storage/src/main/java/inr/numass/storage/NMFile.java +++ b/numass-storage/src/main/java/inr/numass/storage/NMFile.java @@ -17,17 +17,17 @@ package inr.numass.storage; import hep.dataforge.description.ValueDef; import hep.dataforge.meta.Meta; -import hep.dataforge.meta.MetaBuilder; import hep.dataforge.names.NamedMetaHolder; + import java.io.File; import java.io.IOException; import java.io.InputStream; import java.time.Instant; import java.util.ArrayList; import java.util.List; +import java.util.stream.Stream; /** - * * Объект, содержащий только спектры, но не сами события * * @author Darksnake @@ -35,51 +35,33 @@ import java.util.List; @ValueDef(name = "numass.path", info = "Path to this data file in numass repository.") @ValueDef(name = "numass.name", info = "The name of this data file.") public class NMFile extends NamedMetaHolder implements NumassData { - - public static NMFile readStream(InputStream is, String fname, Meta config) throws IOException{ - return new NMFile(new NumassDataReader(is, fname, config).read()); - } - - public static NMFile readFile(File file) throws IOException{ - return new NMFile(new NumassDataReader(file).read()); - } - private final String head; private final List points; public NMFile(RawNMFile file) { super(file.getName(), file.meta()); - this.head = file.getHead(); points = new ArrayList<>(); for (RawNMPoint point : file.getData()) { points.add(new NMPoint(point)); } } + public static NMFile readStream(InputStream is, String fname, Meta config) throws IOException { + return new NMFile(new NumassDataReader(is, fname, config).read()); + } + + public static NMFile readFile(File file) throws IOException { + return new NMFile(new NumassDataReader(file).read()); + } + @Override public String getDescription() { return ""; } - /** - * @return the head - */ - public String getHead() { - return head; - } - @Override - public Meta meta() { - return new MetaBuilder("info").setValue("info", head); - } - - - /** - * @return the points - */ - @Override - public List getNMPoints() { - return points; + public Stream stream() { + return points.stream(); } @Override diff --git a/numass-storage/src/main/java/inr/numass/storage/NumassData.java b/numass-storage/src/main/java/inr/numass/storage/NumassData.java index 79bff076..d9a79ac1 100644 --- a/numass-storage/src/main/java/inr/numass/storage/NumassData.java +++ b/numass-storage/src/main/java/inr/numass/storage/NumassData.java @@ -11,21 +11,32 @@ import hep.dataforge.names.Named; import hep.dataforge.tables.Table; import java.time.Instant; +import java.util.Iterator; import java.util.List; import java.util.function.Supplier; +import java.util.stream.Collectors; +import java.util.stream.Stream; /** - * * @author Alexander Nozik */ -public interface NumassData extends Named, Annotated { +public interface NumassData extends Named, Annotated, Iterable { String getDescription(); @Override Meta meta(); - List getNMPoints(); + Stream stream(); + + @Override + default Iterator iterator() { + return stream().iterator(); + } + + default List getNMPoints() { + return stream().collect(Collectors.toList()); + } boolean isEmpty(); @@ -42,7 +53,7 @@ public interface NumassData extends Named, Annotated { * @return */ default NMPoint getByUset(double U) { - for (NMPoint point : getNMPoints()) { + for (NMPoint point : this) { if (point.getUset() == U) { return point; } @@ -57,7 +68,7 @@ public interface NumassData extends Named, Annotated { * @return */ default NMPoint getByUread(double U) { - for (NMPoint point : getNMPoints()) { + for (NMPoint point : this) { if (point.getUread() == U) { return point; } diff --git a/numass-storage/src/main/java/inr/numass/storage/NumassDataLoader.java b/numass-storage/src/main/java/inr/numass/storage/NumassDataLoader.java index fc519093..f617b419 100644 --- a/numass-storage/src/main/java/inr/numass/storage/NumassDataLoader.java +++ b/numass-storage/src/main/java/inr/numass/storage/NumassDataLoader.java @@ -44,6 +44,7 @@ import java.util.*; import java.util.function.Function; import java.util.function.Supplier; import java.util.stream.Collectors; +import java.util.stream.Stream; import static inr.numass.storage.RawNMPoint.MAX_EVENTS_PER_POINT; import static org.apache.commons.vfs2.FileType.FOLDER; @@ -303,8 +304,8 @@ public class NumassDataLoader extends AbstractLoader implements ObjectLoader getNMPoints() { - return this.getPoints().stream().parallel().map(this::readPoint).collect(Collectors.toList()); + public Stream stream() { + return this.getPoints().stream().parallel().map(this::readPoint); } public List getNMPoints(Function transformation) { @@ -397,8 +398,8 @@ public class NumassDataLoader extends AbstractLoader implements ObjectLoader getNMPoints() { - return NumassDataLoader.this.getNMPoints(transform); + public Stream stream() { + return NumassDataLoader.this.stream(); } @Override diff --git a/numass-storage/src/main/java/inr/numass/storage/NumassDataReader.java b/numass-storage/src/main/java/inr/numass/storage/NumassDataReader.java index b2cc84ff..38abc704 100644 --- a/numass-storage/src/main/java/inr/numass/storage/NumassDataReader.java +++ b/numass-storage/src/main/java/inr/numass/storage/NumassDataReader.java @@ -127,11 +127,8 @@ public class NumassDataReader { } private RawNMFile readFile(String name) throws IOException { - - RawNMFile file = new RawNMFile(name); String head = readHead();//2048 - file.setHead(head.replaceAll("\u0000", "")); - + RawNMFile file = new RawNMFile(name, head.replaceAll("\u0000", "")); LocalDateTime filedate = readDate(head); int lab = readByte(); diff --git a/numass-storage/src/main/java/inr/numass/storage/NumassDataUtils.java b/numass-storage/src/main/java/inr/numass/storage/NumassDataUtils.java new file mode 100644 index 00000000..9de69124 --- /dev/null +++ b/numass-storage/src/main/java/inr/numass/storage/NumassDataUtils.java @@ -0,0 +1,43 @@ +package inr.numass.storage; + +import java.time.Instant; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; +import java.util.stream.Stream; + +/** + * Created by darksnake on 30-Jan-17. + */ +public class NumassDataUtils { + + public static Iterable sumSpectra(Stream spectra) { + Map map = new HashMap<>(); + spectra.forEach(datum -> { + datum.forEach(point -> { + double uset = point.getUset(); + if (map.containsKey(uset)) { + map.put(uset, join(point, map.get(uset))); + } else { + map.put(uset, point); + } + }); + }); + return map.values(); + } + + private static NMPoint join(NMPoint first, NMPoint second) { + if (first.getUset() != second.getUset()) { + throw new RuntimeException("Voltage mismatch"); + } + int[] newArray = new int[first.getSpectrum().length]; + Arrays.setAll(newArray, i -> first.getSpectrum()[i] + second.getSpectrum()[i]); + return new NMPoint( + first.getUset(), + first.getUread(), + Instant.EPOCH, + first.getLength() + second.getLength(), + newArray + ); + } +} diff --git a/numass-storage/src/main/java/inr/numass/storage/NumassStorage.java b/numass-storage/src/main/java/inr/numass/storage/NumassStorage.java index 6e5a2e66..502fc8b4 100644 --- a/numass-storage/src/main/java/inr/numass/storage/NumassStorage.java +++ b/numass-storage/src/main/java/inr/numass/storage/NumassStorage.java @@ -186,6 +186,7 @@ public class NumassStorage extends FileStorage { */ @SuppressWarnings("unchecked") public void pushNumassData(String fileName, ByteBuffer data) throws StorageException { + //FIXME move zip to internal try { FileObject nmFile = getDataDir().resolveFile(fileName + NUMASS_ZIP_EXTENSION); if (!nmFile.exists()) { diff --git a/numass-storage/src/main/java/inr/numass/storage/RawNMFile.java b/numass-storage/src/main/java/inr/numass/storage/RawNMFile.java index 1119dad1..7bdf11f1 100644 --- a/numass-storage/src/main/java/inr/numass/storage/RawNMFile.java +++ b/numass-storage/src/main/java/inr/numass/storage/RawNMFile.java @@ -15,7 +15,11 @@ */ package inr.numass.storage; +import hep.dataforge.description.ValueDef; +import hep.dataforge.meta.Meta; +import hep.dataforge.meta.MetaBuilder; import hep.dataforge.names.NamedMetaHolder; + import java.io.BufferedOutputStream; import java.io.OutputStream; import java.io.PrintWriter; @@ -24,29 +28,34 @@ import java.util.List; /** * Contains the whole data but requires a lot of memory + * * @author Darksnake */ +@ValueDef(name = "info", info = "file text header") public class RawNMFile extends NamedMetaHolder { // public static String TYPE = ":data:numassdatafile"; - private final List points; - - private String head; + private final List points = new ArrayList<>(); - public void setHead(String head) { - this.head = head; - } - - public String getHead() { - return head; - } public RawNMFile(String fileName) { super(fileName); - this.points = new ArrayList<>(); } + public RawNMFile(String name, Meta meta) { + super(name, meta); + } + + public RawNMFile(String name, String header) { + super(name, new MetaBuilder("meta").setValue("info", header)); + } + + public String getHead() { + return meta().getString("info", ""); + } + + @Deprecated public void generatePAW(OutputStream stream) { PrintWriter writer = new PrintWriter(new BufferedOutputStream(stream)); long counter = 0; @@ -63,8 +72,9 @@ public class RawNMFile extends NamedMetaHolder { /** * merge of all point with given Uset + * * @param U - * @return + * @return */ public RawNMPoint getByUset(double U) { RawNMPoint res = null; @@ -80,18 +90,19 @@ public class RawNMFile extends NamedMetaHolder { } return res; } - - + + /** * merge of all point with given Uread + * * @param U - * @return + * @return */ public RawNMPoint getByUread(double U) { RawNMPoint res = null; for (RawNMPoint point : points) { - if (point.getUread()== U) { + if (point.getUread() == U) { if (res == null) { res = point.clone(); } else { @@ -100,7 +111,7 @@ public class RawNMFile extends NamedMetaHolder { } } return res; - } + } /** * @return the data