From 7ae632ed694a8f9e88cb6cf41a2ace904acc0e96 Mon Sep 17 00:00:00 2001 From: darksnake Date: Mon, 31 Jul 2017 17:06:37 +0300 Subject: [PATCH] Numass underflow update --- .../java/inr/numass/data/NumassDataUtils.java | 2 +- .../data/analyzers/AbstractAnalyzer.java | 6 +- .../numass/data/analyzers/SimpleAnalyzer.java | 2 +- .../numass/data/analyzers/TimeAnalyzer.java | 59 +++++++----- .../java/inr/numass/data/api/MetaBlock.java | 4 +- .../inr/numass/data/api/NumassAnalyzer.java | 6 +- .../java/inr/numass/data/api/NumassFrame.java | 2 +- .../numass/data/storage/NumassDataLoader.java | 3 +- numass-main/build.gradle | 4 + .../inr/numass/data/PointAnalyzer.groovy | 31 ++++--- .../scripts/times/MultiPointHistogram.groovy | 6 +- .../scripts/times/TestPointAnalyzer.groovy | 15 +-- .../inr/numass/scripts/times/TestProto.groovy | 14 +-- .../numass/scripts/underflow/Underflow.groovy | 92 +++++++++++++------ 14 files changed, 145 insertions(+), 101 deletions(-) diff --git a/numass-core/src/main/java/inr/numass/data/NumassDataUtils.java b/numass-core/src/main/java/inr/numass/data/NumassDataUtils.java index ef8ef6c1..6a4a33b7 100644 --- a/numass-core/src/main/java/inr/numass/data/NumassDataUtils.java +++ b/numass-core/src/main/java/inr/numass/data/NumassDataUtils.java @@ -120,7 +120,7 @@ public class NumassDataUtils { // Arrays.setAll(array, i -> Math.max(0, point.getSpectrum()[i] - reference.getSpectrum()[i])); // return new NumassPointImpl( // point.getVoltage(), -// point.getStartTime(), +// point.getTime(), // point.getLength(), // array // ); diff --git a/numass-core/src/main/java/inr/numass/data/analyzers/AbstractAnalyzer.java b/numass-core/src/main/java/inr/numass/data/analyzers/AbstractAnalyzer.java index bdb31b06..25d4cded 100644 --- a/numass-core/src/main/java/inr/numass/data/analyzers/AbstractAnalyzer.java +++ b/numass-core/src/main/java/inr/numass/data/analyzers/AbstractAnalyzer.java @@ -31,12 +31,14 @@ public abstract class AbstractAnalyzer implements NumassAnalyzer { } /** - * Return unsorted stream of events including events from frames + * Return unsorted stream of events including events from frames. + * In theory, events after processing could be unsorted due to mixture of frames and events. + * In practice usually block have either frame or events, but not both. * * @param block * @return */ - public Stream getEventStream(NumassBlock block, Meta config) { + public Stream getEvents(NumassBlock block, Meta config) { if (block.getFrames().count() == 0) { return block.getEvents(); } else if (getProcessor() == null) { diff --git a/numass-core/src/main/java/inr/numass/data/analyzers/SimpleAnalyzer.java b/numass-core/src/main/java/inr/numass/data/analyzers/SimpleAnalyzer.java index 3740baf9..9c30e448 100644 --- a/numass-core/src/main/java/inr/numass/data/analyzers/SimpleAnalyzer.java +++ b/numass-core/src/main/java/inr/numass/data/analyzers/SimpleAnalyzer.java @@ -25,7 +25,7 @@ public class SimpleAnalyzer extends AbstractAnalyzer { } public Stream getEventStream(NumassBlock block, int loChannel, int upChannel) { - return getEventStream(block, Meta.empty()).filter(it -> it.getChanel() >= loChannel && it.getChanel() < upChannel); + return getEvents(block, Meta.empty()).filter(it -> it.getChanel() >= loChannel && it.getChanel() < upChannel); } @Override diff --git a/numass-core/src/main/java/inr/numass/data/analyzers/TimeAnalyzer.java b/numass-core/src/main/java/inr/numass/data/analyzers/TimeAnalyzer.java index e8f6102c..96d70af2 100644 --- a/numass-core/src/main/java/inr/numass/data/analyzers/TimeAnalyzer.java +++ b/numass-core/src/main/java/inr/numass/data/analyzers/TimeAnalyzer.java @@ -31,12 +31,12 @@ public class TimeAnalyzer extends AbstractAnalyzer { public Values analyze(NumassBlock block, Meta config) { int loChannel = config.getInt("window.lo", 0); int upChannel = config.getInt("window.up", Integer.MAX_VALUE); - long t0 = config.getValue("t0").longValue(); + long t0 = getT0(block, config); AtomicLong totalN = new AtomicLong(0); AtomicLong totalT = new AtomicLong(0); - extendedEventStream(block, config) + getEventsWithDelay(block, config) .filter(pair -> { short channel = pair.getKey().getChanel(); return channel >= loChannel && channel < upChannel; @@ -73,6 +73,10 @@ public class TimeAnalyzer extends AbstractAnalyzer { } } + private long getT0(NumassBlock block, Meta config) { + return config.getValue("t0",0).longValue(); + } + /** * The chain of event times in nanos * @@ -80,32 +84,41 @@ public class TimeAnalyzer extends AbstractAnalyzer { * @param config * @return */ - public Stream> extendedEventStream(NumassBlock block, Meta config) { - long t0 = config.getValue("t0").longValue(); + public Stream> getEventsWithDelay(NumassBlock block, Meta config) { + long t0 = getT0(block, config); AtomicReference lastEvent = new AtomicReference<>(null); - return super.getEventStream(block, config) //using super implementation - .sorted() - .map(event -> { - if (lastEvent.get() == null) { - lastEvent.set(event); - return new Pair<>(event, 0L); - } else { - long res = event.getTimeOffset() - lastEvent.get().getTimeOffset(); - if (res >= 0) { - lastEvent.set(event); - return new Pair<>(event, res); - } else { - lastEvent.set(null); - return new Pair<>(event, 0L); - } - } - }).filter(pair -> pair.getValue() >= t0); + Stream eventStream = super.getEvents(block, config);//using super implementation + if (config.getBoolean("sort", false)) { + eventStream = eventStream.sorted(); + } + + return eventStream.map(event -> { + if (lastEvent.get() == null) { + lastEvent.set(event); + return new Pair<>(event, 0L); + } else { + long res = event.getTimeOffset() - lastEvent.get().getTimeOffset(); + if (res >= 0) { + lastEvent.set(event); + return new Pair<>(event, res); + } else { + lastEvent.set(null); + return new Pair<>(event, 0L); + } + } + }).filter(pair -> pair.getValue() >= t0); } + /** + * The filtered stream of events + * @param block + * @param config + * @return + */ @Override - public Stream getEventStream(NumassBlock block, Meta config) { - return extendedEventStream(block,config).map(Pair::getKey); + public Stream getEvents(NumassBlock block, Meta config) { + return getEventsWithDelay(block, config).map(Pair::getKey); } } diff --git a/numass-core/src/main/java/inr/numass/data/api/MetaBlock.java b/numass-core/src/main/java/inr/numass/data/api/MetaBlock.java index f6bf1be6..33a785cb 100644 --- a/numass-core/src/main/java/inr/numass/data/api/MetaBlock.java +++ b/numass-core/src/main/java/inr/numass/data/api/MetaBlock.java @@ -32,11 +32,11 @@ public class MetaBlock implements NumassBlock { @Override public Stream getEvents() { - return blocks.stream().flatMap(NumassBlock::getEvents); + return blocks.stream().flatMap(NumassBlock::getEvents).sorted(Comparator.comparing(NumassEvent::getTime)); } @Override public Stream getFrames() { - return blocks.stream().flatMap(NumassBlock::getFrames); + return blocks.stream().flatMap(NumassBlock::getFrames).sorted(Comparator.comparing(NumassFrame::getTime)); } } diff --git a/numass-core/src/main/java/inr/numass/data/api/NumassAnalyzer.java b/numass-core/src/main/java/inr/numass/data/api/NumassAnalyzer.java index 0ac32f42..985d05e3 100644 --- a/numass-core/src/main/java/inr/numass/data/api/NumassAnalyzer.java +++ b/numass-core/src/main/java/inr/numass/data/api/NumassAnalyzer.java @@ -74,7 +74,7 @@ public interface NumassAnalyzer { String COUNT_KEY = "count"; String LENGTH_KEY = "length"; String COUNT_RATE_KEY = "cr"; - String COUNT_RATE_ERROR_KEY = "cr.err"; + String COUNT_RATE_ERROR_KEY = "crErr"; /** * Perform analysis on block. The values for count rate, its error and point length in nanos must @@ -91,7 +91,7 @@ public interface NumassAnalyzer { * @param block * @return */ - Stream getEventStream(NumassBlock block, Meta config); + Stream getEvents(NumassBlock block, Meta config); /** * Analyze the whole set. And return results as a table @@ -121,7 +121,7 @@ public interface NumassAnalyzer { //optimized for fastest computation //TODO requires additional performance optimization AtomicLong[] spectrum = new AtomicLong[MAX_CHANNEL]; - getEventStream(block, config).forEach(event -> { + getEvents(block, config).forEach(event -> { if (spectrum[event.getChanel()] == null) { spectrum[event.getChanel()] = new AtomicLong(1); } else { diff --git a/numass-core/src/main/java/inr/numass/data/api/NumassFrame.java b/numass-core/src/main/java/inr/numass/data/api/NumassFrame.java index 3e77246c..0d7c61a8 100644 --- a/numass-core/src/main/java/inr/numass/data/api/NumassFrame.java +++ b/numass-core/src/main/java/inr/numass/data/api/NumassFrame.java @@ -32,7 +32,7 @@ public class NumassFrame { this.tickSize = tickSize; } - public Instant getStartTime() { + public Instant getTime() { return startTime; } diff --git a/numass-core/src/main/java/inr/numass/data/storage/NumassDataLoader.java b/numass-core/src/main/java/inr/numass/data/storage/NumassDataLoader.java index 400a078c..4b848d4d 100644 --- a/numass-core/src/main/java/inr/numass/data/storage/NumassDataLoader.java +++ b/numass-core/src/main/java/inr/numass/data/storage/NumassDataLoader.java @@ -110,7 +110,6 @@ public class NumassDataLoader extends AbstractLoader implements ObjectLoader points = loaders.collect { loader -> loader.optPoint(hv).get() } def loChannel = 400; - def upChannel = 800; + def upChannel = 2000; - def chain = new TimeAnalyzer().extendedEventStream(new MetaBlock(points), Grind.buildMeta("window.lo": loChannel, "window.up": upChannel)) + def chain = new TimeAnalyzer().getEventsWithDelay(new MetaBlock(points), Grind.buildMeta("window.lo": loChannel, "window.up": upChannel)).mapToLong{it.value} - def histogram = PointAnalyzer.histogram(chain, 5e-6, 500).asTable(); + def histogram = PointAnalyzer.histogram(chain, 1, 500).asTable(); println "finished histogram calculation..." diff --git a/numass-main/src/main/groovy/inr/numass/scripts/times/TestPointAnalyzer.groovy b/numass-main/src/main/groovy/inr/numass/scripts/times/TestPointAnalyzer.groovy index 10296b54..46796b8f 100644 --- a/numass-main/src/main/groovy/inr/numass/scripts/times/TestPointAnalyzer.groovy +++ b/numass-main/src/main/groovy/inr/numass/scripts/times/TestPointAnalyzer.groovy @@ -8,6 +8,7 @@ import hep.dataforge.plots.fx.FXPlotManager import hep.dataforge.tables.ValueMap import inr.numass.NumassPlugin import inr.numass.data.PointAnalyzer +import inr.numass.data.api.NumassAnalyzer import inr.numass.data.api.NumassPoint import inr.numass.data.api.NumassSet import inr.numass.data.storage.NumassStorage @@ -29,8 +30,8 @@ new GrindShell(ctx).eval { NumassStorage storage = NumassStorageFactory.buildLocal(rootDir); - def set = "set_1" - def hv = 14500; + def set = "set_43" + def hv = 16000; def loader = storage.provide("loader::$set", NumassSet.class).get(); def point = loader.provide("$hv", NumassPoint.class).get() @@ -56,17 +57,9 @@ new GrindShell(ctx).eval { def t0 = (1..150).collect { 500 * it } -// point.blocks.eachWithIndex { block, index -> -// def statPlotPoints = t0.collect { -// def result = PointAnalyzer.analyze(block, t0: it, "window.lo": loChannel, "window.up": upChannel) -// ValueMap.ofMap("x": it / 1000, "y": result.getDouble("cr"), "y.err": result.getDouble("cr.err")); -// } -// plot.plot(name: index, frame: "stat-method", showLine: true, statPlotPoints) -// } - def statPlotPoints = t0.collect { def result = PointAnalyzer.analyze(point, t0: it, "window.lo": loChannel, "window.up": upChannel) - ValueMap.ofMap("x": it / 1000, "y": result.getDouble("cr"), "y.err": result.getDouble("cr.err")); + ValueMap.ofMap("x": it / 1000, "y": result.getDouble("cr"), "y.err": result.getDouble(NumassAnalyzer.COUNT_RATE_ERROR_KEY)); } plot.plot(name: "total", frame: "stat-method", showLine: true, statPlotPoints) diff --git a/numass-main/src/main/groovy/inr/numass/scripts/times/TestProto.groovy b/numass-main/src/main/groovy/inr/numass/scripts/times/TestProto.groovy index 81dc5af6..bbaf552f 100644 --- a/numass-main/src/main/groovy/inr/numass/scripts/times/TestProto.groovy +++ b/numass-main/src/main/groovy/inr/numass/scripts/times/TestProto.groovy @@ -8,6 +8,7 @@ import hep.dataforge.plots.fx.FXPlotManager import hep.dataforge.tables.ValueMap import inr.numass.NumassPlugin import inr.numass.data.PointAnalyzer +import inr.numass.data.api.NumassAnalyzer import inr.numass.data.api.NumassPoint import inr.numass.data.storage.ProtoNumassPoint @@ -19,7 +20,7 @@ ctx.pluginManager().load(NumassPlugin.class) new GrindShell(ctx).eval { PlotHelper plot = plots - NumassPoint point = ProtoNumassPoint.readFile(Paths.get("D:\\Work\\Numass\\data\\test\\40_kHz_5s.df")) + NumassPoint point = ProtoNumassPoint.readFile(Paths.get("D:\\Work\\Numass\\data\\2017_05\\Fill_3_events\\set_33\\p0(30s)(HV1=16000).df")) def loChannel = 0; def upChannel = 10000; @@ -41,19 +42,12 @@ new GrindShell(ctx).eval { println "The expected count rate for 30 us delay is $trueCR" - def t0 = (1..150).collect { 500 * it } + def t0 = (1..150).collect { 1000 * it } -// point.blocks.eachWithIndex { block, index -> -// def statPlotPoints = t0.collect { -// def result = PointAnalyzer.analyze(block, t0: it, "window.lo": loChannel, "window.up": upChannel) -// ValueMap.ofMap("x": it / 1000, "y": result.getDouble("cr"), "y.err": result.getDouble("cr.err")); -// } -// plot.plot(name: index, frame: "stat-method", showLine: true, statPlotPoints) -// } def statPlotPoints = t0.collect { def result = PointAnalyzer.analyze(point, t0: it, "window.lo": loChannel, "window.up": upChannel) - ValueMap.ofMap("x": it / 1000, "y": result.getDouble("cr"), "y.err": result.getDouble("cr.err")); + ValueMap.ofMap("x": it / 1000, "y": result.getDouble("cr"), "y.err": result.getDouble(NumassAnalyzer.COUNT_RATE_ERROR_KEY)); } plot.plot(name: "total", frame: "stat-method", showLine: true, thickness: 4, statPlotPoints) diff --git a/numass-main/src/main/groovy/inr/numass/scripts/underflow/Underflow.groovy b/numass-main/src/main/groovy/inr/numass/scripts/underflow/Underflow.groovy index 60158094..6236d855 100644 --- a/numass-main/src/main/groovy/inr/numass/scripts/underflow/Underflow.groovy +++ b/numass-main/src/main/groovy/inr/numass/scripts/underflow/Underflow.groovy @@ -6,10 +6,13 @@ package inr.numass.scripts.underflow +import hep.dataforge.cache.CachePlugin import hep.dataforge.context.Context import hep.dataforge.context.Global -import hep.dataforge.grind.Grind +import hep.dataforge.data.DataNode +import hep.dataforge.data.DataSet import hep.dataforge.grind.GrindShell +import hep.dataforge.grind.actions.GrindPipe import hep.dataforge.grind.helpers.PlotHelper import hep.dataforge.io.ColumnedDataWriter import hep.dataforge.meta.Meta @@ -25,59 +28,84 @@ import inr.numass.data.NumassDataUtils import inr.numass.data.analyzers.TimeAnalyzer import inr.numass.data.api.NumassAnalyzer import inr.numass.data.api.NumassPoint +import inr.numass.data.api.NumassSet import inr.numass.data.api.SimpleNumassPoint import inr.numass.data.storage.NumassStorage import inr.numass.data.storage.NumassStorageFactory import java.util.stream.Collectors +import static hep.dataforge.grind.Grind.buildMeta import static inr.numass.data.api.NumassAnalyzer.CHANNEL_KEY import static inr.numass.data.api.NumassAnalyzer.COUNT_RATE_KEY Context ctx = Global.instance() ctx.pluginManager().load(FXPlotManager) ctx.pluginManager().load(NumassPlugin.class) +ctx.pluginManager().load(CachePlugin.class) + +Meta meta = buildMeta { + data(dir: "D:\\Work\\Numass\\data\\2017_05\\Fill_2", mask: "set_.{1,3}") + generate(t0: 3e4, sort: true) + subtract(reference: 18600) + fit(xlow: 450, xHigh: 700, upper: 3100, binning: 20) +} + new GrindShell(ctx).eval { //Defining root directory - File dataDirectory = new File("D:\\Work\\Numass\\data\\2017_05\\Fill_2") + File dataDirectory = new File(meta.getString("data.dir")) //creating storage instance NumassStorage storage = NumassStorageFactory.buildLocal(dataDirectory); //Reading points - Map> allPoints = StorageUtils + //Free operation. No reading done + List sets = StorageUtils .loaderStream(storage) - .filter { it.key.matches("set_.{1,3}") } + .filter { it.key.matches(meta.getString("data.mask")) } .map { println "loading ${it.key}" - it.value - }.flatMap { it.points } - .collect(Collectors.groupingBy { it.voltage }) + return it.value + }.collect(Collectors.toList()); - Meta analyzerMeta = Grind.buildMeta(t0: 3e4) - NumassAnalyzer analyzer = new TimeAnalyzer() + NumassAnalyzer analyzer = new TimeAnalyzer(); - //creating spectra - Map spectra = allPoints.collectEntries { - def point = new SimpleNumassPoint(it.key, it.value) - println "generating spectrum for ${point.voltage}" - return [(point.voltage): analyzer.getSpectrum(point, analyzerMeta)] + def dataBuilder = DataSet.builder(NumassPoint); + + sets.sort { it.startTime } + .collectMany { it.points.collect() } + .groupBy { it.voltage } + .each { key, value -> + def point = new SimpleNumassPoint(key as double, value as List) + String name = (key as Integer).toString() + dataBuilder.putStatic(name, point, buildMeta(voltage: key)); + } + + DataNode data = dataBuilder.build() + + def generate = GrindPipe. build(name: "generate") { + return analyzer.getSpectrum(delegate.input as NumassPoint, delegate.meta) } - - - def refereceVoltage = 18600d + DataNode spectra = generate.run(context, data, meta.getMeta("generate")); + spectra = context.getFeature(CachePlugin).cacheNode("underflow", meta, spectra) //subtracting reference point - if (refereceVoltage) { - println "subtracting reference point ${refereceVoltage}" - def referencePoint = spectra[refereceVoltage] - spectra = spectra.findAll { it.key != refereceVoltage }.collectEntries { - return [(it.key): NumassDataUtils.subtractSpectrum(it.getValue() as Table, referencePoint as Table)] + Map spectraMap + if (meta.hasValue("subtract.reference")) { + String referenceVoltage = meta["subtract.reference"].stringValue() + println "subtracting reference point ${referenceVoltage}" + def referencePoint = spectra.compute(referenceVoltage) + spectraMap = spectra + .findAll { it.name != referenceVoltage } + .collectEntries { + return [(it.meta["voltage"].doubleValue()): NumassDataUtils.subtractSpectrum(it.get(), referencePoint)] } + } else { + spectraMap = spectra.collectEntries { return [(it.meta["voltage"].doubleValue()): it.get()] } } //Showing selected points @@ -97,22 +125,28 @@ new GrindShell(ctx).eval { //configuring and plotting plotGroup.configure(showLine: true, showSymbol: false, showErrors: false, connectionType: "step") def frame = (plots as PlotHelper).getManager().getPlotFrame("Spectra") - frame.configure("yAxis.type": "log") + frame.configureValue("yAxis.type", "log") frame.addAll(plotGroup) } - showPoints(spectra.findAll { it.key in [16200d, 16400d, 16800d, 17000d, 17200d, 17700d] }) - - println() + showPoints(spectraMap.findAll { it.key in [16200d, 16400d, 16800d, 17000d, 17200d, 17700d] }) Table correctionTable = TableTransform.filter( - UnderflowFitter.fitAllPoints(spectra, 450, 700, 3100, 20), + UnderflowFitter.fitAllPoints( + spectraMap, + meta["fit.xlow"].intValue(), + meta["fit.xHigh"].intValue(), + meta["fit.upper"].intValue(), + meta["fit.binning"].intValue() + ), "correction", 0, - 2) + 2 + ) + ColumnedDataWriter.writeTable(System.out, correctionTable, "underflow parameters") (plots as PlotHelper).plot(correctionTable, name: "correction", frame: "Correction") { - adapter("x.value":"U", "y.value":"correction") + adapter("x.value": "U", "y.value": "correction") } } \ No newline at end of file