Some changes to numass viewer and nummass file envelope

This commit is contained in:
Alexander Nozik 2018-03-13 16:56:39 +03:00
parent ff6a604874
commit 0792ab3c08
15 changed files with 91 additions and 574 deletions

View File

@ -1,161 +0,0 @@
package inr.numass.data;
import hep.dataforge.meta.Meta;
import hep.dataforge.meta.MetaBuilder;
import inr.numass.data.api.NumassPoint;
import inr.numass.data.api.NumassSet;
import inr.numass.data.api.SimpleNumassPoint;
import org.jetbrains.annotations.NotNull;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import java.util.stream.Stream;
/**
* Created by darksnake on 30-Jan-17.
*/
public class NumassDataUtils {
public static NumassSet join(String name, Collection<NumassSet> sets) {
return new NumassSet() {
@Override
public Stream<NumassPoint> getPoints() {
Map<Double,List<NumassPoint>> points = sets.stream().flatMap(NumassSet::getPoints)
.collect(Collectors.groupingBy(NumassPoint::getVoltage));
return points.entrySet().stream().map(entry->new SimpleNumassPoint(entry.getKey(),entry.getValue()));
}
@Override
public Meta getMeta() {
MetaBuilder metaBuilder = new MetaBuilder();
sets.forEach(set -> metaBuilder.putNode(set.getName(), set.getMeta()));
return metaBuilder;
}
@Override
public String getName() {
return name;
}
};
}
@NotNull
public static SpectrumAdapter adapter() {
return new SpectrumAdapter("Uset", "CR", "CRerr", "Time");
}
// public static Collection<NumassPoint> joinSpectra(Stream<NumassSet> spectra) {
// Map<Double, NumassPoint> map = new LinkedHashMap<>();
// spectra.forEach(datum -> {
// datum.forEach(point -> {
// double uset = point.getVoltage();
// if (map.containsKey(uset)) {
// map.put(uset, join(point, map.get(uset)));
// } else {
// map.put(uset, point);
// }
// });
// });
// return map.values();
// }
//
// /**
// * Spectral sum of two points
// *
// * @param first
// * @param second
// * @return
// */
// public static NumassPoint join(NumassPoint first, NumassPoint second) {
// if (first.getVoltage() != second.getVoltage()) {
// throw new RuntimeException("Voltage mismatch");
// }
// int[] newArray = new int[first.getAmplitudeSpectrum().length];
// Arrays.setAll(newArray, i -> first.getAmplitudeSpectrum()[i] + second.getAmplitudeSpectrum()[i]);
// return new NumassPointImpl(
// first.getVoltage(),
// Instant.EPOCH,
// first.getLength() + second.getLength(),
// newArray
// );
// }
//
// public static NumassPoint substractPoint(NumassPoint point, NumassPoint reference) {
// int[] array = new int[point.getAmplitudeSpectrum().length];
// Arrays.setAll(array, i -> Math.max(0, point.getAmplitudeSpectrum()[i] - reference.getAmplitudeSpectrum()[i]));
// return new NumassPointImpl(
// point.getVoltage(),
// point.getTime(),
// point.getLength(),
// array
// );
// }
//
// public static Collection<NumassPoint> substractReferencePoint(Collection<NumassPoint> points, double uset) {
// NumassPoint reference = points.stream().filter(it -> it.getVoltage() == uset).findFirst()
// .orElseThrow(() -> new RuntimeException("Reference point not found"));
// return points.stream().map(it -> substractPoint(it, reference)).collect(Collectors.toList());
// }
//
//
// /**
// * Поправка масштаба высокого.
// *
// * @param data
// * @param beta
// * @return
// */
// public static Table setHVScale(ListTable data, double beta) {
// SpectrumAdapter reader = adapter();
// ListTable.Builder res = new ListTable.Builder(data.getFormat());
// for (Values dp : data) {
// double corrFactor = 1 + beta;
// res.row(reader.buildSpectrumDataPoint(reader.getX(dp).doubleValue() * corrFactor, reader.getCount(dp), reader.getTime(dp)));
// }
// return res.builder();
// }
//
//
// public static Table correctForDeadTime(ListTable data, double dtime) {
// return correctForDeadTime(data, adapter(), dtime);
// }
//
// /**
// * Коррекция на мертвое время в секундах
// *
// * @param data
// * @param dtime
// * @return
// */
// public static Table correctForDeadTime(ListTable data, SpectrumAdapter adapter, double dtime) {
//// SpectrumAdapter adapter = adapter();
// ListTable.Builder res = new ListTable.Builder(data.getFormat());
// for (Values dp : data) {
// double corrFactor = 1 / (1 - dtime * adapter.getCount(dp) / adapter.getTime(dp));
// res.row(adapter.buildSpectrumDataPoint(adapter.getX(dp).doubleValue(), (long) (adapter.getCount(dp) * corrFactor), adapter.getTime(dp)));
// }
// return res.builder();
// }
//
// public static double countRateWithDeadTime(NumassPoint p, int from, int to, double deadTime) {
// double wind = p.getCountInWindow(from, to) / p.getLength();
// double res;
// if (deadTime > 0) {
// double total = p.getTotalCount();
//// double time = p.getLength();
//// res = wind / (1 - total * deadTime / time);
// double timeRatio = deadTime / p.getLength();
// res = wind / total * (1d - Math.sqrt(1d - 4d * total * timeRatio)) / 2d / timeRatio;
// } else {
// res = wind;
// }
// return res;
// }
//
// public static double countRateWithDeadTimeErr(NumassPoint p, int from, int to, double deadTime) {
// return Math.sqrt(countRateWithDeadTime(p, from, to, deadTime) / p.getLength());
// }
}

View File

@ -1,46 +0,0 @@
package inr.numass.data.api;
import java.time.Duration;
import java.time.Instant;
import java.util.*;
import java.util.stream.Stream;
/**
* A block constructed from a set of other blocks. Internal blocks are not necessary subsequent. Blocks are automatically sorted.
* Created by darksnake on 16.07.2017.
*/
public class MetaBlock implements NumassBlock {
private SortedSet<NumassBlock> blocks = new TreeSet<>(Comparator.comparing(NumassBlock::getStartTime));
public MetaBlock(NumassBlock... blocks) {
this.blocks.addAll(Arrays.asList(blocks));
}
public MetaBlock(Collection<NumassBlock> blocks) {
this.blocks.addAll(blocks);
}
@Override
public Instant getStartTime() {
return blocks.first().getStartTime();
}
@Override
public Duration getLength() {
return Duration.ofNanos(blocks.stream().mapToLong(block -> block.getLength().toNanos()).sum());
}
@Override
public Stream<NumassEvent> getEvents() {
return blocks.stream()
.sorted(Comparator.comparing(NumassBlock::getStartTime))
.flatMap(NumassBlock::getEvents);
}
@Override
public Stream<NumassFrame> getFrames() {
return blocks.stream()
.sorted(Comparator.comparing(NumassBlock::getStartTime))
.flatMap(NumassBlock::getFrames);
}
}

View File

@ -1,36 +0,0 @@
package inr.numass.data.api;
import java.time.Duration;
import java.time.Instant;
import java.util.stream.Stream;
/**
* A single continuous measurement block. The block can contain both isolated events and signal frames
* <p>
* Created by darksnake on 06-Jul-17.
*/
public interface NumassBlock {
/**
* The absolute start time of the block
* @return
*/
Instant getStartTime();
/**
* The length of the block
* @return
*/
Duration getLength();
/**
* Stream of isolated events. Could be empty
* @return
*/
Stream<NumassEvent> getEvents();
/**
* Stream of frames. Could be empty
* @return
*/
Stream<NumassFrame> getFrames();
}

View File

@ -1,94 +0,0 @@
package inr.numass.data.api;
import hep.dataforge.meta.Metoid;
import hep.dataforge.values.Value;
import java.time.Duration;
import java.time.Instant;
import java.util.stream.Stream;
/**
* Created by darksnake on 06-Jul-17.
*/
public interface NumassPoint extends Metoid, NumassBlock {
String START_TIME_KEY = "start";
String LENGTH_KEY = "length";
String HV_KEY = "voltage";
String INDEX_KEY = "index";
Stream<NumassBlock> getBlocks();
/**
* Get the voltage setting for the point
*
* @return
*/
default double getVoltage() {
return getMeta().getDouble(HV_KEY, 0);
}
/**
* Get the index for this point in the set
* @return
*/
default int getIndex() {
return getMeta().getInt(INDEX_KEY, -1);
}
/**
* Get the first block if it exists. Throw runtime exception otherwise.
*
* @return
*/
default NumassBlock getFirstBlock() {
return getBlocks().findFirst().orElseThrow(() -> new RuntimeException("The point is empty"));
}
/**
* Get the starting time from meta or from first block
*
* @return
*/
@Override
default Instant getStartTime() {
return getMeta().optValue(START_TIME_KEY).map(Value::timeValue).orElseGet(() -> getFirstBlock().getStartTime());
}
/**
* Get the length key of meta or calculate length as a sum of block lengths. The latter could be a bit slow
*
* @return
*/
@Override
default Duration getLength() {
return Duration.ofNanos(
getMeta().optValue(LENGTH_KEY).map(Value::longValue)
.orElseGet(() -> getBlocks().mapToLong(it -> it.getLength().toNanos()).sum())
);
}
/**
* Get all events it all blocks as a single sequence
* <p>
* Some performance analysis of different stream concatenation approaches is given here: https://www.techempower.com/blog/2016/10/19/efficient-multiple-stream-concatenation-in-java/
* </p>
*
* @return
*/
@Override
default Stream<NumassEvent> getEvents() {
return getBlocks().flatMap(NumassBlock::getEvents);
}
/**
* Get all frames in all blocks as a single sequence
*
* @return
*/
@Override
default Stream<NumassFrame> getFrames() {
return getBlocks().flatMap(NumassBlock::getFrames);
}
}

View File

@ -1,101 +0,0 @@
/*
* To change this license header, choose License Headers in Project Properties.
* To change this template file, choose Tools | Templates
* and open the template in the editor.
*/
package inr.numass.data.api;
import hep.dataforge.meta.Metoid;
import hep.dataforge.names.Named;
import hep.dataforge.providers.Provider;
import hep.dataforge.providers.Provides;
import hep.dataforge.providers.ProvidesNames;
import hep.dataforge.tables.Table;
import hep.dataforge.values.Value;
import org.jetbrains.annotations.NotNull;
import java.time.Instant;
import java.util.Iterator;
import java.util.List;
import java.util.Optional;
import java.util.stream.Collectors;
import java.util.stream.Stream;
/**
* A single set of numass points previously called file.
*
* @author <a href="mailto:altavir@gmail.com">Alexander Nozik</a>
*/
public interface NumassSet extends Named, Metoid, Iterable<NumassPoint>, Provider {
String DESCRIPTION_KEY = "info";
String NUMASS_POINT_PROVIDER_KEY = "point";
Stream<NumassPoint> getPoints();
// default String getDescription() {
// return getMeta().getString(DESCRIPTION_KEY, "");
// }
@NotNull
@Override
default Iterator<NumassPoint> iterator() {
return getPoints().iterator();
}
/**
* Get the first point if it exists. Throw runtime exception otherwise.
*
* @return
*/
default NumassPoint getFirstPoint() {
return getPoints().findFirst().orElseThrow(() -> new RuntimeException("The set is empty"));
}
/**
* Get the starting time from meta or from first point
*
* @return
*/
default Instant getStartTime() {
return getMeta().optValue(NumassPoint.START_TIME_KEY).map(Value::timeValue).orElseGet(() -> getFirstPoint().getStartTime());
}
/**
* Find first point with given voltage
*
* @param voltage
* @return
*/
default Optional<NumassPoint> optPoint(double voltage) {
return getPoints().filter(it -> it.getVoltage() == voltage).findFirst();
}
/**
* List all points with given voltage
*
* @param voltage
* @return
*/
default List<NumassPoint> getPoints(double voltage) {
return getPoints().filter(it -> it.getVoltage() == voltage).collect(Collectors.toList());
}
@Provides(NUMASS_POINT_PROVIDER_KEY)
default Optional<NumassPoint> optPoint(String voltage) {
return optPoint(Double.parseDouble(voltage));
}
@Override
default String defaultTarget() {
return NUMASS_POINT_PROVIDER_KEY;
}
@ProvidesNames(NUMASS_POINT_PROVIDER_KEY)
default Stream<String> listPoints() {
return getPoints().map(it -> Double.toString(it.getVoltage()));
}
default Optional<Table> getHvData() {
return Optional.empty();
}
}

View File

@ -1,11 +0,0 @@
package inr.numass.data.api;
import java.util.stream.Stream;
/**
* An ancestor to numass frame analyzers
* Created by darksnake on 07.07.2017.
*/
public interface SignalProcessor {
Stream<NumassEvent> analyze(NumassFrame frame);
}

View File

@ -1,43 +0,0 @@
package inr.numass.data.api;
import java.io.Serializable;
import java.time.Duration;
import java.time.Instant;
import java.util.List;
import java.util.stream.Stream;
/**
* A simple in-memory implementation of block of events. No frames are allowed
* Created by darksnake on 08.07.2017.
*/
public class SimpleBlock implements NumassBlock, Serializable {
private final Instant startTime;
private final Duration length;
private final List<NumassEvent> events;
public SimpleBlock(Instant startTime, Duration length, List<NumassEvent> events) {
this.startTime = startTime;
this.length = length;
this.events = events;
}
@Override
public Instant getStartTime() {
return startTime;
}
@Override
public Duration getLength() {
return length;
}
@Override
public Stream<NumassEvent> getEvents() {
return events.stream();
}
@Override
public Stream<NumassFrame> getFrames() {
return Stream.empty();
}
}

View File

@ -1,41 +0,0 @@
package inr.numass.data.api;
import hep.dataforge.meta.Meta;
import hep.dataforge.meta.MetaBuilder;
import hep.dataforge.meta.MetaHolder;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Comparator;
import java.util.List;
import java.util.stream.Stream;
/**
* A simple static implementation of NumassPoint
* Created by darksnake on 08.07.2017.
*/
public class SimpleNumassPoint extends MetaHolder implements NumassPoint {
private final List<NumassBlock> blocks;
/**
* Input blocks must be sorted
* @param voltage
* @param blocks
*/
public SimpleNumassPoint(double voltage, Collection<? extends NumassBlock> blocks) {
super(new MetaBuilder("point").setValue(HV_KEY, voltage));
this.blocks = new ArrayList<>(blocks);
this.blocks.sort(Comparator.comparing(NumassBlock::getStartTime));
}
public SimpleNumassPoint(Meta meta, Collection<? extends NumassBlock> blocks) {
super(meta);
this.blocks = new ArrayList<>(blocks);
this.blocks.sort(Comparator.comparing(NumassBlock::getStartTime));
}
@Override
public Stream<NumassBlock> getBlocks() {
return blocks.stream();
}
}

View File

@ -5,23 +5,26 @@ import hep.dataforge.storage.filestorage.FileEnvelope;
import inr.numass.NumassEnvelopeType;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.ByteBuffer;
import java.nio.MappedByteBuffer;
import java.nio.channels.FileChannel;
import java.nio.file.Path;
import java.util.Arrays;
import static java.nio.file.StandardOpenOption.READ;
public class NumassFileEnvelope extends FileEnvelope {
public static byte[] LEGACY_START_SEQUENCE = {'#','!'};
public static byte[] LEGACY_END_SEQUENCE = {'!','#','\r','\n'};
public static FileEnvelope open(Path path, boolean readOnly) {
if (!Files.exists(path)) {
throw new RuntimeException("File envelope does not exist");
}
try (InputStream stream = Files.newInputStream(path, READ)) {
byte[] bytes = new byte[2];
stream.read(bytes);
if (Arrays.equals(bytes, NumassEnvelopeType.Companion.getLEGACY_START_SEQUENCE())) {
// if (!Files.exists(path)) {
// throw new RuntimeException("File envelope does not exist");
// }
try (FileChannel channel = FileChannel.open(path,READ)) {
MappedByteBuffer buffer = channel.map(FileChannel.MapMode.READ_ONLY, 0, 2);
if (buffer.compareTo(ByteBuffer.wrap(LEGACY_START_SEQUENCE)) == 0) {
return new NumassFileEnvelope(path, readOnly);
} else {
return FileEnvelope.Companion.open(path, readOnly);

View File

@ -2,6 +2,8 @@ package inr.numass
import hep.dataforge.io.envelopes.*
import hep.dataforge.values.Value
import inr.numass.data.legacy.NumassFileEnvelope.LEGACY_END_SEQUENCE
import inr.numass.data.legacy.NumassFileEnvelope.LEGACY_START_SEQUENCE
import org.slf4j.LoggerFactory
import java.io.IOException
import java.nio.ByteBuffer
@ -77,8 +79,4 @@ class NumassEnvelopeType : EnvelopeType {
}
}
companion object {
val LEGACY_START_SEQUENCE = byteArrayOf('#'.toByte(), '!'.toByte())
val LEGACY_END_SEQUENCE = byteArrayOf('!'.toByte(), '#'.toByte(), '\r'.toByte(), '\n'.toByte())
}
}

View File

@ -0,0 +1,51 @@
package inr.numass.data
import hep.dataforge.io.envelopes.Envelope
import hep.dataforge.meta.Meta
import hep.dataforge.meta.MetaBuilder
import inr.numass.data.api.NumassPoint
import inr.numass.data.api.NumassSet
import inr.numass.data.api.SimpleNumassPoint
import java.io.InputStream
import java.util.stream.Collectors
import java.util.stream.Stream
import java.util.zip.ZipInputStream
/**
* Created by darksnake on 30-Jan-17.
*/
object NumassDataUtils {
fun join(name: String, sets: Collection<NumassSet>): NumassSet {
return object : NumassSet {
override fun getPoints(): Stream<NumassPoint> {
val points = sets.stream().flatMap<NumassPoint> { it.points }
.collect(Collectors.groupingBy<NumassPoint, Double> { it.voltage })
return points.entries.stream().map { entry -> SimpleNumassPoint(entry.key, entry.value) }
}
override fun getMeta(): Meta {
val metaBuilder = MetaBuilder()
sets.forEach { set -> metaBuilder.putNode(set.name, set.meta) }
return metaBuilder
}
override fun getName(): String {
return name
}
}
}
fun adapter(): SpectrumAdapter {
return SpectrumAdapter("Uset", "CR", "CRerr", "Time")
}
}
/**
* Get valid data stream utilizing compression if it is present
*/
val Envelope.dataStream : InputStream
get() = if(this.meta.getString("compression", "none") == "zlib"){
ZipInputStream(this.data.stream)
} else {
this.data.stream
}

View File

@ -49,12 +49,10 @@ class NumassStorage : FileStorage {
val description: String
get() = meta.getString("description", "")
@Throws(StorageException::class)
protected constructor(parent: FileStorage, config: Meta, shelf: String) : super(parent, config, shelf) {
private constructor(parent: FileStorage, config: Meta, shelf: String) : super(parent, config, shelf) {
super.refresh()
}
@Throws(StorageException::class)
constructor(context: Context, config: Meta, path: Path) : super(context, config, path) {
super.refresh()
}
@ -122,8 +120,8 @@ class NumassStorage : FileStorage {
}
@Throws(StorageException::class)
override fun createShelf(meta: Meta, path: String): NumassStorage {
return NumassStorage(this, meta, path)
override fun createShelf(shelfConfiguration: Meta, shelfName: String): NumassStorage {
return NumassStorage(this, shelfConfiguration, shelfName)
}
/**
@ -177,8 +175,8 @@ class NumassStorage : FileStorage {
companion object {
val FILE_NAME_KEY = "fileName"
val FILE_SIZE_KEY = "fileSize"
const val FILE_NAME_KEY = "fileName"
const val FILE_SIZE_KEY = "fileSize"
fun build(source: String, fileName: String, fileSize: Int): NumassDataPointEvent {
return NumassDataPointEvent(builder(source, fileName, fileSize).buildEventMeta())

View File

@ -7,6 +7,7 @@ import inr.numass.data.api.NumassBlock
import inr.numass.data.api.NumassEvent
import inr.numass.data.api.NumassFrame
import inr.numass.data.api.NumassPoint
import inr.numass.data.dataStream
import inr.numass.data.legacy.NumassFileEnvelope
import java.io.IOException
import java.nio.file.Path
@ -24,7 +25,7 @@ class ProtoNumassPoint(private val envelope: Envelope) : NumassPoint {
private val point: NumassProto.Point
get() = try {
envelope.data.stream.use { stream -> return NumassProto.Point.parseFrom(stream) }
envelope.dataStream.use { stream -> return NumassProto.Point.parseFrom(stream) }
} catch (ex: IOException) {
throw RuntimeException("Failed to read point via protobuf")
}
@ -33,7 +34,7 @@ class ProtoNumassPoint(private val envelope: Envelope) : NumassPoint {
return point.channelsList.stream()
.flatMap { channel ->
channel.blocksList.stream()
.map { block -> ProtoBlock(channel.num.toInt(), block) }
.map { block -> ProtoBlock(channel.num.toInt(), block, meta) }
.sorted(Comparator.comparing<ProtoBlock, Instant> { it.startTime })
}
}
@ -42,7 +43,7 @@ class ProtoNumassPoint(private val envelope: Envelope) : NumassPoint {
return envelope.meta
}
private inner class ProtoBlock(internal val channel: Int, internal val block: NumassProto.Point.Channel.Block) : NumassBlock {
class ProtoBlock(val channel: Int, private val block: NumassProto.Point.Channel.Block, private val meta: Meta) : NumassBlock {
override fun getStartTime(): Instant {
return ofEpochNanos(block.time)

View File

@ -57,7 +57,7 @@ public class SpectrumInformation {
* @return
*/
public NamedMatrix getInformationMatrix(Values set, ListTable data, String... parNames) {
SpectrumAdapter adapter = NumassDataUtils.adapter();
SpectrumAdapter adapter = NumassDataUtils.INSTANCE.adapter();
String[] names = parNames;
if (names.length == 0) {

View File

@ -139,13 +139,17 @@ class StorageView(private val context: Context = Global) : View(title = "Numass
if (it != null) {
root = TreeItem(Container(it.name, it))
root.isExpanded = true
populate { parent ->
val value = parent.value.content
when (value) {
is Storage -> (value.shelves().sorted() + value.loaders().sorted()).map { buildContainer(it, parent.value) }
is NumassSet -> value.points.map { buildContainer(it, parent.value) }.toList().sortedBy { it.id }
else -> null
runGoal("populateTree") {
runLater { statusBar.progress = -1.0 }
populate { parent ->
val value = parent.value.content
when (value) {
is Storage -> (value.shelves().sorted() + value.loaders().sorted()).map { buildContainer(it, parent.value) }
is NumassSet -> value.points.map { buildContainer(it, parent.value) }.toList().sortedBy { it.id }
else -> null
}
}
runLater { statusBar.progress = 0.0 }
}
}
}
@ -179,7 +183,7 @@ class StorageView(private val context: Context = Global) : View(title = "Numass
}
}
contextMenu = ContextMenu()
contextMenu.item("Clear all"){
contextMenu.item("Clear all") {
action {
this@cellFormat.treeItem.uncheckAll()
}
@ -234,9 +238,9 @@ class StorageView(private val context: Context = Global) : View(title = "Numass
}
private fun TreeItem<Container>.uncheckAll(){
private fun TreeItem<Container>.uncheckAll() {
this.value.checked = false
this.children.forEach{it.uncheckAll()}
this.children.forEach { it.uncheckAll() }
}
@ -273,20 +277,15 @@ class StorageView(private val context: Context = Global) : View(title = "Numass
private fun loadDirectory(path: URI) {
statusBar.text = "Loading storage: $path"
statusBar.progress = -1.0;
runGoal("loadDirectory[$path]") {
title = "Load storage ($path)"
progress = -1.0
message = "Building numass storage tree..."
(StorageManager.buildStorage(context, NumassStorageFactory.buildStorageMeta(path, true, false)) as NumassStorage).also {
progress = 1.0
}
(StorageManager.buildStorage(context, NumassStorageFactory.buildStorageMeta(path, true, false)) as NumassStorage)
} ui {
storage = it
storageName = "Storage: " + path
storageName = "Storage: $path"
statusBar.text = "OK"
statusBar.progress = 0.0;
}
}