NPM/numass-framework
4
0
Fork 1
Code Issues Pull Requests Packages Projects Releases Wiki Activity

New numass data structures

Browse Source
This commit is contained in:
Alexander Nozik 2017-07-08 22:04:29 +03:00
parent d6cfdc25c1
commit badc119e78
47 changed files with 523 additions and 1266 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
numass-client/src/main/java/inr/numass/client/NumassClient.java
View File

@ -26,7 +26,7 @@ import hep.dataforge.storage.commons.MessageFactory;
import hep.dataforge.storage.commons.StorageUtils;
import hep.dataforge.values.Value;
import hep.dataforge.values.Values;
import inr.numass.storage.NumassStorage;
import inr.numass.data.storage.NumassStorage;
import org.slf4j.LoggerFactory;
import org.zeroturnaround.zip.ZipUtil;

115
numass-core/src/main/java/inr/numass/data/NumassPointImpl.java
View File

@ -1,115 +0,0 @@
/*
* Copyright 2015 Alexander Nozik.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package inr.numass.data;
import hep.dataforge.tables.ValueMap;
import hep.dataforge.values.Values;
import inr.numass.data.legacy.RawNMPoint;
import java.time.Instant;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.IntStream;
/**
* @author Darksnake
*/
public class NumassPointImpl implements NumassPoint {
//TODO andThen to annotated and move some parameters to meta
private final int[] spectrum;
private Instant startTime;
private long eventsCount;
private double pointLength;
private double u;
public NumassPointImpl(double u, Instant startTime, double pointLength, int[] spectrum) {
this.startTime = startTime;
this.pointLength = pointLength;
this.spectrum = spectrum;
this.u = u;
this.eventsCount = IntStream.of(spectrum).sum();
}
/**
* @return the absouteTime
*/
@Override
public Instant getStartTime() {
if (startTime == null) {
return Instant.EPOCH;
} else {
return startTime;
}
}
@Override
public int getCount(int chanel) {
return spectrum[chanel];
}
@Override
public int getCountInWindow(int from, int to) {
int res = 0;
for (int i = from; i <= to; i++) {
res += spectrum[i];
}
if (res == Integer.MAX_VALUE) {
throw new RuntimeException("integer overflow in spectrum calculation");
}
return res;
}
@Override
public List<Values> getData() {
List<Values> data = new ArrayList<>();
for (int i = 0; i < RawNMPoint.MAX_CHANEL; i++) {
data.add(new ValueMap(dataNames, i, spectrum[i]));
}
return data;
}
/**
* Events count including overflow
*
* @return
*/
@Override
public long getTotalCount() {
return eventsCount;
}
/**
* @return the pointLength
*/
@Override
public double getLength() {
return pointLength;
}
/**
* @return the u
*/
@Override
public double getVoltage() {
return u;
}
@Override
public int[] getSpectrum() {
return spectrum;
}
}

1
numass-core/src/main/java/inr/numass/data/PointBuilders.java
View File

@ -1,6 +1,5 @@
package inr.numass.data;
import inr.numass.data.legacy.RawNMPoint;
import org.jetbrains.annotations.NotNull;
import java.io.IOException;

1
numass-core/src/main/java/inr/numass/data/SetDirectionUtility.java
View File

@ -18,6 +18,7 @@ import java.util.function.Function;
*
* @author Alexander Nozik
*/
@Deprecated
public class SetDirectionUtility {
private static final String FILE_NAME = "numass_set_direction.map";

3
numass-core/src/main/java/inr/numass/data/analyzers/SimpleAnalyzer.java
View File

@ -24,6 +24,9 @@ public class SimpleAnalyzer implements NumassAnalyzer {
public SimpleAnalyzer(@Nullable SignalProcessor processor) {
this.processor = processor;
}
public SimpleAnalyzer() {
this.processor = null;
}
/**
* Return unsorted stream of events including events from frames

19
numass-core/src/main/java/inr/numass/data/api/NumassEvent.java
View File

@ -17,16 +17,21 @@ package inr.numass.data.api;
import org.jetbrains.annotations.NotNull;
import java.io.Serializable;
import java.time.Instant;
/**
* A single numass event with given amplitude ant time.
*
* @author Darksnake
*/
public class NumassEvent implements Comparable<NumassEvent> {
public class NumassEvent implements Comparable<NumassEvent>, Serializable {
// channel
protected final short chanel;
protected final double time;
//time in nanoseconds
protected final long time;
public NumassEvent(short chanel, double time) {
public NumassEvent(short chanel, long time) {
this.chanel = chanel;
this.time = time;
}
@ -41,12 +46,16 @@ public class NumassEvent implements Comparable<NumassEvent> {
/**
* @return the time
*/
public double getTime() {
public long getTime() {
return time;
}
public Instant getAbsoluteTime(Instant offset) {
return offset.plusNanos(time);
}
@Override
public int compareTo(@NotNull NumassEvent o) {
return Double.compare(this.getTime(), o.getTime());
return Long.compare(this.getTime(), o.getTime());
}
}

43
numass-core/src/main/java/inr/numass/data/api/SimpleBlock.java Normal file
View File

@ -0,0 +1,43 @@
package inr.numass.data.api;
import java.io.Serializable;
import java.time.Duration;
import java.time.Instant;
import java.util.List;
import java.util.stream.Stream;
/**
* A simple in-memory implementation of block of events. No frames are allowed
* Created by darksnake on 08.07.2017.
*/
public class SimpleBlock implements NumassBlock, Serializable {
private final Instant startTime;
private final Duration length;
private final List<NumassEvent> events;
public SimpleBlock(Instant startTime, Duration length, List<NumassEvent> events) {
this.startTime = startTime;
this.length = length;
this.events = events;
}
@Override
public Instant getStartTime() {
return startTime;
}
@Override
public Duration getLength() {
return length;
}
@Override
public Stream<NumassEvent> getEvents() {
return events.stream();
}
@Override
public Stream<NumassFrame> getFrames() {
return Stream.empty();
}
}

31
numass-core/src/main/java/inr/numass/data/api/SimpleNumassPoint.java Normal file
View File

@ -0,0 +1,31 @@
package inr.numass.data.api;
import hep.dataforge.meta.Meta;
import hep.dataforge.meta.MetaBuilder;
import hep.dataforge.utils.MetaHolder;
import java.util.List;
import java.util.stream.Stream;
/**
* A simple static implementation of NumassPoint
* Created by darksnake on 08.07.2017.
*/
public class SimpleNumassPoint extends MetaHolder implements NumassPoint {
private final List<NumassBlock> blocks;
public SimpleNumassPoint(double voltage, List<NumassBlock> blocks) {
this.blocks = blocks;
super.setMeta(new MetaBuilder("point").setValue(HV_KEY, voltage));
}
public SimpleNumassPoint(Meta meta, List<NumassBlock> blocks) {
super(meta);
this.blocks = blocks;
}
@Override
public Stream<NumassBlock> getBlocks() {
return blocks.stream();
}
}

517
numass-core/src/main/java/inr/numass/data/legacy/LegacyDataReader.java
View File

@ -1,517 +0,0 @@
/*
* Copyright 2015 Alexander Nozik.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package inr.numass.data.legacy;
import hep.dataforge.data.FileDataFactory;
import hep.dataforge.data.binary.Binary;
import hep.dataforge.meta.Meta;
import inr.numass.data.api.NumassEvent;
import java.io.*;
import java.time.LocalDateTime;
import java.time.ZoneOffset;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.List;
import java.util.Scanner;
/**
*
* @author Darksnake, based on program by S.V.Zadorozhny, 1996
*/
public class LegacyDataReader {
private final InputStream stream;
private String name;
private double HVdev = 2.468555393226049;
private boolean noUset = false;
public LegacyDataReader(Binary file, Meta config) throws IOException {
this(file.getStream(), config.getString(FileDataFactory.FILE_NAME_KEY), config);
}
public LegacyDataReader(File file) throws IOException {
this(new FileInputStream(file), file.getName(), Meta.empty());
}
public LegacyDataReader(String file, String fname, Meta config) throws FileNotFoundException {
this(new FileInputStream(file), fname, config);
if ((fname == null) || (fname.isEmpty())) {
name = file;
}
}
public LegacyDataReader(InputStream is, String fname, Meta config) {
this.stream = new BufferedInputStream(is);
this.name = fname;
HVdev = config.getDouble("HVdev", 2.468555393226049);
noUset = config.getBoolean("noUset", false);
}
public RawNMFile read() throws IOException {
RawNMFile res = readFile(name);
stream.close();
return res;
}
private int[] readBlock(int length) throws IOException {
int[] res = new int[length];
for (int i = 0; i < res.length; i++) {
res[i] = readByte();
}
return res;
}
private int readByte() throws IOException {
return stream.read();
}
private short readChanel(int hb) throws IOException {
assert hb < 127;
return (short) (readByte() + 256 * hb);
}
private LocalDateTime readDate(String head) throws IOException {
// Должны считать 14 символов
Scanner sc = new Scanner(head);
sc.nextLine();
String dateStr = sc.nextLine().trim();
//DD.MM.YY HH:MM
//12:35:16 19-11-2013
DateTimeFormatter format = DateTimeFormatter.ofPattern("HH:mm:ss dd-MM-yyyy");
LocalDateTime date = LocalDateTime.parse(dateStr, format);
return date;
}
private NumassEvent readEvent(int b, double timeDiv) throws IOException {
short chanel;
long time;
int hb = (b & 0x0f);
int lab = (b & 0xf0);
switch (lab) {
case 0x10:
chanel = readChanel(hb);
time = readTime();
break;
case 0x20:
chanel = 0;
time = readTime();
break;
case 0x40:
time = 0;
chanel = readChanel(hb);
break;
case 0x80:
time = 0;
chanel = 0;
break;
default:
throw new IOException("Event head expected");
}
return new NumassEvent(chanel, time / timeDiv);
}
private RawNMFile readFile(String name) throws IOException {
String head = readHead();//2048
RawNMFile file = new RawNMFile(name, head.replaceAll("\u0000", ""));
LocalDateTime filedate = readDate(head);
int lab = readByte();
do {
RawNMPoint point = readPoint(lab, filedate);
//Устанавливаем дату точки
file.putPoint(point);
lab = readByte();
} while (lab != 0xff);
//Check if input file head is numass data file
return file;
}
private String readHead() throws IOException {
byte[] bytes = new byte[2048];
stream.read(bytes);
return new String(bytes);
}
private RawNMPoint readPoint(int head, LocalDateTime filedate) throws IOException {
int[] rx;
int voltage;
short time_out;
double timeDiv = 0;
boolean phoneFlag;
RawNMPoint point = new RawNMPoint();
int lab = head;
//point head
if (lab != 0x0f) {
//Ожидается, что это голова точки
throw new IOException("Point head expected");
}
rx = readBlock(32);
voltage = rx[2] + 256 * rx[3];
voltage = 65536 * voltage + rx[0] + 256 * rx[1];
time_out = (short) (rx[6] + 256 * rx[7]);
phoneFlag = (rx[19] != 0);
switch (time_out) {
case 5:
case 10:
timeDiv = 2e7;
break;
case 15:
case 20:
timeDiv = 1e7;
break;
case 50:
timeDiv = 5e6;
break;
case 100:
timeDiv = 2.5e6;
break;
case 200:
timeDiv = 1.25e6;
break;
default:
throw new IOException("Unknown time divider in input data");
}
if (phoneFlag) {
timeDiv /= 20.0;
time_out *= 20;
}
lab = readByte();
List<NumassEvent> events = new ArrayList<>();
while (lab == 0xBF) {
skip(4);//badHV
lab = readByte();
}
do {
events.add(readEvent(lab, timeDiv));
lab = readByte();
} while (lab != 0xAF);
//point end
skip(37);
int hours = readByte();
int minutes = readByte();
LocalDateTime absoluteTime = filedate.withHour(hours).withMinute(minutes);
//проверяем, не проскочили ли мы полночь
if (absoluteTime.isBefore(filedate)) {
absoluteTime = absoluteTime.plusDays(1);
}
rx = readBlock(4);
int Uread = rx[2] + 256 * rx[3];
Uread = 65536 * Uread + rx[0] + 256 * rx[1];
skip(21);
double uset;
if (noUset) {
uset = Uread / 10d / HVdev;
} else {
uset = voltage / 10d;
}
return new RawNMPoint(uset, Uread / 10d / HVdev, events, time_out, absoluteTime.toInstant(ZoneOffset.UTC));
}
private long readTime() throws IOException {
int[] rx = readBlock(4);
long time = rx[0] + 256 * rx[1] + 65536 * rx[2] + 256 * 65536 * rx[3];
return time;
}
private void skip(int length) throws IOException {
long n = stream.skip(length);
if (n != length) {
stream.skip(length - n);
}
}
}
/*
PROGRAM DAT2PAW;
{$N+}
{$D Copyright (C) 1996 by S.V.Zadorozhny, INR RAS}
Uses Dos, Crt;
CONST
Rec_Len = 32768; { Buffer size }
TYPE
Dim = ARRAY [1..Rec_Len] OF Byte; { Read / Write Buffer }
VAR
DATA : WORD;
TIME : LONGINT;
TD : Double;
i, j, k : WORD;
Voltage, li, CN : LONGINT;
St, St1 : String;
f : File;
txt : Text;
Buf : ^Dim;
Buf_point : Word;
err : INTEGER;
Monitor, Pressure : WORD;
paths : PathStr;
dirs : DirStr;
nams : NameStr;
exts : ExtStr;
name, mask : String;
sr : SearchRec;
rx : Array [1..64] OF Byte;
lab, bt : Byte;
time_out : Word;
hb : Word;
{Gate_L, Gate_H : Word;}
TimDiv : Double;
Phone_Flag, OK : Boolean;
live : word;
(*----------------------------- PROCEDURES ----------------------------------*)
PROCEDURE NextChar;
BEGIN
CASE (live) OF
0 : BEGIN
Write('|');
Inc(live);
END;
1 : BEGIN
Write(#8'/');
Inc(live);
END;
2 : BEGIN
Write(#8'--');
Inc(live);
END;
3 : BEGIN
Write(#8#8'\ ');
Inc(live);
END;
4 : BEGIN
Write(#8#8'|');
live := 1;
END;
END;
END;
PROCEDURE ReadB(VAR bt : BYTE);
VAR fact : INTEGER;
BEGIN
IF Buf_point = 0 THEN
BEGIN
BlockRead(f,Buf^,Rec_Len,fact);
NextChar;
END;
Inc(Buf_point);
bt := Buf^[Buf_point];
IF Buf_point = Rec_Len THEN Buf_point:=0;
END { ReadB };
PROCEDURE SaveTXT;
BEGIN
Writeln(txt, CN, #9, TD:10:5, #9, DATA, #9, time_out, #9, (voltage/10):7:1);
END;
PROCEDURE ReadFile;
BEGIN
Assign(f,name);
{$I-}
Reset(f,1);
{$I+}
IF (IOResult <> 0) THEN
BEGIN
Writeln('Can''t open data file ', name);
Halt(3);
END;
BlockRead(f,Buf^,2048,i); {Skip First Block}
Buf_point := 0;
IF ((Buf^[1] <> Ord('N'))OR(Buf^[2] <> Ord('M'))) THEN
BEGIN
Close(f);
Writeln('File ', name, ' is not a data file !');
Exit;
END;
Write('File : ',name, ' ');
paths := name;
Fsplit(paths, dirs, nams, exts);
Assign(txt, nams + '.paw');
Rewrite(txt);
live := 0;
CN := 0;
REPEAT
ReadB(lab);
hb := lab AND $0F; { High data byte }
lab := lab AND $F0;
CASE (lab) OF
$00 : { New Point }
BEGIN
FOR i:=1 TO 32 DO ReadB(rx[i]);
Voltage := rx[3] + 256 * Word(rx[4]);
Voltage := 65536 * Voltage + rx[1] + 256 * Longint(rx[2]);
time_out := rx[7] + 256 * Word(rx[8]);
{pressure := rx[11] + 256 * Word(rx[12]);}
{Chan := rx[19];}
IF (rx[20] <> 0) THEN Phone_Flag := TRUE ELSE Phone_Flag := FALSE;
CASE (time_out) OF
5, 10 : TimDiv := 20000000.0;
15, 20 : TimDiv := 10000000.0;
50 : TimDiv := 5000000.0;
100 : TimDiv := 2500000.0;
200 : TimDiv := 1250000.0;
END;
IF (Phone_Flag) THEN
BEGIN
TimDiv := TimDiv / 20.0;
time_out := time_out * 20;
END;
END;
$10 : { Event, all present }
BEGIN
FOR i:=1 TO 5 DO ReadB(rx[i]);
TIME := rx[2] + 256 * Longint(rx[3])
+ 65536 * Longint(rx[4])+ 256 * 65536 * Longint(rx[5]);
TD := TIME / TimDiv;
DATA := rx[1] + 256 * hb;
Inc(CN);
SaveTXT;
END;
$20 : { Event, time only }
BEGIN
FOR i:=1 TO 4 DO ReadB(rx[i]);
TIME := rx[1] + 256 * Longint(rx[2])
+ 65536 * Longint(rx[3])+ 256 * 65536 * Longint(rx[4]);
TD := TIME / TimDiv;
DATA := 0;
Inc(CN);
SaveTXT;
END;
$40 : { Event, code only }
BEGIN
ReadB(rx[1]);
DATA := rx[1] + 256 * hb;
TD := 0.0;
Inc(CN);
SaveTXT;
END;
$80 : { Event, nothing }
BEGIN
DATA := 0;
TD := 0.0;
Inc(CN);
SaveTXT;
END;
$B0 : { Bad HV }
BEGIN
FOR i:=1 TO 4 DO ReadB(rx[i]);
END;
$A0 : { End of the point }
BEGIN
FOR i := 1 TO 64 DO ReadB(rx[i]);
END;
END;
UNTIL (lab=$F0);
Close(f);
Close(txt);
Writeln(#8#8' ');
END;
PROCEDURE ReadDataFiles;
BEGIN
mask := ParamStr(1);
FindFirst(mask, anyfile, sr);
WHILE (DosError = 0) DO
BEGIN
name := sr.Name;
ReadFile;
FindNext(sr);
END
END;
(*--------------------------- Main ------------------------------------------*)
BEGIN
ClrScr;
Writeln('TROITSK V-MASS EXPERIMENT DATA FILE -> TEXT FILE CONVERTER');
Writeln('WRITTEN BY S.V.ZADOROZHNY');
Writeln;
IF (ParamCount <> 1) THEN
BEGIN
Writeln('Usage : dat2paw <mask.dat>');
Writeln;
Writeln('--- Press any key to continue ---');
ReadKey;
Halt(1);
END;
New(Buf);
ReadDataFiles;
Writeln;
Writeln('O.K.');
Dispose(Buf);
END. {-- Main --}
*/

68
numass-core/src/main/java/inr/numass/data/legacy/NMFile.java
View File

@ -1,68 +0,0 @@
/*
* Copyright 2015 Alexander Nozik.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package inr.numass.data.legacy;
import hep.dataforge.description.ValueDef;
import hep.dataforge.meta.Meta;
import hep.dataforge.names.NamedMetaHolder;
import inr.numass.data.PointBuilders;
import inr.numass.data.api.NumassPoint;
import inr.numass.data.api.NumassSet;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Stream;
/**
* Объект, содержащий только спектры, но не сами события
*
* @author Darksnake
*/
@ValueDef(name = "numass.path", info = "Path to this data file in numass repository.")
@ValueDef(name = "numass.name", info = "The name of this data file.")
public class NMFile extends NamedMetaHolder implements NumassSet {
private final List<NumassPoint> points;
public NMFile(RawNMFile file) {
super(file.getName(), file.meta());
points = new ArrayList<>();
for (RawNMPoint point : file.getData()) {
points.add(PointBuilders.readRawPoint(point));
}
}
public static NMFile readStream(InputStream is, String fname, Meta config) throws IOException {
return new NMFile(new LegacyDataReader(is, fname, config).read());
}
public static NMFile readFile(File file) throws IOException {
return new NMFile(new LegacyDataReader(file).read());
}
@Override
public Stream<NumassPoint> getPoints() {
return points.stream();
}
@Override
public String getDescription() {
return "";
}
}

265
numass-core/src/main/java/inr/numass/data/legacy/NumassDatFile.java Normal file
View File

@ -0,0 +1,265 @@
package inr.numass.data.legacy;
import hep.dataforge.meta.Meta;
import hep.dataforge.meta.MetaBuilder;
import inr.numass.data.api.*;
import org.apache.commons.io.FilenameUtils;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.nio.channels.SeekableByteChannel;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.Duration;
import java.time.LocalDateTime;
import java.time.ZoneOffset;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Scanner;
import java.util.stream.Stream;
import static inr.numass.data.api.NumassPoint.HV_KEY;
import static java.nio.file.StandardOpenOption.READ;
/**
* Created by darksnake on 08.07.2017.
*/
public class NumassDatFile implements NumassSet {
private final String name;
private final Path path;
private final Meta meta;
public NumassDatFile(Path path, Meta meta) throws IOException {
this(FilenameUtils.getBaseName(path.getFileName().toString()),path,meta);
}
public NumassDatFile(String name, Path path, Meta meta) throws IOException {
this.name = name;
this.path = path;
String head = readHead(path);//2048
this.meta = new MetaBuilder(meta)
.setValue("info", head)
.setValue(NumassPoint.START_TIME_KEY, readDate(head))
.build();
}
@Override
public Meta meta() {
return meta;
}
@Override
public String getName() {
return name;
}
private double getHVdev() {
return meta().getDouble("dat.hvDev", 2.468555393226049);
}
private boolean hasUset() {
return meta().getBoolean("dat.uSet", true);
}
private static String readHead(Path path) throws IOException {
try (SeekableByteChannel channel = Files.newByteChannel(path, READ)) {
channel.position(0);
ByteBuffer buffer = ByteBuffer.allocate(2048);
channel.read(buffer);
return new String(buffer.array()).replaceAll("\u0000", "");
}
}
/**
* Read the block at current position
*
* @param channel
* @param length
* @return
* @throws IOException
*/
private ByteBuffer readBlock(SeekableByteChannel channel, int length) throws IOException {
ByteBuffer res = ByteBuffer.allocate(length);
channel.read(res);
res.order(ByteOrder.LITTLE_ENDIAN);
res.flip();
return res;
}
/**
* Read the point at current position
*
* @param channel
* @return
* @throws IOException
*/
private synchronized NumassPoint readPoint(SeekableByteChannel channel) throws
IOException {
ByteBuffer rx = readBlock(channel, 32);
int voltage = rx.getInt();
short length = rx.getShort();//(short) (rx[6] + 256 * rx[7]);
boolean phoneFlag = rx.get(19) != 0;//(rx[19] != 0);
double timeDiv;
switch (length) {
case 5:
case 10:
timeDiv = 2e7;
break;
case 15:
case 20:
timeDiv = 1e7;
break;
case 50:
timeDiv = 5e6;
break;
case 100:
timeDiv = 2.5e6;
break;
case 200:
timeDiv = 1.25e6;
break;
default:
throw new IOException("Unknown time divider in input data");
}
if (phoneFlag) {
timeDiv /= 20.0;
length *= 20;
}
List<NumassEvent> events = new ArrayList<>();
int lab = readBlock(channel, 1).get();
while (lab == 0xBF) {
ByteBuffer buffer = readBlock(channel, 5);
lab = buffer.get(4);
}
do {
events.add(readEvent(channel, lab, timeDiv));
lab = readBlock(channel, 1).get();
} while (lab != 0xAF);
//point end
ByteBuffer ending = readBlock(channel, 64);
int hours = ending.get(37);
int minutes = ending.get(38);
LocalDateTime start = LocalDateTime.from(getStartTime());
LocalDateTime absoluteTime = start.withHour(hours).withMinute(minutes);
//проверяем, не проскочили ли мы полночь
if (absoluteTime.isBefore(start)) {
absoluteTime = absoluteTime.plusDays(1);
}
int uRead = ending.getInt(39);
double uSet;
if (!this.hasUset()) {
uSet = uRead / 10d / getHVdev();
} else {
uSet = voltage / 10d;
}
NumassBlock block = new SimpleBlock(absoluteTime.toInstant(ZoneOffset.UTC), Duration.ofSeconds(length), events);
Meta pointMeta = new MetaBuilder("point")
.setValue(HV_KEY, uSet)
.setValue("uRead", uRead / 10 / getHVdev())
.setValue("source", "legacy");
return new SimpleNumassPoint(pointMeta, Collections.singletonList(block));
}
@Override
public Stream<NumassPoint> getPoints() {
try (SeekableByteChannel channel = Files.newByteChannel(path, READ)) {
//int lab = readBlock(channel,1).get();
int lab;
List<NumassPoint> points = new ArrayList<>();
do {
//TODO check point start
points.add(readPoint(channel));
lab = readBlock(channel, 1).get();
} while (lab != 0xff);
return points.stream();
} catch (IOException ex) {
throw new RuntimeException(ex);
}
}
private LocalDateTime readDate(String head) throws IOException {
// Должны считать 14 символов
Scanner sc = new Scanner(head);
sc.nextLine();
String dateStr = sc.nextLine().trim();
//DD.MM.YY HH:MM
//12:35:16 19-11-2013
DateTimeFormatter format = DateTimeFormatter.ofPattern("HH:mm:ss dd-MM-yyyy");
return LocalDateTime.parse(dateStr, format);
}
private NumassEvent readEvent(SeekableByteChannel channel, int b, double timeDiv) throws IOException {
short chanel;
long time;
int hb = (b & 0x0f);
int lab = (b & 0xf0);
switch (lab) {
case 0x10:
chanel = readChanel(channel, hb);
time = readTime(channel);
break;
case 0x20:
chanel = 0;
time = readTime(channel);
break;
case 0x40:
time = 0;
chanel = readChanel(channel, hb);
break;
case 0x80:
time = 0;
chanel = 0;
break;
default:
throw new IOException("Event head expected");
}
return new NumassEvent(chanel, (long) (time / timeDiv));
}
private short readChanel(SeekableByteChannel channel, int hb) throws IOException {
assert hb < 127;
ByteBuffer buffer = readBlock(channel, 1);
return (short) (buffer.get() + 256 * hb);
}
private long readTime(SeekableByteChannel channel) throws IOException {
ByteBuffer rx = readBlock(channel, 4);
return rx.getLong();//rx[0] + 256 * rx[1] + 65536 * rx[2] + 256 * 65536 * rx[3];
}
// private void skip(int length) throws IOException {
// long n = stream.skip(length);
// if (n != length) {
// stream.skip(length - n);
// }
// }
}

144
numass-core/src/main/java/inr/numass/data/legacy/RawNMFile.java
View File

@ -1,144 +0,0 @@
/*
* Copyright 2015 Alexander Nozik.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package inr.numass.data.legacy;
import hep.dataforge.description.ValueDef;
import hep.dataforge.meta.Meta;
import hep.dataforge.meta.MetaBuilder;
import hep.dataforge.names.NamedMetaHolder;
import inr.numass.data.api.NumassEvent;
import java.io.BufferedOutputStream;
import java.io.OutputStream;
import java.io.PrintWriter;
import java.util.ArrayList;
import java.util.List;
/**
* Contains the whole data but requires a lot of memory
*
* @author Darksnake
*/
@ValueDef(name = "info", info = "file text header")
public class RawNMFile extends NamedMetaHolder {
// public static String TYPE = ":data:numassdatafile";
private final List<RawNMPoint> points = new ArrayList<>();
public RawNMFile(String fileName) {
super(fileName);
}
public RawNMFile(String name, Meta meta) {
super(name, meta);
}
public RawNMFile(String name, String header) {
super(name, new MetaBuilder("meta").setValue("info", header));
}
public String getHead() {
return meta().getString("info", "");
}
@Deprecated
public void generatePAW(OutputStream stream) {
PrintWriter writer = new PrintWriter(new BufferedOutputStream(stream));
long counter = 0;
for (RawNMPoint point : this.getData()) {
double U = point.getUread();
for (NumassEvent event : point.getEvents()) {
counter++;
writer.printf("%d\t%f\t%d\t%.1f\t%.2f%n", counter, event.getTime(), event.getChanel(), point.getLength(), U);
}
}
writer.flush();
}
/**
* merge of all point with given Uset
*
* @param U
* @return
*/
public RawNMPoint getByUset(double U) {
RawNMPoint res = null;
for (RawNMPoint point : points) {
if (point.getUset() == U) {
if (res == null) {
res = point;
} else {
res = res.merge(point);
}
}
}
return res;
}
/**
* merge of all point with given Uread
*
* @param U
* @return
*/
public RawNMPoint getByUread(double U) {
RawNMPoint res = null;
for (RawNMPoint point : points) {
if (point.getUread() == U) {
if (res == null) {
res = point;
} else {
res = res.merge(point);
}
}
}
return res;
}
/**
* @return the data
*/
public List<RawNMPoint> getData() {
return points;
}
// public void putEvent(double U, short chanel, double time) {
// for (RawNMPoint point : this.getData()) {
// if (U == point.getUread()) {
// point.putEvent(new NumassEvent(chanel, time));
// return;
// }
// }
// RawNMPoint newpoint = new RawNMPoint();
// newpoint.putEvent(new NumassEvent(chanel, time));
// this.putPoint(newpoint);
// }
public void putPoint(RawNMPoint point) {
points.add(point);
}
public int size() {
return this.points.size();
}
}

186
numass-core/src/main/java/inr/numass/data/legacy/RawNMPoint.java
View File

@ -1,186 +0,0 @@
/*
* Copyright 2015 Alexander Nozik.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package inr.numass.data.legacy;
import inr.numass.data.api.NumassEvent;
import java.time.Instant;
import java.util.ArrayList;
import java.util.List;
/**
* Хранит информацию о спектре точки, но не об отдельных событиях.
*
* @author Darksnake
*/
public class RawNMPoint {
public static final int MAX_EVENTS_PER_POINT = 260000;
public static int MAX_CHANEL = 4095;
private Instant startTime;
private final List<NumassEvent> events;
private double length;
private double uread;
private double uset;
public RawNMPoint(double U, List<NumassEvent> events, double t) {
this.uset = U;
this.uread = U;
this.events = events;
this.length = t;
}
public RawNMPoint(double Uset, double Uread, List<NumassEvent> events, double t) {
this.uset = Uset;
this.uread = Uread;
this.events = events;
this.length = t;
}
public RawNMPoint(double uset, double uread, List<NumassEvent> events, double t, Instant absouteTime) {
this.uset = uset;
this.uread = uread;
this.length = t;
this.startTime = absouteTime;
this.events = events;
}
RawNMPoint() {
events = new ArrayList<>();
uset = 0;
uread = 0;
length = Double.NaN;
}
// @Override
// public RawNMPoint clone() {
// ArrayList<NumassEvent> newevents = new ArrayList<>();
// newevents.addAll(this.getEvents());
// return new RawNMPoint(getUset(), getUread(), newevents, getLength());
// }
public Instant getStartTime() {
return startTime;
}
public double getCr() {
return getEventsCount() / getLength();
}
public double getCrError() {
return Math.sqrt(getEventsCount()) / getLength();
}
/**
* @return the events
*/
public List<NumassEvent> getEvents() {
return events;
}
public long getEventsCount() {
return events.size();
}
/**
* Measurement time
*
* @return the tset
*/
public double getLength() {
if (Double.isNaN(length)) {
throw new Error();
}
return length;
}
/**
* @return the Uread
*/
public double getUread() {
if (uread <= 0) {
return getUset();
} else {
return uread;
}
}
/**
* @return the Uset
*/
public double getUset() {
if (uset < 0) {
throw new IllegalStateException();
}
return uset;
}
public RawNMPoint merge(RawNMPoint point) {
List<NumassEvent> events = new ArrayList<>(this.events);
for (NumassEvent newEvent : point.getEvents()) {
events.add(new NumassEvent(newEvent.getChanel(), newEvent.getTime() + this.getLength()));
}
double length = this.length + point.length;
double uread = (this.uread + point.uread) / 2;
return new RawNMPoint(this.uset, uread, events, length, this.startTime);
}
// void putEvent(NumassEvent event) {
// events.add(event);
// }
public RawNMPoint selectChanels(int from, int to) {
assert to > from;
List<NumassEvent> res = new ArrayList<>();
for (NumassEvent event : this.getEvents()) {
if ((event.getChanel() >= from) && (event.getChanel() <= to)) {
res.add(event);
}
}
return new RawNMPoint(getUset(), getUread(), res, getLength());
}
// void setStartTime(Instant absouteTime) {
// this.startTime = absouteTime;
// }
//
// /**
// * @param tset the tset to set
// */
// void setLength(double tset) {
// this.length = tset;
// }
// /**
// * @param Uread the Uread to set
// */
// void setUread(double Uread) {
// assert Uread >= 0;
// this.uread = Uread;
// }
//
// /**
// * @param Uset the Uset to set
// */
// void setUset(double Uset) {
// this.uset = Uset;
// }
}

114
numass-core/src/main/java/inr/numass/data/storage/EnvelopeNumassPoint.java Normal file
View File

@ -0,0 +1,114 @@
package inr.numass.data.storage;
import hep.dataforge.io.envelopes.Envelope;
import hep.dataforge.meta.Meta;
import inr.numass.data.api.NumassBlock;
import inr.numass.data.api.NumassEvent;
import inr.numass.data.api.NumassFrame;
import inr.numass.data.api.NumassPoint;
import org.jetbrains.annotations.NotNull;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.io.InputStream;
import java.nio.ByteBuffer;
import java.time.Duration;
import java.time.Instant;
import java.util.Iterator;
import java.util.stream.Stream;
import java.util.stream.StreamSupport;
/**
* Created by darksnake on 08.07.2017.
*/
public class EnvelopeNumassPoint implements NumassPoint {
private final Envelope envelope;
public EnvelopeNumassPoint(Envelope envelope) {
this.envelope = envelope;
}
@Override
public Stream<NumassBlock> getBlocks() {
return null;
}
@Override
public Meta meta() {
return envelope.meta();
}
private class EnvelopeBlock implements NumassBlock, Iterable<NumassEvent> {
private final Instant startTime;
private final Duration length;
private final long blockOffset;
public EnvelopeBlock(Instant startTime, Duration length, long blockOffset) {
this.startTime = startTime;
this.length = length;
this.blockOffset = blockOffset;
}
@Override
public Instant getStartTime() {
return startTime;
}
@Override
public Duration getLength() {
return length;
}
@Override
public Stream<NumassEvent> getEvents() {
return StreamSupport.stream(this.spliterator(), false);
}
@NotNull
@Override
public Iterator<NumassEvent> iterator() {
double timeCoef = envelope.meta().getDouble("time_coeff", 50);
try {
InputStream stream = envelope.getData().getStream();
stream.skip(blockOffset);
return new Iterator<NumassEvent>() {
@Override
public boolean hasNext() {
try {
return stream.available() > 0;
} catch (IOException e) {
LoggerFactory.getLogger(EnvelopeNumassPoint.this.getClass()).error("Unexpected IOException " +
"when reading block", e);
return false;
}
}
@Override
public NumassEvent next() {
try {
byte[] bytes = new byte[7];
stream.read(bytes);
ByteBuffer buffer = ByteBuffer.wrap(bytes);
short channel = (short) Short.toUnsignedInt(buffer.getShort());
long time = Integer.toUnsignedLong(buffer.getInt());
byte status = buffer.get(); // status is ignored
return new NumassEvent(channel, (long) (time * timeCoef));
} catch (IOException ex) {
LoggerFactory.getLogger(EnvelopeNumassPoint.this.getClass()).error("Unexpected IOException " +
"when reading block", ex);
throw new RuntimeException(ex);
}
}
};
} catch (IOException ex){
throw new RuntimeException(ex);
}
}
@Override
public Stream<NumassFrame> getFrames () {
return Stream.empty();
}
}
}

13
numass-core/src/main/java/inr/numass/storage/NumassDataFactory.java → numass-core/src/main/java/inr/numass/data/storage/NumassDataFactory.java
View File

@ -1,4 +1,4 @@
package inr.numass.storage;
package inr.numass.data.storage;
import hep.dataforge.context.Context;
import hep.dataforge.data.DataFactory;
@ -6,14 +6,15 @@ import hep.dataforge.data.DataFilter;
import hep.dataforge.data.DataTree;
import hep.dataforge.meta.Meta;
import hep.dataforge.storage.commons.StorageUtils;
import inr.numass.data.api.NumassSet;
/**
* Created by darksnake on 03-Feb-17.
*/
public class NumassDataFactory extends DataFactory<NumassData> {
public class NumassDataFactory extends DataFactory<NumassSet> {
public NumassDataFactory() {
super(NumassData.class);
super(NumassSet.class);
}
@Override
@ -22,11 +23,11 @@ public class NumassDataFactory extends DataFactory<NumassData> {
}
@Override
protected void buildChildren(Context context, DataTree.Builder<NumassData> builder, DataFilter filter, Meta meta) {
protected void buildChildren(Context context, DataTree.Builder<NumassSet> builder, DataFilter filter, Meta meta) {
NumassStorage storage = new NumassStorage(context,meta);
StorageUtils.loaderStream(storage).forEach(pair -> {
if (pair.getValue() instanceof NumassData) {
builder.putStatic(pair.getKey(), (NumassData) pair.getValue());
if (pair.getValue() instanceof NumassSet) {
builder.putStatic(pair.getKey(), (NumassSet) pair.getValue());
}
});
}

187
numass-core/src/main/java/inr/numass/storage/NumassDataLoader.java → numass-core/src/main/java/inr/numass/data/storage/NumassDataLoader.java
View File

@ -13,7 +13,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package inr.numass.storage;
package inr.numass.data.storage;
import hep.dataforge.data.Data;
import hep.dataforge.exceptions.StorageException;
@ -22,30 +22,22 @@ import hep.dataforge.io.envelopes.Envelope;
import hep.dataforge.meta.Meta;
import hep.dataforge.meta.MetaBuilder;
import hep.dataforge.providers.Provider;
import hep.dataforge.providers.Provides;
import hep.dataforge.providers.ProvidesNames;
import hep.dataforge.storage.api.ObjectLoader;
import hep.dataforge.storage.api.Storage;
import hep.dataforge.storage.filestorage.FileEnvelope;
import hep.dataforge.storage.filestorage.FileStorage;
import hep.dataforge.storage.loaders.AbstractLoader;
import hep.dataforge.tables.Table;
import inr.numass.data.PointBuilders;
import inr.numass.data.api.NumassEvent;
import inr.numass.data.legacy.RawNMPoint;
import inr.numass.data.api.NumassPoint;
import inr.numass.data.api.NumassSet;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.nio.channels.ReadableByteChannel;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.Instant;
import java.util.*;
import java.util.function.Function;
import java.util.function.Supplier;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@ -54,7 +46,7 @@ import java.util.stream.Stream;
*
* @author darksnake
*/
public class NumassDataLoader extends AbstractLoader implements ObjectLoader<Envelope>, NumassData, Provider {
public class NumassDataLoader extends AbstractLoader implements ObjectLoader<Envelope>, NumassSet, Provider {
public static NumassDataLoader fromFile(Storage storage, Path zipFile) throws IOException {
@ -147,79 +139,6 @@ public class NumassDataLoader extends AbstractLoader implements ObjectLoader<Env
readOnly = true;
}
/**
* Read numass point from envelope and apply transformation (e.g. debuncing)
*
* @param envelope
* @param transformation
* @return
*/
private NumassPoint readPoint(Envelope envelope, Function<RawNMPoint, NumassPoint> transformation) {
return transformation.apply(readRawPoint(envelope));
}
/**
* Read raw point. Requires a low of memory.
*
* @param envelope
* @return
*/
private synchronized RawNMPoint readRawPoint(Envelope envelope) {
List<NumassEvent> events = new ArrayList<>();
double timeCoef = envelope.meta().getDouble("time_coeff", 50);
try (ReadableByteChannel inChannel = envelope.getData().getChannel()) {
ByteBuffer buffer = ByteBuffer.allocate(7 * 10000); // one event is 7b
buffer.order(ByteOrder.LITTLE_ENDIAN);
while (inChannel.read(buffer) > 0) {
buffer.flip();
while (buffer.hasRemaining()) {
short channel = (short) Short.toUnsignedInt(buffer.getShort());
long time = Integer.toUnsignedLong(buffer.getInt());
byte status = buffer.get(); // status is ignored
NumassEvent event = new NumassEvent(channel, (double) time * timeCoef * 1e-9);
events.add(event);
}
buffer.clear(); // do something with the data and clear/compact it.
}
} catch (Exception ex) {
throw new RuntimeException(ex);
}
// LocalDateTime startTime = envelope.meta().get
double u = envelope.meta().getDouble("external_meta.HV1_value", 0);
double pointTime;
if (envelope.meta().hasValue("external_meta.acquisition_time")) {
pointTime = envelope.meta().getValue("external_meta.acquisition_time").doubleValue();
} else {
pointTime = envelope.meta().getValue("acquisition_time").doubleValue();
}
//Check if the point is composite
boolean segmented = envelope.meta().getBoolean("split", false);
if (!segmented && events.size() > RawNMPoint.MAX_EVENTS_PER_POINT) {
pointTime = events.get(events.size() - 1).getTime() - events.get(0).getTime();
}
return new RawNMPoint(u, u,
events,
pointTime,
readTime(envelope.meta()));
}
/**
* Read numass point without transformation
*
* @param envelope
* @return
*/
private NumassPoint readPoint(Envelope envelope) {
return readPoint(envelope, PointBuilders::readRawPoint);
}
private Map<String, Supplier<Envelope>> getItems() {
return itemsProvider;
}
@ -239,7 +158,7 @@ public class NumassDataLoader extends AbstractLoader implements ObjectLoader<Env
}
@Override
public Data<Table> getHVData() {
public Data<Table> getHvData() {
Envelope hvEnvelope = getHVEnvelope();
if (hvEnvelope == null) {
return Data.buildStatic(null);
@ -263,20 +182,7 @@ public class NumassDataLoader extends AbstractLoader implements ObjectLoader<Env
}
}
@Override
public Stream<NumassPoint> stream() {
return this.getPoints().map(this::readPoint);
}
public List<NumassPoint> getNMPoints(Function<RawNMPoint, NumassPoint> transformation) {
return this.getPoints().map(env -> readPoint(env, transformation)).collect(Collectors.toList());
}
public List<RawNMPoint> getRawPoints() {
return this.getPoints().map(this::readRawPoint).collect(Collectors.toList());
}
private Stream<Envelope> getPoints() {
private Stream<Envelope> getPointEnvelopes() {
return getItems().entrySet().stream()
.filter(entry -> entry.getKey().startsWith(POINT_FRAGMENT_NAME) && entry.getValue() != null)
.map(entry -> entry.getValue().get())
@ -284,6 +190,11 @@ public class NumassDataLoader extends AbstractLoader implements ObjectLoader<Env
}
@Override
public Stream<NumassPoint> getPoints() {
return getPointEnvelopes().map(EnvelopeNumassPoint::new);
}
public boolean isReversed() {
return meta().getBoolean("iteration_info.reverse", false);
}
@ -310,21 +221,12 @@ public class NumassDataLoader extends AbstractLoader implements ObjectLoader<Env
}
@Override
public Instant startTime() {
public Instant getStartTime() {
if (meta.hasValue("start_time")) {
return meta().getValue("start_time").timeValue();
} else return null;
//Temporary substitution for meta tag
// Envelope hvEnvelope = getHVEnvelope();
// if (hvEnvelope != null) {
// try {
// return Value.of(new Scanner(hvEnvelope.getData().getStream()).next()).timeValue();
// } catch (IOException ex) {
// return null;
// }
// } else {
// return null;
// }
} else {
return NumassSet.super.getStartTime();
}
}
@Override
@ -337,63 +239,4 @@ public class NumassDataLoader extends AbstractLoader implements ObjectLoader<Env
}
@Override
public String defaultTarget() {
return "nmPoint";
}
@Provides("nmPoint")
public Optional<NumassPoint> optPoint(String u) {
return stream().filter(it -> it.getVoltage() == Double.parseDouble(u)).findFirst();
}
@ProvidesNames("nmPoint")
public Stream<String> listHV() {
return stream().map(it -> Double.toString(it.getVoltage()));
}
@Provides("rawPoint")
public Optional<RawNMPoint> optRawPoint(String u) {
return this.getPoints().map(this::readRawPoint).filter(it -> it.getUset() == Double.parseDouble(u)).findFirst();
}
/**
* Return new NumassData using given transformation for each point
*
* @param transform
* @return
*/
public NumassData applyRawTransformation(Function<RawNMPoint, NumassPoint> transform) {
return new NumassData() {
@Override
public String getDescription() {
return NumassDataLoader.this.getDescription();
}
@Override
public Meta meta() {
return NumassDataLoader.this.meta();
}
@Override
public Stream<NumassPoint> stream() {
return NumassDataLoader.this.stream();
}
@Override
public boolean isEmpty() {
return NumassDataLoader.this.isEmpty();
}
@Override
public Instant startTime() {
return NumassDataLoader.this.startTime();
}
@Override
public String getName() {
return NumassDataLoader.this.getName();
}
};
}
}

16
numass-core/src/main/java/inr/numass/storage/NumassStorage.java → numass-core/src/main/java/inr/numass/data/storage/NumassStorage.java
View File

@ -13,7 +13,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package inr.numass.storage;
package inr.numass.data.storage;
import hep.dataforge.context.Context;
import hep.dataforge.events.Event;
@ -21,7 +21,8 @@ import hep.dataforge.events.EventBuilder;
import hep.dataforge.exceptions.StorageException;
import hep.dataforge.meta.Meta;
import hep.dataforge.storage.filestorage.FileStorage;
import inr.numass.data.legacy.NMFile;
import inr.numass.data.api.NumassSet;
import inr.numass.data.legacy.NumassDatFile;
import org.slf4j.LoggerFactory;
import java.io.IOException;
@ -32,7 +33,8 @@ import java.nio.file.Path;
import java.util.ArrayList;
import java.util.List;
import static java.nio.file.StandardOpenOption.*;
import static java.nio.file.StandardOpenOption.CREATE;
import static java.nio.file.StandardOpenOption.WRITE;
/**
@ -64,7 +66,7 @@ public class NumassStorage extends FileStorage {
protected void updateDirectoryLoaders() {
try {
this.loaders.clear();
Files.list(getDataDir()).forEach( file -> {
Files.list(getDataDir()).forEach(file -> {
try {
if (Files.isDirectory(file)) {
Path metaFile = file.resolve(NumassDataLoader.META_FRAGMENT_NAME);
@ -141,14 +143,14 @@ public class NumassStorage extends FileStorage {
*
* @return
*/
public List<NumassData> legacyFiles() {
public List<NumassSet> legacyFiles() {
try {
List<NumassData> files = new ArrayList<>();
List<NumassSet> files = new ArrayList<>();
Files.list(getDataDir()).forEach(file -> {
if (Files.isRegularFile(file) && file.getFileName().toString().toLowerCase().endsWith(".dat")) {
String name = file.getFileName().toString();
try {
files.add(NMFile.readStream(Files.newInputStream(file, READ), name, Meta.buildEmpty("numassData")));
files.add(new NumassDatFile(file, Meta.empty()));
} catch (Exception ex) {
LoggerFactory.getLogger(getClass()).error("Error while reading legacy numass file " + file.getFileName(), ex);
}

2
numass-core/src/main/java/inr/numass/storage/NumassStorageFactory.java → numass-core/src/main/java/inr/numass/data/storage/NumassStorageFactory.java
View File

@ -1,4 +1,4 @@
package inr.numass.storage;
package inr.numass.data.storage;
import hep.dataforge.context.Context;
import hep.dataforge.context.Global;

2
numass-core/src/main/resources/META-INF/services/hep.dataforge.data.DataLoader
View File

@ -1 +1 @@
inr.numass.storage.NumassDataFactory
inr.numass.data.storage.NumassDataFactory

2
numass-core/src/main/resources/META-INF/services/hep.dataforge.storage.api.StorageType
View File

@ -1 +1 @@
inr.numass.storage.NumassStorageFactory
inr.numass.data.storage.NumassStorageFactory

1
numass-main/src/main/groovy/inr/numass/data/PointAnalyzer.groovy
View File

@ -4,7 +4,6 @@ import groovy.transform.CompileStatic
import hep.dataforge.maths.histogram.Histogram
import hep.dataforge.maths.histogram.UnivariateHistogram
import inr.numass.data.api.NumassEvent
import inr.numass.data.legacy.RawNMPoint
import java.util.stream.DoubleStream

2
numass-main/src/main/groovy/inr/numass/scripts/FindBorder.groovy
View File

@ -9,7 +9,7 @@ package inr.numass.scripts
import hep.dataforge.grind.GrindMetaBuilder
import hep.dataforge.meta.Meta
import inr.numass.actions.FindBorderAction
import inr.numass.storage.NumassDataLoader
import inr.numass.data.storage.NumassDataLoader
File dataDir = new File("D:\\Work\\Numass\\data\\2016_04\\T2_data\\Fill_2_2\\set_6_e26d123e54010000")
if(!dataDir.exists()){

4
numass-main/src/main/groovy/inr/numass/scripts/SimulatePileup.groovy
View File

@ -8,9 +8,7 @@ package inr.numass.scripts
import hep.dataforge.grind.Grind
import hep.dataforge.values.Values
import inr.numass.data.NumassPointImpl
import inr.numass.data.legacy.RawNMPoint
import inr.numass.storage.NumassDataLoader
import inr.numass.data.storage.NumassDataLoader
import inr.numass.utils.NMEventGeneratorWithPulser
import inr.numass.utils.PileUpSimulator
import inr.numass.utils.TritiumUtils

5
numass-main/src/main/groovy/inr/numass/scripts/Underflow.groovy
View File

@ -10,9 +10,8 @@ import hep.dataforge.io.ColumnedDataWriter
import hep.dataforge.storage.commons.StorageUtils
import hep.dataforge.tables.Table
import inr.numass.data.NumassDataUtils
import inr.numass.data.NumassPointImpl
import inr.numass.storage.NumassStorage
import inr.numass.storage.NumassStorageFactory
import inr.numass.data.storage.NumassStorage
import inr.numass.data.storage.NumassStorageFactory
import inr.numass.utils.UnderflowCorrection
//File rootDir = new File("D:\\Work\\Numass\\data\\2016_10\\Fill_1")

7
numass-main/src/main/groovy/inr/numass/scripts/times/MultiPointHistogram.groovy
View File

@ -7,10 +7,9 @@ import hep.dataforge.grind.helpers.PlotHelper
import hep.dataforge.plots.fx.FXPlotManager
import inr.numass.NumassPlugin
import inr.numass.data.PointAnalyzer
import inr.numass.data.legacy.RawNMPoint
import inr.numass.storage.NumassDataLoader
import inr.numass.storage.NumassStorage
import inr.numass.storage.NumassStorageFactory
import inr.numass.data.storage.NumassDataLoader
import inr.numass.data.storage.NumassStorage
import inr.numass.data.storage.NumassStorageFactory
/**
* Created by darksnake on 06-Jul-17.

5
numass-main/src/main/groovy/inr/numass/scripts/times/TestPointAnalyzer.groovy
View File

@ -8,9 +8,8 @@ import hep.dataforge.plots.fx.FXPlotManager
import hep.dataforge.tables.ValueMap
import inr.numass.NumassPlugin
import inr.numass.data.PointAnalyzer
import inr.numass.data.legacy.RawNMPoint
import inr.numass.storage.NumassStorage
import inr.numass.storage.NumassStorageFactory
import inr.numass.data.storage.NumassStorage
import inr.numass.data.storage.NumassStorageFactory
/**
* Created by darksnake on 27-Jun-17.

2
numass-main/src/main/java/inr/numass/NumassIO.java
View File

@ -20,8 +20,6 @@ import hep.dataforge.data.binary.Binary;
import hep.dataforge.io.BasicIOManager;
import hep.dataforge.meta.Meta;
import hep.dataforge.names.Name;
import inr.numass.data.legacy.LegacyDataReader;
import inr.numass.data.legacy.RawNMFile;
import org.apache.commons.io.FilenameUtils;
import org.apache.commons.io.output.TeeOutputStream;

2
numass-main/src/main/java/inr/numass/actions/DebunchAction.java
View File

@ -21,8 +21,6 @@ import hep.dataforge.description.TypedActionDef;
import hep.dataforge.description.ValueDef;
import hep.dataforge.exceptions.ContentException;
import hep.dataforge.meta.Laminate;
import inr.numass.data.legacy.RawNMFile;
import inr.numass.data.legacy.RawNMPoint;
import inr.numass.debunch.DebunchReport;
import inr.numass.debunch.FrameAnalizer;

1
numass-main/src/main/java/inr/numass/actions/FindBorderAction.java
View File

@ -25,7 +25,6 @@ import hep.dataforge.tables.ListTable;
import hep.dataforge.tables.Table;
import hep.dataforge.tables.ValueMap;
import hep.dataforge.values.Value;
import inr.numass.data.legacy.NMFile;
import org.apache.commons.math3.analysis.UnivariateFunction;
import java.io.OutputStream;

1
numass-main/src/main/java/inr/numass/actions/JoinNumassDataAction.java
View File

@ -4,7 +4,6 @@ import hep.dataforge.actions.ManyToOneAction;
import hep.dataforge.context.Context;
import hep.dataforge.description.TypedActionDef;
import hep.dataforge.meta.Laminate;
import inr.numass.data.NumassPointImpl;
import java.util.Collection;
import java.util.Map;

3
numass-main/src/main/java/inr/numass/actions/PrepareDataAction.java
View File

@ -32,10 +32,9 @@ import hep.dataforge.tables.ValueMap;
import hep.dataforge.values.Values;
import inr.numass.data.NumassPoint;
import inr.numass.data.PointBuilders;
import inr.numass.data.legacy.RawNMPoint;
import inr.numass.data.storage.NumassDataLoader;
import inr.numass.debunch.DebunchReport;
import inr.numass.debunch.FrameAnalizer;
import inr.numass.storage.NumassDataLoader;
import inr.numass.utils.ExpressionUtils;
import java.io.OutputStream;

2
numass-main/src/main/java/inr/numass/actions/ReadLegacyDataAction.java
View File

@ -23,8 +23,6 @@ import hep.dataforge.description.TypedActionDef;
import hep.dataforge.description.ValueDef;
import hep.dataforge.exceptions.ContentException;
import hep.dataforge.meta.Laminate;
import inr.numass.data.legacy.NMFile;
import inr.numass.data.legacy.RawNMFile;
import static hep.dataforge.values.ValueType.NUMBER;
import static inr.numass.NumassIO.getNumassData;

1
numass-main/src/main/java/inr/numass/debunch/DebunchData.java
View File

@ -16,7 +16,6 @@
package inr.numass.debunch;
import inr.numass.data.api.NumassEvent;
import inr.numass.data.legacy.RawNMPoint;
import java.util.ArrayList;
import java.util.Collections;

1
numass-main/src/main/java/inr/numass/debunch/DebunchReport.java
View File

@ -16,7 +16,6 @@
package inr.numass.debunch;
import inr.numass.data.api.NumassEvent;
import inr.numass.data.legacy.RawNMPoint;
import java.util.List;

1
numass-main/src/main/java/inr/numass/debunch/DebunchReportImpl.java
View File

@ -16,7 +16,6 @@
package inr.numass.debunch;
import inr.numass.data.api.NumassEvent;
import inr.numass.data.legacy.RawNMPoint;
import java.util.ArrayList;
import java.util.List;

2
numass-main/src/main/java/inr/numass/debunch/Debuncher.java
View File

@ -15,8 +15,6 @@
*/
package inr.numass.debunch;
import inr.numass.data.legacy.RawNMPoint;
/**
*
* @author Darksnake

1
numass-main/src/main/java/inr/numass/debunch/FrameAnalizer.java
View File

@ -15,7 +15,6 @@
*/
package inr.numass.debunch;
import inr.numass.data.legacy.RawNMPoint;
import org.apache.commons.math3.analysis.UnivariateFunction;
import org.apache.commons.math3.analysis.interpolation.LinearInterpolator;
import org.apache.commons.math3.util.FastMath;

1
numass-main/src/main/java/inr/numass/utils/BunchGenerator.java
View File

@ -16,7 +16,6 @@
package inr.numass.utils;
import inr.numass.data.api.NumassEvent;
import inr.numass.data.legacy.RawNMPoint;
import org.apache.commons.math3.random.MersenneTwister;
import org.apache.commons.math3.random.RandomGenerator;
import org.apache.commons.math3.random.SynchronizedRandomGenerator;

1
numass-main/src/main/java/inr/numass/utils/NMEventGenerator.java
View File

@ -17,7 +17,6 @@ package inr.numass.utils;
import hep.dataforge.meta.Meta;
import inr.numass.data.api.NumassEvent;
import inr.numass.data.legacy.RawNMPoint;
import org.apache.commons.math3.distribution.EnumeratedRealDistribution;
import org.apache.commons.math3.distribution.RealDistribution;
import org.apache.commons.math3.random.EmpiricalDistribution;

1
numass-main/src/main/java/inr/numass/utils/PileUpSimulator.java
View File

@ -7,7 +7,6 @@ package inr.numass.utils;
import inr.numass.data.PointBuilders;
import inr.numass.data.api.NumassEvent;
import inr.numass.data.legacy.RawNMPoint;
import org.apache.commons.math3.random.RandomGenerator;
import java.util.ArrayList;

2
numass-server/src/main/java/inr/numass/server/NumassRun.java
View File

@ -27,7 +27,7 @@ import hep.dataforge.storage.api.Storage;
import hep.dataforge.storage.commons.LoaderFactory;
import hep.dataforge.storage.commons.MessageFactory;
import hep.dataforge.values.Value;
import inr.numass.storage.NumassStorage;
import inr.numass.data.storage.NumassStorage;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

2
numass-server/src/main/java/inr/numass/server/NumassServer.java
View File

@ -27,7 +27,7 @@ import hep.dataforge.storage.commons.AbstractNetworkListener;
import hep.dataforge.storage.commons.LoaderFactory;
import hep.dataforge.storage.commons.StorageManager;
import hep.dataforge.storage.commons.StorageUtils;
import inr.numass.storage.NumassStorage;
import inr.numass.data.storage.NumassStorage;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import ratpack.server.RatpackServer;

2
numass-server/src/main/java/inr/numass/server/ServerRunner.java
View File

@ -5,7 +5,7 @@ import hep.dataforge.context.Global;
import hep.dataforge.io.MetaFileReader;
import hep.dataforge.meta.Meta;
import hep.dataforge.meta.SimpleConfigurable;
import inr.numass.storage.NumassStorage;
import inr.numass.data.storage.NumassStorage;
import org.slf4j.LoggerFactory;
import java.io.File;

2
numass-server/src/main/java/inr/numass/server/TestServer.java
View File

@ -11,7 +11,7 @@ import hep.dataforge.server.ServerManager;
import hep.dataforge.server.storage.StorageServerUtils;
import hep.dataforge.storage.commons.StorageManager;
import hep.dataforge.storage.filestorage.FileStorageFactory;
import inr.numass.storage.NumassStorage;
import inr.numass.data.storage.NumassStorage;
import java.io.BufferedReader;
import java.io.File;

2
numass-test/src/main/groovy/inr/numass/scripts/TestServer.groovy
View File

@ -16,8 +16,8 @@
package inr.numass.scripts
import hep.dataforge.meta.Meta
import inr.numass.data.storage.NumassStorage
import inr.numass.server.NumassServer
import inr.numass.storage.NumassStorage
String path = "D:\\Work\\Numass\\data\\2016_10\\"

2
numass-viewer/src/main/java/inr/numass/viewer/TestDirectoryViewer.java
View File

@ -16,7 +16,7 @@
package inr.numass.viewer;
import hep.dataforge.storage.commons.StorageManager;
import inr.numass.storage.NumassDataLoader;
import inr.numass.data.storage.NumassDataLoader;
import javafx.application.Application;
import javafx.scene.Scene;
import javafx.stage.Stage;

2
numass-viewer/src/main/kotlin/inr/numass/viewer/MainView.kt
View File

@ -15,7 +15,7 @@ import hep.dataforge.storage.api.PointLoader
import hep.dataforge.storage.api.Storage
import hep.dataforge.storage.filestorage.FileStorageFactory
import inr.numass.NumassProperties
import inr.numass.storage.NumassStorage
import inr.numass.data.storage.NumassStorage
import javafx.application.Platform
import javafx.beans.property.SimpleObjectProperty
import javafx.geometry.Insets