New numass data structures

numass-client/src/main/java/inr/numass/client/NumassClient.java

@@ -26,7 +26,7 @@ import hep.dataforge.storage.commons.MessageFactory;
 import hep.dataforge.storage.commons.StorageUtils;
 import hep.dataforge.values.Value;
 import hep.dataforge.values.Values;
-import inr.numass.storage.NumassStorage;
+import inr.numass.data.storage.NumassStorage;
 import org.slf4j.LoggerFactory;
 import org.zeroturnaround.zip.ZipUtil;
 

numass-core/src/main/java/inr/numass/data/NumassPointImpl.java

@@ -1,115 +0,0 @@
-/* 
- * Copyright 2015 Alexander Nozik.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package inr.numass.data;
-
-import hep.dataforge.tables.ValueMap;
-import hep.dataforge.values.Values;
-import inr.numass.data.legacy.RawNMPoint;
-
-import java.time.Instant;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.stream.IntStream;
-
-/**
- * @author Darksnake
- */
-public class NumassPointImpl implements NumassPoint {
-    //TODO andThen to annotated and move some parameters to meta
-    private final int[] spectrum;
-    private Instant startTime;
-    private long eventsCount;
-    private double pointLength;
-    private double u;
-
-    public NumassPointImpl(double u, Instant startTime, double pointLength, int[] spectrum) {
-        this.startTime = startTime;
-        this.pointLength = pointLength;
-        this.spectrum = spectrum;
-        this.u = u;
-        this.eventsCount = IntStream.of(spectrum).sum();
-    }
-
-    /**
-     * @return the absouteTime
-     */
-    @Override
-    public Instant getStartTime() {
-        if (startTime == null) {
-            return Instant.EPOCH;
-        } else {
-            return startTime;
-        }
-    }
-
-    @Override
-    public int getCount(int chanel) {
-        return spectrum[chanel];
-    }
-
-    @Override
-    public int getCountInWindow(int from, int to) {
-        int res = 0;
-        for (int i = from; i <= to; i++) {
-            res += spectrum[i];
-        }
-        if (res == Integer.MAX_VALUE) {
-            throw new RuntimeException("integer overflow in spectrum calculation");
-        }
-        return res;
-    }
-
-    @Override
-    public List<Values> getData() {
-        List<Values> data = new ArrayList<>();
-        for (int i = 0; i < RawNMPoint.MAX_CHANEL; i++) {
-            data.add(new ValueMap(dataNames, i, spectrum[i]));
-        }
-        return data;
-    }
-
-    /**
-     * Events count including overflow
-     *
-     * @return
-     */
-    @Override
-    public long getTotalCount() {
-        return eventsCount;
-    }
-
-
-    /**
-     * @return the pointLength
-     */
-    @Override
-    public double getLength() {
-        return pointLength;
-    }
-
-    /**
-     * @return the u
-     */
-    @Override
-    public double getVoltage() {
-        return u;
-    }
-
-    @Override
-    public int[] getSpectrum() {
-        return spectrum;
-    }
-}

numass-core/src/main/java/inr/numass/data/PointBuilders.java

@@ -1,6 +1,5 @@
 package inr.numass.data;
 
-import inr.numass.data.legacy.RawNMPoint;
 import org.jetbrains.annotations.NotNull;
 
 import java.io.IOException;

numass-core/src/main/java/inr/numass/data/SetDirectionUtility.java

@@ -18,6 +18,7 @@ import java.util.function.Function;
  *
  * @author Alexander Nozik
  */
+@Deprecated
 public class SetDirectionUtility {
 
     private static final String FILE_NAME = "numass_set_direction.map";

numass-core/src/main/java/inr/numass/data/analyzers/SimpleAnalyzer.java

@@ -24,6 +24,9 @@ public class SimpleAnalyzer implements NumassAnalyzer {
     public SimpleAnalyzer(@Nullable SignalProcessor processor) {
         this.processor = processor;
     }
+    public SimpleAnalyzer() {
+        this.processor = null;
+    }
 
     /**
      * Return unsorted stream of events including events from frames

numass-core/src/main/java/inr/numass/data/api/NumassEvent.java

@@ -17,16 +17,21 @@ package inr.numass.data.api;
 
 import org.jetbrains.annotations.NotNull;
 
+import java.io.Serializable;
+import java.time.Instant;
+
 /**
  * A single numass event with given amplitude ant time.
  *
  * @author Darksnake
  */
-public class NumassEvent implements Comparable<NumassEvent> {
+public class NumassEvent implements Comparable<NumassEvent>, Serializable {
+    // channel
     protected final short chanel;
-    protected final double time;
+    //time in nanoseconds
+    protected final long time;
 
-    public NumassEvent(short chanel, double time) {
+    public NumassEvent(short chanel, long time) {
         this.chanel = chanel;
         this.time = time;
     }
@@ -41,12 +46,16 @@ public class NumassEvent implements Comparable<NumassEvent> {
     /**
      * @return the time
      */
-    public double getTime() {
+    public long getTime() {
         return time;
     }
 
+    public Instant getAbsoluteTime(Instant offset) {
+        return offset.plusNanos(time);
+    }
+
     @Override
     public int compareTo(@NotNull NumassEvent o) {
-        return Double.compare(this.getTime(), o.getTime());
+        return Long.compare(this.getTime(), o.getTime());
     }
 }

numass-core/src/main/java/inr/numass/data/api/SimpleBlock.java

@@ -0,0 +1,43 @@
+package inr.numass.data.api;
+
+import java.io.Serializable;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.List;
+import java.util.stream.Stream;
+
+/**
+ * A simple in-memory implementation of block of events. No frames are allowed
+ * Created by darksnake on 08.07.2017.
+ */
+public class SimpleBlock implements NumassBlock, Serializable {
+    private final Instant startTime;
+    private final Duration length;
+    private final List<NumassEvent> events;
+
+    public SimpleBlock(Instant startTime, Duration length, List<NumassEvent> events) {
+        this.startTime = startTime;
+        this.length = length;
+        this.events = events;
+    }
+
+    @Override
+    public Instant getStartTime() {
+        return startTime;
+    }
+
+    @Override
+    public Duration getLength() {
+        return length;
+    }
+
+    @Override
+    public Stream<NumassEvent> getEvents() {
+        return events.stream();
+    }
+
+    @Override
+    public Stream<NumassFrame> getFrames() {
+        return Stream.empty();
+    }
+}

numass-core/src/main/java/inr/numass/data/api/SimpleNumassPoint.java

@@ -0,0 +1,31 @@
+package inr.numass.data.api;
+
+import hep.dataforge.meta.Meta;
+import hep.dataforge.meta.MetaBuilder;
+import hep.dataforge.utils.MetaHolder;
+
+import java.util.List;
+import java.util.stream.Stream;
+
+/**
+ * A simple static implementation of NumassPoint
+ * Created by darksnake on 08.07.2017.
+ */
+public class SimpleNumassPoint extends MetaHolder implements NumassPoint {
+    private final List<NumassBlock> blocks;
+
+    public SimpleNumassPoint(double voltage, List<NumassBlock> blocks) {
+        this.blocks = blocks;
+        super.setMeta(new MetaBuilder("point").setValue(HV_KEY, voltage));
+    }
+
+    public SimpleNumassPoint(Meta meta, List<NumassBlock> blocks) {
+        super(meta);
+        this.blocks = blocks;
+    }
+
+    @Override
+    public Stream<NumassBlock> getBlocks() {
+        return blocks.stream();
+    }
+}

numass-core/src/main/java/inr/numass/data/legacy/LegacyDataReader.java

@@ -1,517 +0,0 @@
-/* 
- * Copyright 2015 Alexander Nozik.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package inr.numass.data.legacy;
-
-import hep.dataforge.data.FileDataFactory;
-import hep.dataforge.data.binary.Binary;
-import hep.dataforge.meta.Meta;
-import inr.numass.data.api.NumassEvent;
-
-import java.io.*;
-import java.time.LocalDateTime;
-import java.time.ZoneOffset;
-import java.time.format.DateTimeFormatter;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Scanner;
-
-/**
- *
- * @author Darksnake, based on program by S.V.Zadorozhny, 1996
- */
-public class LegacyDataReader {
-
-    private final InputStream stream;
-    private String name;
-    private double HVdev = 2.468555393226049;
-    private boolean noUset = false;
-
-    public LegacyDataReader(Binary file, Meta config) throws IOException {
-        this(file.getStream(), config.getString(FileDataFactory.FILE_NAME_KEY), config);
-    }
-    
-    public LegacyDataReader(File file) throws IOException {
-        this(new FileInputStream(file), file.getName(), Meta.empty());
-    }    
-
-    public LegacyDataReader(String file, String fname, Meta config) throws FileNotFoundException {
-        this(new FileInputStream(file), fname, config);
-        if ((fname == null) || (fname.isEmpty())) {
-            name = file;
-        }
-    }
-
-    public LegacyDataReader(InputStream is, String fname, Meta config) {
-        this.stream = new BufferedInputStream(is);
-        this.name = fname;
-        HVdev = config.getDouble("HVdev", 2.468555393226049);
-        noUset = config.getBoolean("noUset", false);
-    }
-
-    public RawNMFile read() throws IOException {
-        RawNMFile res = readFile(name);
-        stream.close();
-        return res;
-    }
-
-    private int[] readBlock(int length) throws IOException {
-        int[] res = new int[length];
-        for (int i = 0; i < res.length; i++) {
-            res[i] = readByte();
-        }
-        return res;
-    }
-
-    private int readByte() throws IOException {
-        return stream.read();
-    }
-
-    private short readChanel(int hb) throws IOException {
-        assert hb < 127;
-        return (short) (readByte() + 256 * hb);
-    }
-
-    private LocalDateTime readDate(String head) throws IOException {
-        // Должны считать 14 символов
-        Scanner sc = new Scanner(head);
-        sc.nextLine();
-        String dateStr = sc.nextLine().trim();
-        //DD.MM.YY HH:MM
-        //12:35:16 19-11-2013
-        DateTimeFormatter format = DateTimeFormatter.ofPattern("HH:mm:ss dd-MM-yyyy");
-
-        LocalDateTime date = LocalDateTime.parse(dateStr, format);
-
-        return date;
-    }
-
-    private NumassEvent readEvent(int b, double timeDiv) throws IOException {
-        short chanel;
-        long time;
-
-        int hb = (b & 0x0f);
-        int lab = (b & 0xf0);
-
-        switch (lab) {
-            case 0x10:
-                chanel = readChanel(hb);
-                time = readTime();
-                break;
-            case 0x20:
-                chanel = 0;
-                time = readTime();
-                break;
-            case 0x40:
-                time = 0;
-                chanel = readChanel(hb);
-                break;
-            case 0x80:
-                time = 0;
-                chanel = 0;
-                break;
-            default:
-                throw new IOException("Event head expected");
-        }
-
-        return new NumassEvent(chanel, time / timeDiv);
-    }
-
-    private RawNMFile readFile(String name) throws IOException {
-        String head = readHead();//2048
-        RawNMFile file = new RawNMFile(name, head.replaceAll("\u0000", ""));
-        LocalDateTime filedate = readDate(head);
-
-        int lab = readByte();
-        do {
-            RawNMPoint point = readPoint(lab, filedate);
-            //Устанавливаем дату точки
-            file.putPoint(point);
-            lab = readByte();
-        } while (lab != 0xff);
-
-        //Check if input file head is numass data file
-        return file;
-    }
-
-    private String readHead() throws IOException {
-        byte[] bytes = new byte[2048];
-        stream.read(bytes);
-        return new String(bytes);
-    }
-
-    private RawNMPoint readPoint(int head, LocalDateTime filedate) throws IOException {
-
-        int[] rx;
-
-        int voltage;
-        short time_out;
-        double timeDiv = 0;
-        boolean phoneFlag;
-
-        RawNMPoint point = new RawNMPoint();
-
-        int lab = head;
-
-        //point head
-        if (lab != 0x0f) {
-            //Ожидается, что это голова точки
-            throw new IOException("Point head expected");
-        }
-
-        rx = readBlock(32);
-
-        voltage = rx[2] + 256 * rx[3];
-        voltage = 65536 * voltage + rx[0] + 256 * rx[1];
-
-        time_out = (short) (rx[6] + 256 * rx[7]);
-        phoneFlag = (rx[19] != 0);
-
-        switch (time_out) {
-            case 5:
-            case 10:
-                timeDiv = 2e7;
-                break;
-            case 15:
-            case 20:
-                timeDiv = 1e7;
-                break;
-            case 50:
-                timeDiv = 5e6;
-                break;
-            case 100:
-                timeDiv = 2.5e6;
-                break;
-            case 200:
-                timeDiv = 1.25e6;
-                break;
-            default:
-                throw new IOException("Unknown time divider in input data");
-        }
-
-        if (phoneFlag) {
-            timeDiv /= 20.0;
-            time_out *= 20;
-        }
-
-        lab = readByte();
-
-        List<NumassEvent> events = new ArrayList<>();
-        while (lab == 0xBF) {
-            skip(4);//badHV
-            lab = readByte();
-        }
-        do {
-            events.add(readEvent(lab, timeDiv));
-            lab = readByte();
-        } while (lab != 0xAF);
-
-        //point end
-        skip(37);
-
-        int hours = readByte();
-        int minutes = readByte();
-
-        LocalDateTime absoluteTime = filedate.withHour(hours).withMinute(minutes);
-
-        //проверяем, не проскочили ли мы полночь
-        if (absoluteTime.isBefore(filedate)) {
-            absoluteTime = absoluteTime.plusDays(1);
-        }
-
-
-        rx = readBlock(4);
-        int Uread = rx[2] + 256 * rx[3];
-        Uread = 65536 * Uread + rx[0] + 256 * rx[1];
-
-        skip(21);
-
-        double uset;
-        if (noUset) {
-            uset = Uread / 10d / HVdev;
-        } else {
-            uset = voltage / 10d;
-        }
-
-        return new RawNMPoint(uset, Uread / 10d / HVdev, events, time_out, absoluteTime.toInstant(ZoneOffset.UTC));
-    }
-
-    private long readTime() throws IOException {
-        int[] rx = readBlock(4);
-        long time = rx[0] + 256 * rx[1] + 65536 * rx[2] + 256 * 65536 * rx[3];
-        return time;
-    }
-
-    private void skip(int length) throws IOException {
-        long n = stream.skip(length);
-        if (n != length) {
-            stream.skip(length - n);
-        }
-    }
-}
-
-/*
- PROGRAM DAT2PAW;
-
- {$N+}
-
- {$D Copyright (C) 1996 by S.V.Zadorozhny, INR RAS}
-
- Uses Dos, Crt;
-
- CONST
- Rec_Len = 32768;  { Buffer size }
-
- TYPE
- Dim   = ARRAY [1..Rec_Len] OF Byte;      { Read / Write Buffer  }
-
- VAR
- DATA : WORD;
- TIME : LONGINT;
- TD : Double;
- i, j, k : WORD;
- Voltage, li, CN : LONGINT;
- St, St1 : String;
- f : File;
- txt : Text;
- Buf : ^Dim;
- Buf_point : Word;
- err : INTEGER;
- Monitor, Pressure : WORD;
-
- paths : PathStr;
- dirs : DirStr;
- nams : NameStr;
- exts : ExtStr;
- name, mask : String;
- sr : SearchRec;
-
- rx : Array [1..64] OF Byte;
- lab, bt : Byte;
- time_out : Word;
- hb : Word;
-
- {Gate_L, Gate_H : Word;}
-
- TimDiv : Double;
- Phone_Flag, OK : Boolean;
-
- live : word;
-
- (*----------------------------- PROCEDURES ----------------------------------*)
- PROCEDURE NextChar;
- BEGIN
- CASE (live) OF
- 0 : BEGIN
- Write('|');
- Inc(live);
- END;
- 1 : BEGIN
- Write(#8'/');
- Inc(live);
- END;
- 2 : BEGIN
- Write(#8'--');
- Inc(live);
- END;
- 3 : BEGIN
- Write(#8#8'\ ');
- Inc(live);
- END;
- 4 : BEGIN
- Write(#8#8'|');
- live := 1;
- END;
- END;
- END;
-
-
- PROCEDURE ReadB(VAR bt : BYTE);
- VAR fact : INTEGER;
- BEGIN
- IF Buf_point = 0 THEN
- BEGIN
- BlockRead(f,Buf^,Rec_Len,fact);
- NextChar;
- END;
- Inc(Buf_point);
- bt := Buf^[Buf_point];
- IF Buf_point = Rec_Len THEN Buf_point:=0;
- END  { ReadB };
-
- PROCEDURE SaveTXT;
- BEGIN
- Writeln(txt, CN, #9, TD:10:5, #9, DATA, #9, time_out, #9, (voltage/10):7:1);
- END;
-
- PROCEDURE ReadFile;
- BEGIN
- Assign(f,name);
- {$I-}
- Reset(f,1);
- {$I+}
- IF (IOResult <> 0) THEN
- BEGIN
- Writeln('Can''t open data file ', name);
- Halt(3);
- END;
-
- BlockRead(f,Buf^,2048,i);     {Skip First Block}
- Buf_point := 0;
-
- IF ((Buf^[1] <> Ord('N'))OR(Buf^[2] <> Ord('M'))) THEN
- BEGIN
- Close(f);
- Writeln('File ', name, ' is not a data file !');
- Exit;
- END;
-
- Write('File : ',name, '   ');
-
- paths := name;
- Fsplit(paths, dirs, nams, exts);
- Assign(txt, nams + '.paw');
- Rewrite(txt);
-
- live := 0;
- CN := 0;
-
- REPEAT
- ReadB(lab);
- hb  := lab AND $0F;     { High data byte }
- lab := lab AND $F0;
- CASE (lab) OF
- $00 :         { New Point }
- BEGIN
- FOR i:=1 TO 32 DO ReadB(rx[i]);
- Voltage := rx[3] + 256 * Word(rx[4]);
- Voltage := 65536 * Voltage + rx[1] + 256 * Longint(rx[2]);
- time_out := rx[7] + 256 * Word(rx[8]);
- {pressure := rx[11] + 256 * Word(rx[12]);}
- {Chan := rx[19];}
- IF (rx[20] <> 0) THEN Phone_Flag := TRUE ELSE Phone_Flag := FALSE;
- CASE (time_out) OF
- 5, 10 : TimDiv := 20000000.0;
- 15, 20 : TimDiv := 10000000.0;
- 50 : TimDiv :=  5000000.0;
- 100 : TimDiv :=  2500000.0;
- 200 : TimDiv :=  1250000.0;
- END;
- IF (Phone_Flag) THEN
- BEGIN
- TimDiv := TimDiv / 20.0;
- time_out := time_out * 20;
- END;
- END;
-
- $10 :         { Event, all present }
- BEGIN
- FOR i:=1 TO 5 DO ReadB(rx[i]);
- TIME := rx[2] + 256 * Longint(rx[3])
- + 65536 * Longint(rx[4])+ 256 * 65536 * Longint(rx[5]);
- TD := TIME / TimDiv;
- DATA := rx[1] + 256 * hb;
- Inc(CN);
- SaveTXT;
- END;
-
- $20 :         { Event, time only }
- BEGIN
- FOR i:=1 TO 4 DO ReadB(rx[i]);
- TIME := rx[1] + 256 * Longint(rx[2])
- + 65536 * Longint(rx[3])+ 256 * 65536 * Longint(rx[4]);
- TD := TIME / TimDiv;
- DATA := 0;
- Inc(CN);
- SaveTXT;
- END;
-
- $40 :         { Event, code only }
- BEGIN
- ReadB(rx[1]);
- DATA := rx[1] + 256 * hb;
- TD := 0.0;
- Inc(CN);
- SaveTXT;
- END;
-
- $80 :         { Event, nothing }
- BEGIN
- DATA := 0;
- TD := 0.0;
- Inc(CN);
- SaveTXT;
- END;
-
- $B0 :         { Bad HV }
- BEGIN
- FOR i:=1 TO 4 DO ReadB(rx[i]);
- END;
-
- $A0 :         { End of the point }
- BEGIN
- FOR i := 1 TO 64 DO ReadB(rx[i]);
- END;
- END;
-
- UNTIL (lab=$F0);
- Close(f);
- Close(txt);
- Writeln(#8#8'  ');
- END;
-
-
- PROCEDURE ReadDataFiles;
- BEGIN
- mask := ParamStr(1);
- FindFirst(mask, anyfile, sr);
- WHILE (DosError = 0) DO
- BEGIN
- name := sr.Name;
- ReadFile;
- FindNext(sr);
- END
- END;
-
- (*--------------------------- Main ------------------------------------------*)
-
- BEGIN
- ClrScr;
-
- Writeln('TROITSK V-MASS EXPERIMENT DATA FILE -> TEXT FILE CONVERTER');
- Writeln('WRITTEN BY S.V.ZADOROZHNY');
- Writeln;
-
-
- IF (ParamCount <> 1) THEN
- BEGIN
- Writeln('Usage : dat2paw <mask.dat>');
- Writeln;
- Writeln('--- Press any key to continue ---');
- ReadKey;
- Halt(1);
- END;
-
- New(Buf);
-
- ReadDataFiles;
-
- Writeln;
- Writeln('O.K.');
-
- Dispose(Buf);
- END.                       {-- Main --}
-
- */

numass-core/src/main/java/inr/numass/data/legacy/NMFile.java

@@ -1,68 +0,0 @@
-/* 
- * Copyright 2015 Alexander Nozik.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package inr.numass.data.legacy;
-
-import hep.dataforge.description.ValueDef;
-import hep.dataforge.meta.Meta;
-import hep.dataforge.names.NamedMetaHolder;
-import inr.numass.data.PointBuilders;
-import inr.numass.data.api.NumassPoint;
-import inr.numass.data.api.NumassSet;
-
-import java.io.File;
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.stream.Stream;
-
-/**
- * Объект, содержащий только спектры, но не сами события
- *
- * @author Darksnake
- */
-@ValueDef(name = "numass.path", info = "Path to this data file in numass repository.")
-@ValueDef(name = "numass.name", info = "The name of this data file.")
-public class NMFile extends NamedMetaHolder implements NumassSet {
-
-    private final List<NumassPoint> points;
-
-    public NMFile(RawNMFile file) {
-        super(file.getName(), file.meta());
-        points = new ArrayList<>();
-        for (RawNMPoint point : file.getData()) {
-            points.add(PointBuilders.readRawPoint(point));
-        }
-    }
-
-    public static NMFile readStream(InputStream is, String fname, Meta config) throws IOException {
-        return new NMFile(new LegacyDataReader(is, fname, config).read());
-    }
-
-    public static NMFile readFile(File file) throws IOException {
-        return new NMFile(new LegacyDataReader(file).read());
-    }
-
-    @Override
-    public Stream<NumassPoint> getPoints() {
-        return points.stream();
-    }
-
-    @Override
-    public String getDescription() {
-        return "";
-    }
-}

numass-core/src/main/java/inr/numass/data/legacy/NumassDatFile.java

@@ -0,0 +1,265 @@
+package inr.numass.data.legacy;
+
+import hep.dataforge.meta.Meta;
+import hep.dataforge.meta.MetaBuilder;
+import inr.numass.data.api.*;
+import org.apache.commons.io.FilenameUtils;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.nio.channels.SeekableByteChannel;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.time.Duration;
+import java.time.LocalDateTime;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Scanner;
+import java.util.stream.Stream;
+
+import static inr.numass.data.api.NumassPoint.HV_KEY;
+import static java.nio.file.StandardOpenOption.READ;
+
+/**
+ * Created by darksnake on 08.07.2017.
+ */
+public class NumassDatFile implements NumassSet {
+    private final String name;
+    private final Path path;
+    private final Meta meta;
+
+    public NumassDatFile(Path path, Meta meta) throws IOException {
+        this(FilenameUtils.getBaseName(path.getFileName().toString()),path,meta);
+    }
+
+    public NumassDatFile(String name, Path path, Meta meta) throws IOException {
+        this.name = name;
+        this.path = path;
+        String head = readHead(path);//2048
+        this.meta = new MetaBuilder(meta)
+                .setValue("info", head)
+                .setValue(NumassPoint.START_TIME_KEY, readDate(head))
+                .build();
+    }
+
+    @Override
+    public Meta meta() {
+        return meta;
+    }
+
+    @Override
+    public String getName() {
+        return name;
+    }
+
+    private double getHVdev() {
+        return meta().getDouble("dat.hvDev", 2.468555393226049);
+    }
+
+    private boolean hasUset() {
+        return meta().getBoolean("dat.uSet", true);
+    }
+
+    private static String readHead(Path path) throws IOException {
+        try (SeekableByteChannel channel = Files.newByteChannel(path, READ)) {
+            channel.position(0);
+            ByteBuffer buffer = ByteBuffer.allocate(2048);
+            channel.read(buffer);
+            return new String(buffer.array()).replaceAll("\u0000", "");
+        }
+    }
+
+    /**
+     * Read the block at current position
+     *
+     * @param channel
+     * @param length
+     * @return
+     * @throws IOException
+     */
+    private ByteBuffer readBlock(SeekableByteChannel channel, int length) throws IOException {
+        ByteBuffer res = ByteBuffer.allocate(length);
+        channel.read(res);
+        res.order(ByteOrder.LITTLE_ENDIAN);
+        res.flip();
+        return res;
+    }
+
+    /**
+     * Read the point at current position
+     *
+     * @param channel
+     * @return
+     * @throws IOException
+     */
+    private synchronized NumassPoint readPoint(SeekableByteChannel channel) throws
+            IOException {
+
+        ByteBuffer rx = readBlock(channel, 32);
+
+        int voltage = rx.getInt();
+
+        short length = rx.getShort();//(short) (rx[6] + 256 * rx[7]);
+        boolean phoneFlag = rx.get(19) != 0;//(rx[19] != 0);
+
+
+        double timeDiv;
+        switch (length) {
+            case 5:
+            case 10:
+                timeDiv = 2e7;
+                break;
+            case 15:
+            case 20:
+                timeDiv = 1e7;
+                break;
+            case 50:
+                timeDiv = 5e6;
+                break;
+            case 100:
+                timeDiv = 2.5e6;
+                break;
+            case 200:
+                timeDiv = 1.25e6;
+                break;
+            default:
+                throw new IOException("Unknown time divider in input data");
+        }
+
+        if (phoneFlag) {
+            timeDiv /= 20.0;
+            length *= 20;
+        }
+
+        List<NumassEvent> events = new ArrayList<>();
+        int lab = readBlock(channel, 1).get();
+
+        while (lab == 0xBF) {
+            ByteBuffer buffer = readBlock(channel, 5);
+            lab = buffer.get(4);
+        }
+
+        do {
+            events.add(readEvent(channel, lab, timeDiv));
+            lab = readBlock(channel, 1).get();
+        } while (lab != 0xAF);
+
+        //point end
+        ByteBuffer ending = readBlock(channel, 64);
+
+        int hours = ending.get(37);
+        int minutes = ending.get(38);
+
+        LocalDateTime start = LocalDateTime.from(getStartTime());
+        LocalDateTime absoluteTime = start.withHour(hours).withMinute(minutes);
+
+        //проверяем, не проскочили ли мы полночь
+        if (absoluteTime.isBefore(start)) {
+            absoluteTime = absoluteTime.plusDays(1);
+        }
+
+
+        int uRead = ending.getInt(39);
+
+        double uSet;
+        if (!this.hasUset()) {
+            uSet = uRead / 10d / getHVdev();
+        } else {
+            uSet = voltage / 10d;
+        }
+
+        NumassBlock block = new SimpleBlock(absoluteTime.toInstant(ZoneOffset.UTC), Duration.ofSeconds(length), events);
+
+        Meta pointMeta = new MetaBuilder("point")
+                .setValue(HV_KEY, uSet)
+                .setValue("uRead", uRead / 10 / getHVdev())
+                .setValue("source", "legacy");
+
+
+        return new SimpleNumassPoint(pointMeta, Collections.singletonList(block));
+    }
+
+    @Override
+    public Stream<NumassPoint> getPoints() {
+        try (SeekableByteChannel channel = Files.newByteChannel(path, READ)) {
+
+            //int lab = readBlock(channel,1).get();
+            int lab;
+            List<NumassPoint> points = new ArrayList<>();
+            do {
+                //TODO check point start
+                points.add(readPoint(channel));
+                lab = readBlock(channel, 1).get();
+            } while (lab != 0xff);
+            return points.stream();
+        } catch (IOException ex) {
+            throw new RuntimeException(ex);
+        }
+
+    }
+
+    private LocalDateTime readDate(String head) throws IOException {
+        // Должны считать 14 символов
+        Scanner sc = new Scanner(head);
+        sc.nextLine();
+        String dateStr = sc.nextLine().trim();
+        //DD.MM.YY HH:MM
+        //12:35:16 19-11-2013
+        DateTimeFormatter format = DateTimeFormatter.ofPattern("HH:mm:ss dd-MM-yyyy");
+
+        return LocalDateTime.parse(dateStr, format);
+    }
+
+    private NumassEvent readEvent(SeekableByteChannel channel, int b, double timeDiv) throws IOException {
+        short chanel;
+        long time;
+
+        int hb = (b & 0x0f);
+        int lab = (b & 0xf0);
+
+        switch (lab) {
+            case 0x10:
+                chanel = readChanel(channel, hb);
+                time = readTime(channel);
+                break;
+            case 0x20:
+                chanel = 0;
+                time = readTime(channel);
+                break;
+            case 0x40:
+                time = 0;
+                chanel = readChanel(channel, hb);
+                break;
+            case 0x80:
+                time = 0;
+                chanel = 0;
+                break;
+            default:
+                throw new IOException("Event head expected");
+        }
+
+        return new NumassEvent(chanel, (long) (time / timeDiv));
+    }
+
+    private short readChanel(SeekableByteChannel channel, int hb) throws IOException {
+        assert hb < 127;
+        ByteBuffer buffer = readBlock(channel, 1);
+        return (short) (buffer.get() + 256 * hb);
+    }
+
+    private long readTime(SeekableByteChannel channel) throws IOException {
+        ByteBuffer rx = readBlock(channel, 4);
+        return rx.getLong();//rx[0] + 256 * rx[1] + 65536 * rx[2] + 256 * 65536 * rx[3];
+    }
+
+//    private void skip(int length) throws IOException {
+//        long n = stream.skip(length);
+//        if (n != length) {
+//            stream.skip(length - n);
+//        }
+//    }
+}

numass-core/src/main/java/inr/numass/data/legacy/RawNMFile.java

@@ -1,144 +0,0 @@
-/* 
- * Copyright 2015 Alexander Nozik.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package inr.numass.data.legacy;
-
-import hep.dataforge.description.ValueDef;
-import hep.dataforge.meta.Meta;
-import hep.dataforge.meta.MetaBuilder;
-import hep.dataforge.names.NamedMetaHolder;
-import inr.numass.data.api.NumassEvent;
-
-import java.io.BufferedOutputStream;
-import java.io.OutputStream;
-import java.io.PrintWriter;
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * Contains the whole data but requires a lot of memory
- *
- * @author Darksnake
- */
-@ValueDef(name = "info", info = "file text header")
-public class RawNMFile extends NamedMetaHolder {
-
-//    public static String TYPE = ":data:numassdatafile";
-
-    private final List<RawNMPoint> points = new ArrayList<>();
-
-
-    public RawNMFile(String fileName) {
-        super(fileName);
-    }
-
-    public RawNMFile(String name, Meta meta) {
-        super(name, meta);
-    }
-
-    public RawNMFile(String name, String header) {
-        super(name, new MetaBuilder("meta").setValue("info", header));
-    }
-
-    public String getHead() {
-        return meta().getString("info", "");
-    }
-
-    @Deprecated
-    public void generatePAW(OutputStream stream) {
-        PrintWriter writer = new PrintWriter(new BufferedOutputStream(stream));
-        long counter = 0;
-        for (RawNMPoint point : this.getData()) {
-            double U = point.getUread();
-            for (NumassEvent event : point.getEvents()) {
-                counter++;
-                writer.printf("%d\t%f\t%d\t%.1f\t%.2f%n", counter, event.getTime(), event.getChanel(), point.getLength(), U);
-            }
-
-        }
-        writer.flush();
-    }
-
-    /**
-     * merge of all point with given Uset
-     *
-     * @param U
-     * @return
-     */
-    public RawNMPoint getByUset(double U) {
-        RawNMPoint res = null;
-
-        for (RawNMPoint point : points) {
-            if (point.getUset() == U) {
-                if (res == null) {
-                    res = point;
-                } else {
-                    res = res.merge(point);
-                }
-            }
-        }
-        return res;
-    }
-
-
-    /**
-     * merge of all point with given Uread
-     *
-     * @param U
-     * @return
-     */
-    public RawNMPoint getByUread(double U) {
-        RawNMPoint res = null;
-
-        for (RawNMPoint point : points) {
-            if (point.getUread() == U) {
-                if (res == null) {
-                    res = point;
-                } else {
-                    res = res.merge(point);
-                }
-            }
-        }
-        return res;
-    }
-
-    /**
-     * @return the data
-     */
-    public List<RawNMPoint> getData() {
-        return points;
-    }
-
-//    public void putEvent(double U, short chanel, double time) {
-//        for (RawNMPoint point : this.getData()) {
-//            if (U == point.getUread()) {
-//                point.putEvent(new NumassEvent(chanel, time));
-//                return;
-//            }
-//        }
-//        RawNMPoint newpoint = new RawNMPoint();
-//        newpoint.putEvent(new NumassEvent(chanel, time));
-//        this.putPoint(newpoint);
-//    }
-
-    public void putPoint(RawNMPoint point) {
-        points.add(point);
-    }
-
-    public int size() {
-        return this.points.size();
-    }
-
-}

numass-core/src/main/java/inr/numass/data/legacy/RawNMPoint.java

@@ -1,186 +0,0 @@
-/* 
- * Copyright 2015 Alexander Nozik.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package inr.numass.data.legacy;
-
-import inr.numass.data.api.NumassEvent;
-
-import java.time.Instant;
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * Хранит информацию о спектре точки, но не об отдельных событиях.
- *
- * @author Darksnake
- */
-public class RawNMPoint {
-
-    public static final int MAX_EVENTS_PER_POINT = 260000;
-    public static int MAX_CHANEL = 4095;
-
-    private Instant startTime;
-    private final List<NumassEvent> events;
-    private double length;
-    private double uread;
-
-    private double uset;
-
-    public RawNMPoint(double U, List<NumassEvent> events, double t) {
-        this.uset = U;
-        this.uread = U;
-        this.events = events;
-        this.length = t;
-    }
-
-    public RawNMPoint(double Uset, double Uread, List<NumassEvent> events, double t) {
-        this.uset = Uset;
-        this.uread = Uread;
-        this.events = events;
-        this.length = t;
-    }
-
-    public RawNMPoint(double uset, double uread, List<NumassEvent> events, double t, Instant absouteTime) {
-        this.uset = uset;
-        this.uread = uread;
-        this.length = t;
-        this.startTime = absouteTime;
-        this.events = events;
-    }
-
-    RawNMPoint() {
-        events = new ArrayList<>();
-        uset = 0;
-        uread = 0;
-        length = Double.NaN;
-    }
-
-//    @Override
-//    public RawNMPoint clone() {
-//        ArrayList<NumassEvent> newevents = new ArrayList<>();
-//        newevents.addAll(this.getEvents());
-//        return new RawNMPoint(getUset(), getUread(), newevents, getLength());
-//    }
-
-    public Instant getStartTime() {
-        return startTime;
-    }
-
-    public double getCr() {
-        return getEventsCount() / getLength();
-    }
-
-    public double getCrError() {
-        return Math.sqrt(getEventsCount()) / getLength();
-    }
-
-    /**
-     * @return the events
-     */
-    public List<NumassEvent> getEvents() {
-        return events;
-    }
-
-    public long getEventsCount() {
-        return events.size();
-    }
-
-    /**
-     * Measurement time
-     *
-     * @return the tset
-     */
-    public double getLength() {
-        if (Double.isNaN(length)) {
-            throw new Error();
-        }
-
-        return length;
-    }
-
-    /**
-     * @return the Uread
-     */
-    public double getUread() {
-        if (uread <= 0) {
-            return getUset();
-        } else {
-            return uread;
-        }
-    }
-
-    /**
-     * @return the Uset
-     */
-    public double getUset() {
-        if (uset < 0) {
-            throw new IllegalStateException();
-        }
-        return uset;
-    }
-
-    public RawNMPoint merge(RawNMPoint point) {
-        List<NumassEvent> events = new ArrayList<>(this.events);
-        for (NumassEvent newEvent : point.getEvents()) {
-            events.add(new NumassEvent(newEvent.getChanel(), newEvent.getTime() + this.getLength()));
-        }
-        double length = this.length + point.length;
-        double uread = (this.uread + point.uread) / 2;
-        return new RawNMPoint(this.uset, uread, events, length, this.startTime);
-    }
-
-//    void putEvent(NumassEvent event) {
-//        events.add(event);
-//    }
-
-    public RawNMPoint selectChanels(int from, int to) {
-        assert to > from;
-
-        List<NumassEvent> res = new ArrayList<>();
-        for (NumassEvent event : this.getEvents()) {
-            if ((event.getChanel() >= from) && (event.getChanel() <= to)) {
-                res.add(event);
-            }
-        }
-        return new RawNMPoint(getUset(), getUread(), res, getLength());
-    }
-
-//    void setStartTime(Instant absouteTime) {
-//        this.startTime = absouteTime;
-//    }
-//
-//    /**
-//     * @param tset the tset to set
-//     */
-//    void setLength(double tset) {
-//        this.length = tset;
-//    }
-
-//    /**
-//     * @param Uread the Uread to set
-//     */
-//    void setUread(double Uread) {
-//        assert Uread >= 0;
-//        this.uread = Uread;
-//    }
-//
-//    /**
-//     * @param Uset the Uset to set
-//     */
-//    void setUset(double Uset) {
-//        this.uset = Uset;
-//    }
-
-}

numass-core/src/main/java/inr/numass/data/storage/EnvelopeNumassPoint.java

@@ -0,0 +1,114 @@
+package inr.numass.data.storage;
+
+import hep.dataforge.io.envelopes.Envelope;
+import hep.dataforge.meta.Meta;
+import inr.numass.data.api.NumassBlock;
+import inr.numass.data.api.NumassEvent;
+import inr.numass.data.api.NumassFrame;
+import inr.numass.data.api.NumassPoint;
+import org.jetbrains.annotations.NotNull;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.ByteBuffer;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.Iterator;
+import java.util.stream.Stream;
+import java.util.stream.StreamSupport;
+
+/**
+ * Created by darksnake on 08.07.2017.
+ */
+public class EnvelopeNumassPoint implements NumassPoint {
+    private final Envelope envelope;
+
+    public EnvelopeNumassPoint(Envelope envelope) {
+        this.envelope = envelope;
+    }
+
+    @Override
+    public Stream<NumassBlock> getBlocks() {
+        return null;
+    }
+
+    @Override
+    public Meta meta() {
+        return envelope.meta();
+    }
+
+    private class EnvelopeBlock implements NumassBlock, Iterable<NumassEvent> {
+        private final Instant startTime;
+        private final Duration length;
+        private final long blockOffset;
+
+        public EnvelopeBlock(Instant startTime, Duration length, long blockOffset) {
+            this.startTime = startTime;
+            this.length = length;
+            this.blockOffset = blockOffset;
+        }
+
+        @Override
+        public Instant getStartTime() {
+            return startTime;
+        }
+
+        @Override
+        public Duration getLength() {
+            return length;
+        }
+
+        @Override
+        public Stream<NumassEvent> getEvents() {
+            return StreamSupport.stream(this.spliterator(), false);
+        }
+
+        @NotNull
+        @Override
+        public Iterator<NumassEvent> iterator() {
+            double timeCoef = envelope.meta().getDouble("time_coeff", 50);
+            try {
+                InputStream stream = envelope.getData().getStream();
+                stream.skip(blockOffset);
+                return new Iterator<NumassEvent>() {
+
+                    @Override
+                    public boolean hasNext() {
+                        try {
+                            return stream.available() > 0;
+                        } catch (IOException e) {
+                            LoggerFactory.getLogger(EnvelopeNumassPoint.this.getClass()).error("Unexpected IOException " +
+                                    "when reading block", e);
+                            return false;
+                        }
+                    }
+
+                    @Override
+                    public NumassEvent next() {
+                        try {
+                            byte[] bytes = new byte[7];
+                            stream.read(bytes);
+                            ByteBuffer buffer = ByteBuffer.wrap(bytes);
+                            short channel = (short) Short.toUnsignedInt(buffer.getShort());
+                            long time = Integer.toUnsignedLong(buffer.getInt());
+                            byte status = buffer.get(); // status is ignored
+                            return new NumassEvent(channel, (long) (time * timeCoef));
+                        } catch (IOException ex) {
+                            LoggerFactory.getLogger(EnvelopeNumassPoint.this.getClass()).error("Unexpected IOException " +
+                                    "when reading block", ex);
+                            throw new RuntimeException(ex);
+                        }
+                    }
+                };
+            } catch (IOException ex){
+                throw new RuntimeException(ex);
+            }
+        }
+
+            @Override
+            public Stream<NumassFrame> getFrames () {
+                return Stream.empty();
+            }
+        }
+    }

numass-core/src/main/java/inr/numass/data/storage/NumassDataFactory.java

@@ -1,4 +1,4 @@
-package inr.numass.storage;
+package inr.numass.data.storage;
 
 import hep.dataforge.context.Context;
 import hep.dataforge.data.DataFactory;
@@ -6,14 +6,15 @@ import hep.dataforge.data.DataFilter;
 import hep.dataforge.data.DataTree;
 import hep.dataforge.meta.Meta;
 import hep.dataforge.storage.commons.StorageUtils;
+import inr.numass.data.api.NumassSet;
 
 /**
  * Created by darksnake on 03-Feb-17.
  */
-public class NumassDataFactory extends DataFactory<NumassData> {
+public class NumassDataFactory extends DataFactory<NumassSet> {
 
     public NumassDataFactory() {
-        super(NumassData.class);
+        super(NumassSet.class);
     }
 
     @Override
@@ -22,11 +23,11 @@ public class NumassDataFactory extends DataFactory<NumassData> {
     }
 
     @Override
-    protected void buildChildren(Context context, DataTree.Builder<NumassData> builder, DataFilter filter, Meta meta) {
+    protected void buildChildren(Context context, DataTree.Builder<NumassSet> builder, DataFilter filter, Meta meta) {
         NumassStorage storage = new NumassStorage(context,meta);
         StorageUtils.loaderStream(storage).forEach(pair -> {
-            if (pair.getValue() instanceof NumassData) {
+            if (pair.getValue() instanceof NumassSet) {
-                builder.putStatic(pair.getKey(), (NumassData) pair.getValue());
+                builder.putStatic(pair.getKey(), (NumassSet) pair.getValue());
             }
         });
     }

numass-core/src/main/java/inr/numass/data/storage/NumassDataLoader.java

@@ -13,7 +13,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package inr.numass.storage;
+package inr.numass.data.storage;
 
 import hep.dataforge.data.Data;
 import hep.dataforge.exceptions.StorageException;
@@ -22,30 +22,22 @@ import hep.dataforge.io.envelopes.Envelope;
 import hep.dataforge.meta.Meta;
 import hep.dataforge.meta.MetaBuilder;
 import hep.dataforge.providers.Provider;
-import hep.dataforge.providers.Provides;
-import hep.dataforge.providers.ProvidesNames;
 import hep.dataforge.storage.api.ObjectLoader;
 import hep.dataforge.storage.api.Storage;
 import hep.dataforge.storage.filestorage.FileEnvelope;
 import hep.dataforge.storage.filestorage.FileStorage;
 import hep.dataforge.storage.loaders.AbstractLoader;
 import hep.dataforge.tables.Table;
-import inr.numass.data.PointBuilders;
+import inr.numass.data.api.NumassPoint;
-import inr.numass.data.api.NumassEvent;
+import inr.numass.data.api.NumassSet;
-import inr.numass.data.legacy.RawNMPoint;
 import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.nio.ByteOrder;
-import java.nio.channels.ReadableByteChannel;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.time.Instant;
 import java.util.*;
-import java.util.function.Function;
 import java.util.function.Supplier;
-import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
 
@@ -54,7 +46,7 @@ import java.util.stream.Stream;
  *
  * @author darksnake
  */
-public class NumassDataLoader extends AbstractLoader implements ObjectLoader<Envelope>, NumassData, Provider {
+public class NumassDataLoader extends AbstractLoader implements ObjectLoader<Envelope>, NumassSet, Provider {
 
 
     public static NumassDataLoader fromFile(Storage storage, Path zipFile) throws IOException {
@@ -147,79 +139,6 @@ public class NumassDataLoader extends AbstractLoader implements ObjectLoader<Env
         readOnly = true;
     }
 
-
-    /**
-     * Read numass point from envelope and apply transformation (e.g. debuncing)
-     *
-     * @param envelope
-     * @param transformation
-     * @return
-     */
-    private NumassPoint readPoint(Envelope envelope, Function<RawNMPoint, NumassPoint> transformation) {
-        return transformation.apply(readRawPoint(envelope));
-    }
-
-    /**
-     * Read raw point. Requires a low of memory.
-     *
-     * @param envelope
-     * @return
-     */
-    private synchronized RawNMPoint readRawPoint(Envelope envelope) {
-        List<NumassEvent> events = new ArrayList<>();
-
-        double timeCoef = envelope.meta().getDouble("time_coeff", 50);
-        try (ReadableByteChannel inChannel = envelope.getData().getChannel()) {
-            ByteBuffer buffer = ByteBuffer.allocate(7 * 10000); // one event is 7b
-            buffer.order(ByteOrder.LITTLE_ENDIAN);
-            while (inChannel.read(buffer) > 0) {
-                buffer.flip();
-                while (buffer.hasRemaining()) {
-                    short channel = (short) Short.toUnsignedInt(buffer.getShort());
-                    long time = Integer.toUnsignedLong(buffer.getInt());
-                    byte status = buffer.get(); // status is ignored
-                    NumassEvent event = new NumassEvent(channel, (double) time * timeCoef * 1e-9);
-                    events.add(event);
-                }
-                buffer.clear(); // do something with the data and clear/compact it.
-            }
-        } catch (Exception ex) {
-            throw new RuntimeException(ex);
-        }
-
-
-//        LocalDateTime startTime = envelope.meta().get
-        double u = envelope.meta().getDouble("external_meta.HV1_value", 0);
-        double pointTime;
-        if (envelope.meta().hasValue("external_meta.acquisition_time")) {
-            pointTime = envelope.meta().getValue("external_meta.acquisition_time").doubleValue();
-        } else {
-            pointTime = envelope.meta().getValue("acquisition_time").doubleValue();
-        }
-
-        //Check if the point is composite
-        boolean segmented = envelope.meta().getBoolean("split", false);
-
-        if (!segmented && events.size() > RawNMPoint.MAX_EVENTS_PER_POINT) {
-            pointTime = events.get(events.size() - 1).getTime() - events.get(0).getTime();
-        }
-
-        return new RawNMPoint(u, u,
-                events,
-                pointTime,
-                readTime(envelope.meta()));
-    }
-
-    /**
-     * Read numass point without transformation
-     *
-     * @param envelope
-     * @return
-     */
-    private NumassPoint readPoint(Envelope envelope) {
-        return readPoint(envelope, PointBuilders::readRawPoint);
-    }
-
     private Map<String, Supplier<Envelope>> getItems() {
         return itemsProvider;
     }
@@ -239,7 +158,7 @@ public class NumassDataLoader extends AbstractLoader implements ObjectLoader<Env
     }
 
     @Override
-    public Data<Table> getHVData() {
+    public Data<Table> getHvData() {
         Envelope hvEnvelope = getHVEnvelope();
         if (hvEnvelope == null) {
             return Data.buildStatic(null);
@@ -263,20 +182,7 @@ public class NumassDataLoader extends AbstractLoader implements ObjectLoader<Env
         }
     }
 
-    @Override
+    private Stream<Envelope> getPointEnvelopes() {
-    public Stream<NumassPoint> stream() {
-        return this.getPoints().map(this::readPoint);
-    }
-
-    public List<NumassPoint> getNMPoints(Function<RawNMPoint, NumassPoint> transformation) {
-        return this.getPoints().map(env -> readPoint(env, transformation)).collect(Collectors.toList());
-    }
-
-    public List<RawNMPoint> getRawPoints() {
-        return this.getPoints().map(this::readRawPoint).collect(Collectors.toList());
-    }
-
-    private Stream<Envelope> getPoints() {
         return getItems().entrySet().stream()
                 .filter(entry -> entry.getKey().startsWith(POINT_FRAGMENT_NAME) && entry.getValue() != null)
                 .map(entry -> entry.getValue().get())
@@ -284,6 +190,11 @@ public class NumassDataLoader extends AbstractLoader implements ObjectLoader<Env
 
     }
 
+    @Override
+    public Stream<NumassPoint> getPoints() {
+        return getPointEnvelopes().map(EnvelopeNumassPoint::new);
+    }
+
     public boolean isReversed() {
         return meta().getBoolean("iteration_info.reverse", false);
     }
@@ -310,21 +221,12 @@ public class NumassDataLoader extends AbstractLoader implements ObjectLoader<Env
     }
 
     @Override
-    public Instant startTime() {
+    public Instant getStartTime() {
         if (meta.hasValue("start_time")) {
             return meta().getValue("start_time").timeValue();
-        } else return null;
+        } else {
-        //Temporary substitution for meta tag
+            return NumassSet.super.getStartTime();
-//        Envelope hvEnvelope = getHVEnvelope();
+        }
-//        if (hvEnvelope != null) {
-//            try {
-//                return Value.of(new Scanner(hvEnvelope.getData().getStream()).next()).timeValue();
-//            } catch (IOException ex) {
-//                return null;
-//            }
-//        } else {
-//            return null;
-//        }
     }
 
     @Override
@@ -337,63 +239,4 @@ public class NumassDataLoader extends AbstractLoader implements ObjectLoader<Env
 
     }
 
-    @Override
-    public String defaultTarget() {
-        return "nmPoint";
-    }
-
-    @Provides("nmPoint")
-    public Optional<NumassPoint> optPoint(String u) {
-        return stream().filter(it -> it.getVoltage() == Double.parseDouble(u)).findFirst();
-    }
-
-    @ProvidesNames("nmPoint")
-    public Stream<String> listHV() {
-        return stream().map(it -> Double.toString(it.getVoltage()));
-    }
-
-    @Provides("rawPoint")
-    public Optional<RawNMPoint> optRawPoint(String u) {
-        return this.getPoints().map(this::readRawPoint).filter(it -> it.getUset() == Double.parseDouble(u)).findFirst();
-    }
-
-    /**
-     * Return new NumassData using given transformation for each point
-     *
-     * @param transform
-     * @return
-     */
-    public NumassData applyRawTransformation(Function<RawNMPoint, NumassPoint> transform) {
-        return new NumassData() {
-            @Override
-            public String getDescription() {
-                return NumassDataLoader.this.getDescription();
-            }
-
-            @Override
-            public Meta meta() {
-                return NumassDataLoader.this.meta();
-            }
-
-            @Override
-            public Stream<NumassPoint> stream() {
-                return NumassDataLoader.this.stream();
-            }
-
-            @Override
-            public boolean isEmpty() {
-                return NumassDataLoader.this.isEmpty();
-            }
-
-            @Override
-            public Instant startTime() {
-                return NumassDataLoader.this.startTime();
-            }
-
-            @Override
-            public String getName() {
-                return NumassDataLoader.this.getName();
-            }
-        };
-    }
 }

numass-core/src/main/java/inr/numass/data/storage/NumassStorage.java

@@ -13,7 +13,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package inr.numass.storage;
+package inr.numass.data.storage;
 
 import hep.dataforge.context.Context;
 import hep.dataforge.events.Event;
@@ -21,7 +21,8 @@ import hep.dataforge.events.EventBuilder;
 import hep.dataforge.exceptions.StorageException;
 import hep.dataforge.meta.Meta;
 import hep.dataforge.storage.filestorage.FileStorage;
-import inr.numass.data.legacy.NMFile;
+import inr.numass.data.api.NumassSet;
+import inr.numass.data.legacy.NumassDatFile;
 import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
@@ -32,7 +33,8 @@ import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.List;
 
-import static java.nio.file.StandardOpenOption.*;
+import static java.nio.file.StandardOpenOption.CREATE;
+import static java.nio.file.StandardOpenOption.WRITE;
 
 
 /**
@@ -64,7 +66,7 @@ public class NumassStorage extends FileStorage {
     protected void updateDirectoryLoaders() {
         try {
             this.loaders.clear();
-            Files.list(getDataDir()).forEach( file -> {
+            Files.list(getDataDir()).forEach(file -> {
                 try {
                     if (Files.isDirectory(file)) {
                         Path metaFile = file.resolve(NumassDataLoader.META_FRAGMENT_NAME);
@@ -141,14 +143,14 @@ public class NumassStorage extends FileStorage {
      *
      * @return
      */
-    public List<NumassData> legacyFiles() {
+    public List<NumassSet> legacyFiles() {
         try {
-            List<NumassData> files = new ArrayList<>();
+            List<NumassSet> files = new ArrayList<>();
             Files.list(getDataDir()).forEach(file -> {
                 if (Files.isRegularFile(file) && file.getFileName().toString().toLowerCase().endsWith(".dat")) {
                     String name = file.getFileName().toString();
                     try {
-                        files.add(NMFile.readStream(Files.newInputStream(file, READ), name, Meta.buildEmpty("numassData")));
+                        files.add(new NumassDatFile(file, Meta.empty()));
                     } catch (Exception ex) {
                         LoggerFactory.getLogger(getClass()).error("Error while reading legacy numass file " + file.getFileName(), ex);
                     }

numass-core/src/main/java/inr/numass/data/storage/NumassStorageFactory.java

@@ -1,4 +1,4 @@
-package inr.numass.storage;
+package inr.numass.data.storage;
 
 import hep.dataforge.context.Context;
 import hep.dataforge.context.Global;

numass-core/src/main/resources/META-INF/services/hep.dataforge.data.DataLoader

@@ -1 +1 @@
-inr.numass.storage.NumassDataFactory
+inr.numass.data.storage.NumassDataFactory

numass-core/src/main/resources/META-INF/services/hep.dataforge.storage.api.StorageType

@@ -1 +1 @@
-inr.numass.storage.NumassStorageFactory
+inr.numass.data.storage.NumassStorageFactory

numass-main/src/main/groovy/inr/numass/data/PointAnalyzer.groovy

@@ -4,7 +4,6 @@ import groovy.transform.CompileStatic
 import hep.dataforge.maths.histogram.Histogram
 import hep.dataforge.maths.histogram.UnivariateHistogram
 import inr.numass.data.api.NumassEvent
-import inr.numass.data.legacy.RawNMPoint
 
 import java.util.stream.DoubleStream
 

numass-main/src/main/groovy/inr/numass/scripts/FindBorder.groovy

@@ -9,7 +9,7 @@ package inr.numass.scripts
 import hep.dataforge.grind.GrindMetaBuilder
 import hep.dataforge.meta.Meta
 import inr.numass.actions.FindBorderAction
-import inr.numass.storage.NumassDataLoader
+import inr.numass.data.storage.NumassDataLoader
 
 File dataDir = new File("D:\\Work\\Numass\\data\\2016_04\\T2_data\\Fill_2_2\\set_6_e26d123e54010000")
 if(!dataDir.exists()){

numass-main/src/main/groovy/inr/numass/scripts/SimulatePileup.groovy

@@ -8,9 +8,7 @@ package inr.numass.scripts
 
 import hep.dataforge.grind.Grind
 import hep.dataforge.values.Values
-import inr.numass.data.NumassPointImpl
+import inr.numass.data.storage.NumassDataLoader
-import inr.numass.data.legacy.RawNMPoint
-import inr.numass.storage.NumassDataLoader
 import inr.numass.utils.NMEventGeneratorWithPulser
 import inr.numass.utils.PileUpSimulator
 import inr.numass.utils.TritiumUtils

numass-main/src/main/groovy/inr/numass/scripts/Underflow.groovy

@@ -10,9 +10,8 @@ import hep.dataforge.io.ColumnedDataWriter
 import hep.dataforge.storage.commons.StorageUtils
 import hep.dataforge.tables.Table
 import inr.numass.data.NumassDataUtils
-import inr.numass.data.NumassPointImpl
+import inr.numass.data.storage.NumassStorage
-import inr.numass.storage.NumassStorage
+import inr.numass.data.storage.NumassStorageFactory
-import inr.numass.storage.NumassStorageFactory
 import inr.numass.utils.UnderflowCorrection
 
 //File rootDir = new File("D:\\Work\\Numass\\data\\2016_10\\Fill_1")

numass-main/src/main/groovy/inr/numass/scripts/times/MultiPointHistogram.groovy

@@ -7,10 +7,9 @@ import hep.dataforge.grind.helpers.PlotHelper
 import hep.dataforge.plots.fx.FXPlotManager
 import inr.numass.NumassPlugin
 import inr.numass.data.PointAnalyzer
-import inr.numass.data.legacy.RawNMPoint
+import inr.numass.data.storage.NumassDataLoader
-import inr.numass.storage.NumassDataLoader
+import inr.numass.data.storage.NumassStorage
-import inr.numass.storage.NumassStorage
+import inr.numass.data.storage.NumassStorageFactory
-import inr.numass.storage.NumassStorageFactory
 
 /**
  * Created by darksnake on 06-Jul-17.

numass-main/src/main/groovy/inr/numass/scripts/times/TestPointAnalyzer.groovy

@@ -8,9 +8,8 @@ import hep.dataforge.plots.fx.FXPlotManager
 import hep.dataforge.tables.ValueMap
 import inr.numass.NumassPlugin
 import inr.numass.data.PointAnalyzer
-import inr.numass.data.legacy.RawNMPoint
+import inr.numass.data.storage.NumassStorage
-import inr.numass.storage.NumassStorage
+import inr.numass.data.storage.NumassStorageFactory
-import inr.numass.storage.NumassStorageFactory
 
 /**
  * Created by darksnake on 27-Jun-17.

numass-main/src/main/java/inr/numass/NumassIO.java

@@ -20,8 +20,6 @@ import hep.dataforge.data.binary.Binary;
 import hep.dataforge.io.BasicIOManager;
 import hep.dataforge.meta.Meta;
 import hep.dataforge.names.Name;
-import inr.numass.data.legacy.LegacyDataReader;
-import inr.numass.data.legacy.RawNMFile;
 import org.apache.commons.io.FilenameUtils;
 import org.apache.commons.io.output.TeeOutputStream;
 

numass-main/src/main/java/inr/numass/actions/DebunchAction.java

@@ -21,8 +21,6 @@ import hep.dataforge.description.TypedActionDef;
 import hep.dataforge.description.ValueDef;
 import hep.dataforge.exceptions.ContentException;
 import hep.dataforge.meta.Laminate;
-import inr.numass.data.legacy.RawNMFile;
-import inr.numass.data.legacy.RawNMPoint;
 import inr.numass.debunch.DebunchReport;
 import inr.numass.debunch.FrameAnalizer;
 

numass-main/src/main/java/inr/numass/actions/FindBorderAction.java

@@ -25,7 +25,6 @@ import hep.dataforge.tables.ListTable;
 import hep.dataforge.tables.Table;
 import hep.dataforge.tables.ValueMap;
 import hep.dataforge.values.Value;
-import inr.numass.data.legacy.NMFile;
 import org.apache.commons.math3.analysis.UnivariateFunction;
 
 import java.io.OutputStream;

numass-main/src/main/java/inr/numass/actions/JoinNumassDataAction.java

@@ -4,7 +4,6 @@ import hep.dataforge.actions.ManyToOneAction;
 import hep.dataforge.context.Context;
 import hep.dataforge.description.TypedActionDef;
 import hep.dataforge.meta.Laminate;
-import inr.numass.data.NumassPointImpl;
 
 import java.util.Collection;
 import java.util.Map;

numass-main/src/main/java/inr/numass/actions/PrepareDataAction.java

@@ -32,10 +32,9 @@ import hep.dataforge.tables.ValueMap;
 import hep.dataforge.values.Values;
 import inr.numass.data.NumassPoint;
 import inr.numass.data.PointBuilders;
-import inr.numass.data.legacy.RawNMPoint;
+import inr.numass.data.storage.NumassDataLoader;
 import inr.numass.debunch.DebunchReport;
 import inr.numass.debunch.FrameAnalizer;
-import inr.numass.storage.NumassDataLoader;
 import inr.numass.utils.ExpressionUtils;
 
 import java.io.OutputStream;

numass-main/src/main/java/inr/numass/actions/ReadLegacyDataAction.java

@@ -23,8 +23,6 @@ import hep.dataforge.description.TypedActionDef;
 import hep.dataforge.description.ValueDef;
 import hep.dataforge.exceptions.ContentException;
 import hep.dataforge.meta.Laminate;
-import inr.numass.data.legacy.NMFile;
-import inr.numass.data.legacy.RawNMFile;
 
 import static hep.dataforge.values.ValueType.NUMBER;
 import static inr.numass.NumassIO.getNumassData;

numass-main/src/main/java/inr/numass/debunch/DebunchData.java

@@ -16,7 +16,6 @@
 package inr.numass.debunch;
 
 import inr.numass.data.api.NumassEvent;
-import inr.numass.data.legacy.RawNMPoint;
 
 import java.util.ArrayList;
 import java.util.Collections;

numass-main/src/main/java/inr/numass/debunch/DebunchReport.java

@@ -16,7 +16,6 @@
 package inr.numass.debunch;
 
 import inr.numass.data.api.NumassEvent;
-import inr.numass.data.legacy.RawNMPoint;
 
 import java.util.List;
 

numass-main/src/main/java/inr/numass/debunch/DebunchReportImpl.java

@@ -16,7 +16,6 @@
 package inr.numass.debunch;
 
 import inr.numass.data.api.NumassEvent;
-import inr.numass.data.legacy.RawNMPoint;
 
 import java.util.ArrayList;
 import java.util.List;

numass-main/src/main/java/inr/numass/debunch/Debuncher.java

@@ -15,8 +15,6 @@
  */
 package inr.numass.debunch;
 
-import inr.numass.data.legacy.RawNMPoint;
-
 /**
  *
  * @author Darksnake

numass-main/src/main/java/inr/numass/debunch/FrameAnalizer.java

@@ -15,7 +15,6 @@
  */
 package inr.numass.debunch;
 
-import inr.numass.data.legacy.RawNMPoint;
 import org.apache.commons.math3.analysis.UnivariateFunction;
 import org.apache.commons.math3.analysis.interpolation.LinearInterpolator;
 import org.apache.commons.math3.util.FastMath;

numass-main/src/main/java/inr/numass/utils/BunchGenerator.java

@@ -16,7 +16,6 @@
 package inr.numass.utils;
 
 import inr.numass.data.api.NumassEvent;
-import inr.numass.data.legacy.RawNMPoint;
 import org.apache.commons.math3.random.MersenneTwister;
 import org.apache.commons.math3.random.RandomGenerator;
 import org.apache.commons.math3.random.SynchronizedRandomGenerator;

numass-main/src/main/java/inr/numass/utils/NMEventGenerator.java

@@ -17,7 +17,6 @@ package inr.numass.utils;
 
 import hep.dataforge.meta.Meta;
 import inr.numass.data.api.NumassEvent;
-import inr.numass.data.legacy.RawNMPoint;
 import org.apache.commons.math3.distribution.EnumeratedRealDistribution;
 import org.apache.commons.math3.distribution.RealDistribution;
 import org.apache.commons.math3.random.EmpiricalDistribution;

numass-main/src/main/java/inr/numass/utils/PileUpSimulator.java

@@ -7,7 +7,6 @@ package inr.numass.utils;
 
 import inr.numass.data.PointBuilders;
 import inr.numass.data.api.NumassEvent;
-import inr.numass.data.legacy.RawNMPoint;
 import org.apache.commons.math3.random.RandomGenerator;
 
 import java.util.ArrayList;

numass-server/src/main/java/inr/numass/server/NumassRun.java

@@ -27,7 +27,7 @@ import hep.dataforge.storage.api.Storage;
 import hep.dataforge.storage.commons.LoaderFactory;
 import hep.dataforge.storage.commons.MessageFactory;
 import hep.dataforge.values.Value;
-import inr.numass.storage.NumassStorage;
+import inr.numass.data.storage.NumassStorage;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 

numass-server/src/main/java/inr/numass/server/NumassServer.java

@@ -27,7 +27,7 @@ import hep.dataforge.storage.commons.AbstractNetworkListener;
 import hep.dataforge.storage.commons.LoaderFactory;
 import hep.dataforge.storage.commons.StorageManager;
 import hep.dataforge.storage.commons.StorageUtils;
-import inr.numass.storage.NumassStorage;
+import inr.numass.data.storage.NumassStorage;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import ratpack.server.RatpackServer;

numass-server/src/main/java/inr/numass/server/ServerRunner.java

@@ -5,7 +5,7 @@ import hep.dataforge.context.Global;
 import hep.dataforge.io.MetaFileReader;
 import hep.dataforge.meta.Meta;
 import hep.dataforge.meta.SimpleConfigurable;
-import inr.numass.storage.NumassStorage;
+import inr.numass.data.storage.NumassStorage;
 import org.slf4j.LoggerFactory;
 
 import java.io.File;

numass-server/src/main/java/inr/numass/server/TestServer.java

@@ -11,7 +11,7 @@ import hep.dataforge.server.ServerManager;
 import hep.dataforge.server.storage.StorageServerUtils;
 import hep.dataforge.storage.commons.StorageManager;
 import hep.dataforge.storage.filestorage.FileStorageFactory;
-import inr.numass.storage.NumassStorage;
+import inr.numass.data.storage.NumassStorage;
 
 import java.io.BufferedReader;
 import java.io.File;

numass-test/src/main/groovy/inr/numass/scripts/TestServer.groovy

@@ -16,8 +16,8 @@
 package inr.numass.scripts
 
 import hep.dataforge.meta.Meta
+import inr.numass.data.storage.NumassStorage
 import inr.numass.server.NumassServer
-import inr.numass.storage.NumassStorage
 
 String path = "D:\\Work\\Numass\\data\\2016_10\\"
 

numass-viewer/src/main/java/inr/numass/viewer/TestDirectoryViewer.java

@@ -16,7 +16,7 @@
 package inr.numass.viewer;
 
 import hep.dataforge.storage.commons.StorageManager;
-import inr.numass.storage.NumassDataLoader;
+import inr.numass.data.storage.NumassDataLoader;
 import javafx.application.Application;
 import javafx.scene.Scene;
 import javafx.stage.Stage;

numass-viewer/src/main/kotlin/inr/numass/viewer/MainView.kt

@@ -15,7 +15,7 @@ import hep.dataforge.storage.api.PointLoader
 import hep.dataforge.storage.api.Storage
 import hep.dataforge.storage.filestorage.FileStorageFactory
 import inr.numass.NumassProperties
-import inr.numass.storage.NumassStorage
+import inr.numass.data.storage.NumassStorage
 import javafx.application.Platform
 import javafx.beans.property.SimpleObjectProperty
 import javafx.geometry.Insets