diff --git a/numass-main/src/main/java/inr/numass/actions/PrepareDataAction.java b/numass-main/src/main/java/inr/numass/actions/PrepareDataAction.java index f34279de..6047d0e5 100644 --- a/numass-main/src/main/java/inr/numass/actions/PrepareDataAction.java +++ b/numass-main/src/main/java/inr/numass/actions/PrepareDataAction.java @@ -26,8 +26,11 @@ import hep.dataforge.io.XMLMetaWriter; import hep.dataforge.meta.Laminate; import hep.dataforge.meta.Meta; import hep.dataforge.tables.*; +import inr.numass.debunch.DebunchReport; +import inr.numass.debunch.FrameAnalizer; import inr.numass.storage.NMPoint; import inr.numass.storage.NumassData; +import inr.numass.storage.NumassDataLoader; import inr.numass.storage.RawNMPoint; import inr.numass.utils.ExpressionUtils; @@ -99,6 +102,14 @@ public class PrepareDataAction extends OneToOneAction { utransform = Function.identity(); } + if(meta.hasMeta("debunch")){ + if(dataFile instanceof NumassDataLoader){ + dataFile = ((NumassDataLoader) dataFile).applyRawTransformation(raw->debunch(context,raw,meta.getMeta("debunch"))); + } else { + throw new RuntimeException("Debunch not available"); + } + } + List dataList = new ArrayList<>(); for (NMPoint point : dataFile) { @@ -183,6 +194,22 @@ public class PrepareDataAction extends OneToOneAction { }; } + private NMPoint debunch(Context context, RawNMPoint point, Meta meta) { + int upper = meta.getInt("upperchanel", RawNMPoint.MAX_CHANEL); + int lower = meta.getInt("lowerchanel", 0); + double rejectionprob = meta.getDouble("rejectprob", 1e-10); + double framelength = meta.getDouble("framelength", 1); + double maxCR = meta.getDouble("maxcr", 500d); + + double cr = point.selectChanels(lower, upper).getCR(); + if (cr < maxCR) { + DebunchReport report = new FrameAnalizer(rejectionprob, framelength, lower, upper).debunchPoint(point); + return new NMPoint(report.getPoint()); + } else { + return new NMPoint(point); + } + } + private interface Correction { /** diff --git a/numass-main/src/main/java/inr/numass/actions/ReadStorageAction.java b/numass-main/src/main/java/inr/numass/actions/ReadStorageAction.java deleted file mode 100644 index 91e5aa86..00000000 --- a/numass-main/src/main/java/inr/numass/actions/ReadStorageAction.java +++ /dev/null @@ -1,60 +0,0 @@ -package inr.numass.actions; - -import hep.dataforge.actions.OneToManyAction; -import hep.dataforge.context.Context; -import hep.dataforge.meta.Laminate; -import hep.dataforge.meta.Meta; -import hep.dataforge.storage.api.Loader; -import inr.numass.debunch.DebunchReport; -import inr.numass.debunch.FrameAnalizer; -import inr.numass.storage.*; - -import java.util.Map; - -/** - * Created by darksnake on 29-Jan-17. - */ -public class ReadStorageAction extends OneToManyAction { - @Override - protected Map prepareMeta(Context context, String inputName, Laminate meta) { - return null; - } - - @Override - protected NumassData execute(Context context, String inputName, String outputName, NumassStorage input, Laminate meta) { - try(Loader loader = input.getLoader(outputName)) { - if (loader instanceof NumassDataLoader) { - NumassDataLoader nd = (NumassDataLoader) loader; - return buildData(context, nd, meta); - } else { - throw new RuntimeException("Numass loader expected"); - } - } catch (Exception ex){ - throw new RuntimeException(ex); - } - } - - private NumassData buildData(Context context, NumassDataLoader loader, Meta meta) { - if (meta.hasNode("debunch")) { - return loader.applyRawTransformation(rp -> debunch(context, rp, meta.getMeta("debunch"))); - } else { - return loader; - } - } - - private NMPoint debunch(Context context, RawNMPoint point, Meta meta) { - int upper = meta.getInt("upperchanel", RawNMPoint.MAX_CHANEL); - int lower = meta.getInt("lowerchanel", 0); - double rejectionprob = meta.getDouble("rejectprob", 1e-10); - double framelength = meta.getDouble("framelength", 1); - double maxCR = meta.getDouble("maxcr", 500d); - - double cr = point.selectChanels(lower, upper).getCR(); - if (cr < maxCR) { - DebunchReport report = new FrameAnalizer(rejectionprob, framelength, lower, upper).debunchPoint(point); - return new NMPoint(report.getPoint()); - } else { - return new NMPoint(point); - } - } -} diff --git a/numass-main/src/main/java/inr/numass/tasks/NumassPrepareTask.java b/numass-main/src/main/java/inr/numass/tasks/NumassPrepareTask.java index 4f00b31f..3a99e843 100644 --- a/numass-main/src/main/java/inr/numass/tasks/NumassPrepareTask.java +++ b/numass-main/src/main/java/inr/numass/tasks/NumassPrepareTask.java @@ -7,25 +7,20 @@ package inr.numass.tasks; import hep.dataforge.actions.GenericAction; import hep.dataforge.context.Context; -import hep.dataforge.data.*; +import hep.dataforge.data.DataFilter; +import hep.dataforge.data.DataNode; +import hep.dataforge.data.DataTree; import hep.dataforge.description.DescriptorBuilder; import hep.dataforge.description.NodeDescriptor; -import hep.dataforge.goals.Work; import hep.dataforge.meta.Meta; import hep.dataforge.meta.Template; -import hep.dataforge.storage.api.Loader; -import hep.dataforge.storage.commons.StorageUtils; import hep.dataforge.tables.Table; import hep.dataforge.workspace.AbstractTask; import hep.dataforge.workspace.TaskModel; import inr.numass.actions.MergeDataAction; import inr.numass.actions.MonitorCorrectAction; import inr.numass.actions.PrepareDataAction; -import inr.numass.debunch.DebunchReport; -import inr.numass.debunch.FrameAnalizer; -import inr.numass.storage.*; - -import java.net.URI; +import inr.numass.storage.NumassData; /** * Prepare data task @@ -40,10 +35,23 @@ public class NumassPrepareTask extends AbstractTask { Context context = model.getContext(); //acquiring initial data. Data node could not be empty + DataNode data; + Meta dataMeta = config.getMeta("data"); - URI storageUri = input.getCheckedData("dataRoot", URI.class).get(); - DataSet.Builder dataBuilder = readData(getWork(model, input.getName()), context, storageUri, dataMeta); - DataNode data = dataBuilder.build(); + if (dataMeta.hasValue("from")) { + data = input.getCheckedNode(dataMeta.getString("from"), NumassData.class); + } else { + data = input.checked(NumassData.class); + } + + DataFilter filter = new DataFilter().configure(dataMeta); + + data = filter.filter(data); + +// Meta dataMeta = config.getMeta("data"); +// URI storageUri = input.getCheckedData("dataRoot", URI.class).get(); +// DataSet.Builder dataBuilder = readData(getWork(model, input.getName()), context, storageUri, dataMeta); +// DataNode data = dataBuilder.build(); //preparing table data Meta prepareMeta = config.getMeta("prepare"); @@ -80,73 +88,73 @@ public class NumassPrepareTask extends AbstractTask
{ return model; } - private DataSet.Builder readData(Work callback, Context context, URI numassRoot, Meta meta) { +// private DataSet.Builder readData(Work callback, Context context, URI numassRoot, Meta meta) { +// +// NumassStorage storage = NumassStorage.buildNumassRoot(numassRoot, true, false); +// DataFilter filter = new DataFilter().configure(meta); +// +// boolean forwardOnly = meta.getBoolean("forwardOnly", false); +// boolean reverseOnly = meta.getBoolean("reverseOnly", false); +//// SetDirectionUtility.load(context); +// +// DataSet.Builder builder = DataSet.builder(NumassData.class); +// callback.setMaxProgress(StorageUtils.loaderStream(storage).count()); +// StorageUtils.loaderStream(storage).forEach(pair -> { +// Loader loader = pair.getValue(); +// if (loader instanceof NumassDataLoader) { +// NumassDataLoader nd = (NumassDataLoader) loader; +// Data datum = buildData(context, nd, meta); +// if (filter.acceptData(pair.getKey(), datum)) { +// boolean accept = true; +// if (forwardOnly || reverseOnly) { +// boolean reversed = nd.isReversed(); +// accept = (reverseOnly && reversed) || (forwardOnly && !reversed); +// } +// if (accept) { +// builder.putData(pair.getKey(), datum); +// } +// } +// } +// callback.increaseProgress(1d); +// }); +// +// if (meta.getBoolean("loadLegacy", false)) { +// storage.legacyFiles().forEach(nd -> { +// Data datum = Data.buildStatic(nd); +// if (filter.acceptData(nd.getName(), datum)) { +// builder.putData("legacy." + nd.getName(), datum); +// } +// }); +// } +// //FIXME remove in later revisions +//// SetDirectionUtility.save(context); +// +// return builder; +// } - NumassStorage storage = NumassStorage.buildNumassRoot(numassRoot, true, false); - DataFilter filter = new DataFilter().configure(meta); - - boolean forwardOnly = meta.getBoolean("forwardOnly", false); - boolean reverseOnly = meta.getBoolean("reverseOnly", false); -// SetDirectionUtility.load(context); - - DataSet.Builder builder = DataSet.builder(NumassData.class); - callback.setMaxProgress(StorageUtils.loaderStream(storage).count()); - StorageUtils.loaderStream(storage).forEach(pair -> { - Loader loader = pair.getValue(); - if (loader instanceof NumassDataLoader) { - NumassDataLoader nd = (NumassDataLoader) loader; - Data datum = buildData(context, nd, meta); - if (filter.acceptData(pair.getKey(), datum)) { - boolean accept = true; - if (forwardOnly || reverseOnly) { - boolean reversed = nd.isReversed(); - accept = (reverseOnly && reversed) || (forwardOnly && !reversed); - } - if (accept) { - builder.putData(pair.getKey(), datum); - } - } - } - callback.increaseProgress(1d); - }); - - if (meta.getBoolean("loadLegacy", false)) { - storage.legacyFiles().forEach(nd -> { - Data datum = Data.buildStatic(nd); - if (filter.acceptData(nd.getName(), datum)) { - builder.putData("legacy." + nd.getName(), datum); - } - }); - } - //FIXME remove in later revisions -// SetDirectionUtility.save(context); - - return builder; - } - - private Data buildData(Context context, NumassDataLoader loader, Meta meta) { - if (meta.hasNode("debunch")) { - return Data.buildStatic(loader.applyRawTransformation(rp -> debunch(context, rp, meta.getMeta("debunch")))); - } else { - return Data.buildStatic(loader); - } - } - - private NMPoint debunch(Context context, RawNMPoint point, Meta meta) { - int upper = meta.getInt("upperchanel", RawNMPoint.MAX_CHANEL); - int lower = meta.getInt("lowerchanel", 0); - double rejectionprob = meta.getDouble("rejectprob", 1e-10); - double framelength = meta.getDouble("framelength", 1); - double maxCR = meta.getDouble("maxcr", 500d); - - double cr = point.selectChanels(lower, upper).getCR(); - if (cr < maxCR) { - DebunchReport report = new FrameAnalizer(rejectionprob, framelength, lower, upper).debunchPoint(point); - return new NMPoint(report.getPoint()); - } else { - return new NMPoint(point); - } - } +// private Data buildData(Context context, NumassDataLoader loader, Meta meta) { +// if (meta.hasNode("debunch")) { +// return Data.buildStatic(loader.applyRawTransformation(rp -> debunch(context, rp, meta.getMeta("debunch")))); +// } else { +// return Data.buildStatic(loader); +// } +// } +// +// private NMPoint debunch(Context context, RawNMPoint point, Meta meta) { +// int upper = meta.getInt("upperchanel", RawNMPoint.MAX_CHANEL); +// int lower = meta.getInt("lowerchanel", 0); +// double rejectionprob = meta.getDouble("rejectprob", 1e-10); +// double framelength = meta.getDouble("framelength", 1); +// double maxCR = meta.getDouble("maxcr", 500d); +// +// double cr = point.selectChanels(lower, upper).getCR(); +// if (cr < maxCR) { +// DebunchReport report = new FrameAnalizer(rejectionprob, framelength, lower, upper).debunchPoint(point); +// return new NMPoint(report.getPoint()); +// } else { +// return new NMPoint(point); +// } +// } private DataNode runAction(GenericAction action, Context context, DataNode data, Meta meta) { return action.run(context, data, meta); diff --git a/numass-main/src/main/java/inr/numass/workbench/NumassWorkbenchController.java b/numass-main/src/main/java/inr/numass/workbench/NumassWorkbenchController.java index afccd04d..e04e64d5 100644 --- a/numass-main/src/main/java/inr/numass/workbench/NumassWorkbenchController.java +++ b/numass-main/src/main/java/inr/numass/workbench/NumassWorkbenchController.java @@ -360,7 +360,7 @@ public class NumassWorkbenchController implements Initializable, StagePaneHolder statusBar.setProgress(-1); }); - DataNode data = new FileDataFactory().load(getContext(), getDataConfiguration()); + DataNode data = new FileDataFactory().build(getContext(), getDataConfiguration()); try { ActionUtils.runAction(getContext(), data, getActionConfiguration()).computeAll(); Platform.runLater(() -> statusBar.setText("Execution complete")); diff --git a/numass-storage/src/main/java/inr/numass/storage/NMPoint.java b/numass-storage/src/main/java/inr/numass/storage/NMPoint.java index 0d01e4ad..3855a457 100644 --- a/numass-storage/src/main/java/inr/numass/storage/NMPoint.java +++ b/numass-storage/src/main/java/inr/numass/storage/NMPoint.java @@ -28,7 +28,7 @@ import static java.util.Arrays.sort; * @author Darksnake */ public class NMPoint { - //TODO transform to annotated and move some parameters to meta + //TODO andThen to annotated and move some parameters to meta static final String[] dataNames = {"chanel", "count"}; private final int[] spectrum; private Instant startTime; diff --git a/numass-storage/src/main/java/inr/numass/storage/NumassDataFactory.java b/numass-storage/src/main/java/inr/numass/storage/NumassDataFactory.java new file mode 100644 index 00000000..c4da4ba3 --- /dev/null +++ b/numass-storage/src/main/java/inr/numass/storage/NumassDataFactory.java @@ -0,0 +1,37 @@ +package inr.numass.storage; + +import hep.dataforge.context.Context; +import hep.dataforge.data.DataFactory; +import hep.dataforge.data.DataFilter; +import hep.dataforge.data.DataTree; +import hep.dataforge.meta.Meta; +import hep.dataforge.storage.commons.StorageUtils; + +/** + * Created by darksnake on 03-Feb-17. + */ +public class NumassDataFactory extends DataFactory { + + public NumassDataFactory() { + super(NumassData.class); + } + + @Override + public String getName() { + return "numass"; + } + + @Override + protected void buildChildren(Context context, DataTree.Builder builder, DataFilter filter, Meta meta) { + NumassStorage storage = NumassStorage.buildNumassRoot( + meta.getString("path"), + meta.getBoolean("readOnly", true), + meta.getBoolean("monitor", false) + ); + StorageUtils.loaderStream(storage).forEach(pair -> { + if (pair.getValue() instanceof NumassData) { + builder.putStatic(pair.getKey(), (NumassData) pair.getValue()); + } + }); + } +} diff --git a/numass-storage/src/main/java/inr/numass/storage/NumassStorage.java b/numass-storage/src/main/java/inr/numass/storage/NumassStorage.java index 502fc8b4..4116d007 100644 --- a/numass-storage/src/main/java/inr/numass/storage/NumassStorage.java +++ b/numass-storage/src/main/java/inr/numass/storage/NumassStorage.java @@ -56,17 +56,6 @@ public class NumassStorage extends FileStorage { public static final String NUMASS_DATA_LOADER_TYPE = "numassData"; public static final String GROUP_META_FILE = "numass_group_meta"; - protected NumassStorage(FileStorage parent, String path, Meta config) throws StorageException { - super(parent, path, config); - super.refresh(); - //TODO read meta from numass_group_meta to .numass element - } - - protected NumassStorage(FileObject dir, Meta config) throws StorageException { - super(dir, config); - super.refresh(); - } - /** * Create root numass storage * @@ -131,6 +120,17 @@ public class NumassStorage extends FileStorage { } } + protected NumassStorage(FileStorage parent, String path, Meta config) throws StorageException { + super(parent, path, config); + super.refresh(); + //TODO read meta from numass_group_meta to .numass element + } + + protected NumassStorage(FileObject dir, Meta config) throws StorageException { + super(dir, config); + super.refresh(); + } + @Override protected void updateDirectoryLoaders() { try { diff --git a/numass-storage/src/main/resources/META-INF/services/hep.dataforge.data.DataLoader b/numass-storage/src/main/resources/META-INF/services/hep.dataforge.data.DataLoader new file mode 100644 index 00000000..4e319597 --- /dev/null +++ b/numass-storage/src/main/resources/META-INF/services/hep.dataforge.data.DataLoader @@ -0,0 +1 @@ +inr.numass.storage.NumassDataFactory \ No newline at end of file