NPM/numass-framework
4
0
Fork 1
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Multiple fixes to storage

Browse Source
This commit is contained in:
Alexander Nozik 2017-12-08 12:40:47 +03:00
parent 35fa048010
commit 038a40dbee
9 changed files with 55 additions and 103 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
numass-core/src/main/java/inr/numass/data/analyzers/AbstractAnalyzer.java
View File

@ -18,10 +18,10 @@ import static inr.numass.data.api.NumassPoint.HV_KEY;
* Created by darksnake on 11.07.2017. * Created by darksnake on 11.07.2017.
*/ */
public abstract class AbstractAnalyzer implements NumassAnalyzer { public abstract class AbstractAnalyzer implements NumassAnalyzer {
public static String WINDOW_KEY = "window"; public static final String WINDOW_KEY = "window";
public static String TIME_KEY = "timestamp"; public static final String TIME_KEY = "timestamp";
public static String[] NAME_LIST = {LENGTH_KEY, COUNT_KEY, COUNT_RATE_KEY, COUNT_RATE_ERROR_KEY, WINDOW_KEY, TIME_KEY}; public static final String[] NAME_LIST = {LENGTH_KEY, COUNT_KEY, COUNT_RATE_KEY, COUNT_RATE_ERROR_KEY, WINDOW_KEY, TIME_KEY};
// public static String[] NAME_LIST_WITH_HV = {HV_KEY, LENGTH_KEY, COUNT_KEY, COUNT_RATE_KEY, COUNT_RATE_ERROR_KEY, WINDOW_KEY, TIME_KEY}; // public static String[] NAME_LIST_WITH_HV = {HV_KEY, LENGTH_KEY, COUNT_KEY, COUNT_RATE_KEY, COUNT_RATE_ERROR_KEY, WINDOW_KEY, TIME_KEY};
@Nullable @Nullable
private final SignalProcessor processor; private final SignalProcessor processor;

28
numass-core/src/main/java/inr/numass/data/analyzers/SmartAnalyzer.java
View File

@ -1,11 +1,9 @@
package inr.numass.data.analyzers; package inr.numass.data.analyzers;
import hep.dataforge.description.ValueDef;
import hep.dataforge.meta.Meta; import hep.dataforge.meta.Meta;
import hep.dataforge.tables.TableFormat; import hep.dataforge.tables.TableFormat;
import hep.dataforge.tables.ValueMap; import hep.dataforge.tables.ValueMap;
import hep.dataforge.values.Value; import hep.dataforge.values.Value;
import hep.dataforge.values.ValueType;
import hep.dataforge.values.Values; import hep.dataforge.values.Values;
import inr.numass.data.api.NumassAnalyzer; import inr.numass.data.api.NumassAnalyzer;
import inr.numass.data.api.NumassBlock; import inr.numass.data.api.NumassBlock;
@ -48,7 +46,7 @@ public class SmartAnalyzer extends AbstractAnalyzer {
throw new IllegalArgumentException("Analyzer not found"); throw new IllegalArgumentException("Analyzer not found");
} }
} else { } else {
if(config.hasValue("t0")||config.hasValue("t0")){ if(config.hasValue("t0")||config.hasMeta("t0")){
return timeAnalyzer; return timeAnalyzer;
} else { } else {
return simpleAnalyzer; return simpleAnalyzer;
@ -64,35 +62,11 @@ public class SmartAnalyzer extends AbstractAnalyzer {
return new ValueMap(map); return new ValueMap(map);
} }
private double estimateCountRate(NumassBlock block) {
return (double) block.getEvents().count() / block.getLength().toMillis() * 1000;
}
@Override @Override
public Stream<NumassEvent> getEvents(NumassBlock block, Meta config) { public Stream<NumassEvent> getEvents(NumassBlock block, Meta config) {
return getAnalyzer(config).getEvents(block, config); return getAnalyzer(config).getEvents(block, config);
} }
@ValueDef(name = "t0", type = ValueType.NUMBER, info = "Constant t0 cut")
@ValueDef(name = "t0.crFraction", type = ValueType.NUMBER, info = "The relative fraction of events that should be removed by time cut")
@ValueDef(name = "t0.min", type = ValueType.NUMBER, def = "0", info = "Minimal t0")
private int getT0(NumassBlock block, Meta meta) {
if (meta.hasValue("t0")) {
return meta.getInt("t0");
} else if (meta.hasMeta("t0")) {
double fraction = meta.getDouble("t0.crFraction");
double cr = estimateCountRate(block);
if (cr < meta.getDouble("t0.minCR", 0)) {
return 0;
} else {
return (int) Math.max(-1e9 / cr * Math.log(1d - fraction), meta.getDouble("t0.min", 0));
}
} else {
return 0;
}
}
@Override @Override
protected TableFormat getTableFormat(Meta config) { protected TableFormat getTableFormat(Meta config) {
if (config.hasValue(TimeAnalyzer.T0_KEY) || config.hasMeta(TimeAnalyzer.T0_KEY)) { if (config.hasValue(TimeAnalyzer.T0_KEY) || config.hasMeta(TimeAnalyzer.T0_KEY)) {

32
numass-core/src/main/java/inr/numass/data/analyzers/TimeAnalyzer.java
View File

@ -1,10 +1,12 @@
package inr.numass.data.analyzers; package inr.numass.data.analyzers;
import hep.dataforge.description.ValueDef;
import hep.dataforge.meta.Meta; import hep.dataforge.meta.Meta;
import hep.dataforge.tables.TableFormat; import hep.dataforge.tables.TableFormat;
import hep.dataforge.tables.TableFormatBuilder; import hep.dataforge.tables.TableFormatBuilder;
import hep.dataforge.tables.ValueMap; import hep.dataforge.tables.ValueMap;
import hep.dataforge.values.Value; import hep.dataforge.values.Value;
import hep.dataforge.values.ValueType;
import hep.dataforge.values.Values; import hep.dataforge.values.Values;
import inr.numass.data.api.NumassBlock; import inr.numass.data.api.NumassBlock;
import inr.numass.data.api.NumassEvent; import inr.numass.data.api.NumassEvent;
@ -29,7 +31,7 @@ import static inr.numass.data.api.NumassPoint.HV_KEY;
public class TimeAnalyzer extends AbstractAnalyzer { public class TimeAnalyzer extends AbstractAnalyzer {
public static String T0_KEY = "t0"; public static String T0_KEY = "t0";
public static String[] NAME_LIST = {LENGTH_KEY, COUNT_KEY, COUNT_RATE_KEY, COUNT_RATE_ERROR_KEY, WINDOW_KEY, TIME_KEY, T0_KEY}; public static final String[] NAME_LIST = {LENGTH_KEY, COUNT_KEY, COUNT_RATE_KEY, COUNT_RATE_ERROR_KEY, WINDOW_KEY, TIME_KEY, T0_KEY};
// public static String[] NAME_LIST_WITH_HV = {HV_KEY, LENGTH_KEY, COUNT_KEY, COUNT_RATE_KEY, COUNT_RATE_ERROR_KEY, WINDOW_KEY, TIME_KEY, T0_KEY}; // public static String[] NAME_LIST_WITH_HV = {HV_KEY, LENGTH_KEY, COUNT_KEY, COUNT_RATE_KEY, COUNT_RATE_ERROR_KEY, WINDOW_KEY, TIME_KEY, T0_KEY};
public TimeAnalyzer(@Nullable SignalProcessor processor) { public TimeAnalyzer(@Nullable SignalProcessor processor) {
@ -81,7 +83,9 @@ public class TimeAnalyzer extends AbstractAnalyzer {
@Override @Override
public Values analyzePoint(NumassPoint point, Meta config) { public Values analyzePoint(NumassPoint point, Meta config) {
//Average count rates, do not sum events //Average count rates, do not sum events
Values res = point.getBlocks().map(it -> analyze(it, config)).reduce(null, this::combineBlockResults); Values res = point.getBlocks()
.map(it -> analyze(it, config))
.reduce(null, this::combineBlockResults);
Map<String, Value> map = new HashMap<>(res.asMap()); Map<String, Value> map = new HashMap<>(res.asMap());
map.put(HV_KEY, Value.of(point.getVoltage())); map.put(HV_KEY, Value.of(point.getVoltage()));
@ -127,8 +131,28 @@ public class TimeAnalyzer extends AbstractAnalyzer {
); );
} }
private long getT0(NumassBlock block, Meta config) { @ValueDef(name = "t0", type = ValueType.NUMBER, info = "Constant t0 cut")
return config.getValue("t0", 0).longValue(); @ValueDef(name = "t0.crFraction", type = ValueType.NUMBER, info = "The relative fraction of events that should be removed by time cut")
@ValueDef(name = "t0.min", type = ValueType.NUMBER, def = "0", info = "Minimal t0")
private int getT0(NumassBlock block, Meta meta) {
if (meta.hasValue("t0")) {
return meta.getInt("t0");
} else if (meta.hasMeta("t0")) {
double fraction = meta.getDouble("t0.crFraction");
double cr = estimateCountRate(block);
if (cr < meta.getDouble("t0.minCR", 0)) {
return 0;
} else {
return (int) Math.max(-1e9 / cr * Math.log(1d - fraction), meta.getDouble("t0.min", 0));
}
} else {
return 0;
}
}
private double estimateCountRate(NumassBlock block) {
return (double) block.getEvents().count() / block.getLength().toMillis() * 1000;
} }
/** /**

21
numass-core/src/main/java/inr/numass/data/storage/NumassStorageFactory.java
View File

@ -2,11 +2,12 @@ package inr.numass.data.storage;
import com.github.robtimus.filesystems.sftp.SFTPEnvironment; import com.github.robtimus.filesystems.sftp.SFTPEnvironment;
import hep.dataforge.context.Context; import hep.dataforge.context.Context;
import hep.dataforge.context.Global;
import hep.dataforge.meta.Meta; import hep.dataforge.meta.Meta;
import hep.dataforge.meta.MetaBuilder; import hep.dataforge.meta.MetaBuilder;
import hep.dataforge.storage.api.Storage; import hep.dataforge.storage.api.Storage;
import hep.dataforge.storage.api.StorageType; import hep.dataforge.storage.api.StorageType;
import hep.dataforge.storage.commons.StorageManager;
import hep.dataforge.storage.filestorage.FileStorage;
import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.NotNull;
import java.io.File; import java.io.File;
@ -28,13 +29,9 @@ public class NumassStorageFactory implements StorageType {
* @return * @return
*/ */
@NotNull @NotNull
public static NumassStorage buildLocal(File file, boolean monitor) { public static FileStorage buildLocal(Context context, File file, boolean readOnly, boolean monitor) {
Path path = file.toPath(); StorageManager manager = context.loadFeature("hep.dataforge:storage", StorageManager.class);
Meta meta = new MetaBuilder("storage") return (FileStorage) manager.buildStorage(buildStorageMeta(file.toURI(),readOnly,monitor));
.setValue("path", path)
.setValue("monitor", monitor);
return new NumassStorage(Global.instance(), meta, path);
} }
@Override @Override
@ -70,4 +67,12 @@ public class NumassStorageFactory implements StorageType {
return new NumassStorage(context, meta, context.getIo().getWorkDirectory()); return new NumassStorage(context, meta, context.getIo().getWorkDirectory());
} }
} }
public static MetaBuilder buildStorageMeta(URI path, boolean readOnly, boolean monitor) {
return new MetaBuilder("storage")
.setValue("path", path.toString())
.setValue("type", "numass")
.setValue("readOnly", readOnly)
.setValue("monitor", monitor);
}
} }

55
numass-main/src/main/resources/fxml/NumassWorkbench.fxml
View File

@ -1,55 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<?import javafx.scene.control.Accordion?>
<?import javafx.scene.control.Button?>
<?import javafx.scene.control.ScrollPane?>
<?import javafx.scene.control.Separator?>
<?import javafx.scene.control.SplitPane?>
<?import javafx.scene.control.Tab?>
<?import javafx.scene.control.TabPane?>
<?import javafx.scene.control.ToggleButton?>
<?import javafx.scene.control.ToolBar?>
<?import javafx.scene.layout.AnchorPane?>
<?import javafx.scene.layout.BorderPane?>
<?import javafx.scene.layout.HBox?>
<?import javafx.scene.layout.Pane?>
<?import org.controlsfx.control.StatusBar?>
<AnchorPane id="AnchorPane" prefHeight="600.0" prefWidth="800.0" xmlns="http://javafx.com/javafx/8.0.60" xmlns:fx="http://javafx.com/fxml/1" fx:controller="inr.numass.workbench.NumassWorkbenchController">
<children>
<BorderPane layoutX="-165.0" layoutY="100.0" prefHeight="200.0" prefWidth="765.0" AnchorPane.bottomAnchor="0.0" AnchorPane.leftAnchor="0.0" AnchorPane.rightAnchor="0.0" AnchorPane.topAnchor="0.0">
<bottom>
<StatusBar fx:id="statusBar" BorderPane.alignment="CENTER" />
</bottom>
<center>
<SplitPane dividerPositions="0.6177944862155389" BorderPane.alignment="CENTER">
<items>
<TabPane fx:id="stagesPane" prefHeight="200.0" prefWidth="200.0" side="LEFT" tabClosingPolicy="UNAVAILABLE">
<tabs>
<Tab fx:id="logTab" closable="false" text="Log" />
</tabs>
</TabPane>
<ScrollPane fitToHeight="true" fitToWidth="true">
<content>
<Accordion fx:id="metaContainer" prefHeight="65536.0" prefWidth="248.0" />
</content>
</ScrollPane>
</items>
</SplitPane>
</center>
<top>
<ToolBar prefHeight="40.0" prefWidth="200.0" BorderPane.alignment="CENTER">
<items>
<ToggleButton fx:id="consoleButton" mnemonicParsing="false" text="Console" />
<ToggleButton fx:id="processButton" mnemonicParsing="false" text="Processes" />
<Separator orientation="VERTICAL" />
<Pane HBox.hgrow="ALWAYS" />
<Separator orientation="VERTICAL" />
<Button mnemonicParsing="false" onAction="#onLoadConfigClick" text="Load" />
<ToggleButton fx:id="runButton" disable="true" mnemonicParsing="false" onAction="#onRunButtonClick" text="Run" />
</items>
</ToolBar>
</top>
</BorderPane>
</children>
</AnchorPane>

3
numass-server/src/main/java/inr/numass/server/ServerRunner.java
View File

@ -5,6 +5,7 @@ import hep.dataforge.context.Global;
import hep.dataforge.io.MetaFileReader; import hep.dataforge.io.MetaFileReader;
import hep.dataforge.meta.Meta; import hep.dataforge.meta.Meta;
import hep.dataforge.meta.SimpleConfigurable; import hep.dataforge.meta.SimpleConfigurable;
import hep.dataforge.storage.commons.StorageManager;
import inr.numass.data.storage.NumassStorage; import inr.numass.data.storage.NumassStorage;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
@ -57,7 +58,7 @@ public class ServerRunner extends SimpleConfigurable implements AutoCloseable {
Meta storageMeta = getMeta().getMetaOrEmpty(NUMASS_REPO_ELEMENT); Meta storageMeta = getMeta().getMetaOrEmpty(NUMASS_REPO_ELEMENT);
context.getLogger().info("Initializing file storage with meta: {}",storageMeta); context.getLogger().info("Initializing file storage with meta: {}",storageMeta);
root = new NumassStorage(context,storageMeta); root = (NumassStorage) StorageManager.buildStorage(context,storageMeta);
context.getLogger().info("Starting numass server"); context.getLogger().info("Starting numass server");
if (root != null) { if (root != null) {

4
numass-server/src/main/java/inr/numass/server/TestServer.java
View File

@ -10,8 +10,8 @@ import hep.dataforge.context.Global;
import hep.dataforge.server.ServerManager; import hep.dataforge.server.ServerManager;
import hep.dataforge.server.storage.StorageServerUtils; import hep.dataforge.server.storage.StorageServerUtils;
import hep.dataforge.storage.commons.StorageManager; import hep.dataforge.storage.commons.StorageManager;
import hep.dataforge.storage.filestorage.FileStorageFactory;
import inr.numass.data.storage.NumassStorage; import inr.numass.data.storage.NumassStorage;
import inr.numass.data.storage.NumassStorageFactory;
import java.io.BufferedReader; import java.io.BufferedReader;
import java.io.File; import java.io.File;
@ -36,7 +36,7 @@ public class TestServer {
File path = new File("/D:/temp/test"); File path = new File("/D:/temp/test");
context.getLogger().info("Starting test numass storage servlet in '{}'", path); context.getLogger().info("Starting test numass storage servlet in '{}'", path);
NumassStorage storage = new NumassStorage(context, FileStorageFactory.buildStorageMeta(path, true, true)); NumassStorage storage = (NumassStorage) storageManager.buildStorage(NumassStorageFactory.buildStorageMeta(path.toURI(), true, true));
StorageServerUtils.addStorage(serverManager,storage,"numass-storage"); StorageServerUtils.addStorage(serverManager,storage,"numass-storage");
serverManager.startServer(); serverManager.startServer();

5
numass-viewer/src/main/kotlin/inr/numass/viewer/StorageView.kt
View File

@ -11,12 +11,13 @@ import hep.dataforge.meta.Metoid
import hep.dataforge.storage.api.Loader import hep.dataforge.storage.api.Loader
import hep.dataforge.storage.api.Storage import hep.dataforge.storage.api.Storage
import hep.dataforge.storage.api.TableLoader import hep.dataforge.storage.api.TableLoader
import hep.dataforge.storage.filestorage.FileStorageFactory import hep.dataforge.storage.commons.StorageManager
import inr.numass.NumassProperties import inr.numass.NumassProperties
import inr.numass.data.api.NumassPoint import inr.numass.data.api.NumassPoint
import inr.numass.data.api.NumassSet import inr.numass.data.api.NumassSet
import inr.numass.data.storage.NumassDataLoader import inr.numass.data.storage.NumassDataLoader
import inr.numass.data.storage.NumassStorage import inr.numass.data.storage.NumassStorage
import inr.numass.data.storage.NumassStorageFactory
import javafx.beans.property.SimpleBooleanProperty import javafx.beans.property.SimpleBooleanProperty
import javafx.beans.property.SimpleObjectProperty import javafx.beans.property.SimpleObjectProperty
import javafx.beans.property.SimpleStringProperty import javafx.beans.property.SimpleStringProperty
@ -260,7 +261,7 @@ class StorageView(private val context: Context = Global.instance()) : View(title
title = "Load storage ($path)" title = "Load storage ($path)"
progress = -1.0 progress = -1.0
message = "Building numass storage tree..." message = "Building numass storage tree..."
NumassStorage(context, FileStorageFactory.buildStorageMeta(path, true, true)).also { (StorageManager.buildStorage(context,NumassStorageFactory.buildStorageMeta(path, true, true))as NumassStorage).also {
progress = 1.0 progress = 1.0
} }
} ui { } ui {

4
numass-viewer/src/main/kotlin/inr/numass/viewer/test/ComponentTest.kt
View File

@ -1,6 +1,7 @@
package inr.numass.viewer.test package inr.numass.viewer.test
import hep.dataforge.fx.dfIcon import hep.dataforge.fx.dfIcon
import hep.dataforge.kodex.GLOBAL
import hep.dataforge.tables.Table import hep.dataforge.tables.Table
import inr.numass.data.api.NumassPoint import inr.numass.data.api.NumassPoint
import inr.numass.data.api.NumassSet import inr.numass.data.api.NumassSet
@ -36,7 +37,8 @@ class ViewerComponentsTest : View(title = "Numass viewer test", icon = ImageView
action { action {
runAsync { runAsync {
val rootDir = File("D:\\Work\\Numass\\data\\2017_05\\Fill_2") val rootDir = File("D:\\Work\\Numass\\data\\2017_05\\Fill_2")
val set: NumassSet = NumassStorageFactory.buildLocal(rootDir).provide("loader::set_2", NumassSet::class.java) val set: NumassSet = NumassStorageFactory.buildLocal(GLOBAL, rootDir, true, true)
.provide("loader::set_2", NumassSet::class.java)
.orElseThrow { RuntimeException("err") } .orElseThrow { RuntimeException("err") }
update(set); update(set);
} }