NPM/numass-framework
4
0
Fork 1
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Multiple fixes to storage

Browse Source
This commit is contained in:
Alexander Nozik 2017-12-08 12:40:47 +03:00
parent 35fa048010
commit 038a40dbee
9 changed files with 55 additions and 103 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
numass-core/src/main/java/inr/numass/data/analyzers/AbstractAnalyzer.java
View File

@ -18,10 +18,10 @@ import static inr.numass.data.api.NumassPoint.HV_KEY;
* Created by darksnake on 11.07.2017.
*/
public abstract class AbstractAnalyzer implements NumassAnalyzer {
public static String WINDOW_KEY = "window";
public static String TIME_KEY = "timestamp";
public static final String WINDOW_KEY = "window";
public static final String TIME_KEY = "timestamp";
public static String[] NAME_LIST = {LENGTH_KEY, COUNT_KEY, COUNT_RATE_KEY, COUNT_RATE_ERROR_KEY, WINDOW_KEY, TIME_KEY};
public static final String[] NAME_LIST = {LENGTH_KEY, COUNT_KEY, COUNT_RATE_KEY, COUNT_RATE_ERROR_KEY, WINDOW_KEY, TIME_KEY};
// public static String[] NAME_LIST_WITH_HV = {HV_KEY, LENGTH_KEY, COUNT_KEY, COUNT_RATE_KEY, COUNT_RATE_ERROR_KEY, WINDOW_KEY, TIME_KEY};
@Nullable
private final SignalProcessor processor;

28
numass-core/src/main/java/inr/numass/data/analyzers/SmartAnalyzer.java
View File

@ -1,11 +1,9 @@
package inr.numass.data.analyzers;
import hep.dataforge.description.ValueDef;
import hep.dataforge.meta.Meta;
import hep.dataforge.tables.TableFormat;
import hep.dataforge.tables.ValueMap;
import hep.dataforge.values.Value;
import hep.dataforge.values.ValueType;
import hep.dataforge.values.Values;
import inr.numass.data.api.NumassAnalyzer;
import inr.numass.data.api.NumassBlock;
@ -48,7 +46,7 @@ public class SmartAnalyzer extends AbstractAnalyzer {
throw new IllegalArgumentException("Analyzer not found");
}
} else {
if(config.hasValue("t0")||config.hasValue("t0")){
if(config.hasValue("t0")||config.hasMeta("t0")){
return timeAnalyzer;
} else {
return simpleAnalyzer;
@ -64,35 +62,11 @@ public class SmartAnalyzer extends AbstractAnalyzer {
return new ValueMap(map);
}
private double estimateCountRate(NumassBlock block) {
return (double) block.getEvents().count() / block.getLength().toMillis() * 1000;
}
@Override
public Stream<NumassEvent> getEvents(NumassBlock block, Meta config) {
return getAnalyzer(config).getEvents(block, config);
}
@ValueDef(name = "t0", type = ValueType.NUMBER, info = "Constant t0 cut")
@ValueDef(name = "t0.crFraction", type = ValueType.NUMBER, info = "The relative fraction of events that should be removed by time cut")
@ValueDef(name = "t0.min", type = ValueType.NUMBER, def = "0", info = "Minimal t0")
private int getT0(NumassBlock block, Meta meta) {
if (meta.hasValue("t0")) {
return meta.getInt("t0");
} else if (meta.hasMeta("t0")) {
double fraction = meta.getDouble("t0.crFraction");
double cr = estimateCountRate(block);
if (cr < meta.getDouble("t0.minCR", 0)) {
return 0;
} else {
return (int) Math.max(-1e9 / cr * Math.log(1d - fraction), meta.getDouble("t0.min", 0));
}
} else {
return 0;
}
}
@Override
protected TableFormat getTableFormat(Meta config) {
if (config.hasValue(TimeAnalyzer.T0_KEY) || config.hasMeta(TimeAnalyzer.T0_KEY)) {

32
numass-core/src/main/java/inr/numass/data/analyzers/TimeAnalyzer.java
View File

@ -1,10 +1,12 @@
package inr.numass.data.analyzers;
import hep.dataforge.description.ValueDef;
import hep.dataforge.meta.Meta;
import hep.dataforge.tables.TableFormat;
import hep.dataforge.tables.TableFormatBuilder;
import hep.dataforge.tables.ValueMap;
import hep.dataforge.values.Value;
import hep.dataforge.values.ValueType;
import hep.dataforge.values.Values;
import inr.numass.data.api.NumassBlock;
import inr.numass.data.api.NumassEvent;
@ -29,7 +31,7 @@ import static inr.numass.data.api.NumassPoint.HV_KEY;
public class TimeAnalyzer extends AbstractAnalyzer {
public static String T0_KEY = "t0";
public static String[] NAME_LIST = {LENGTH_KEY, COUNT_KEY, COUNT_RATE_KEY, COUNT_RATE_ERROR_KEY, WINDOW_KEY, TIME_KEY, T0_KEY};
public static final String[] NAME_LIST = {LENGTH_KEY, COUNT_KEY, COUNT_RATE_KEY, COUNT_RATE_ERROR_KEY, WINDOW_KEY, TIME_KEY, T0_KEY};
// public static String[] NAME_LIST_WITH_HV = {HV_KEY, LENGTH_KEY, COUNT_KEY, COUNT_RATE_KEY, COUNT_RATE_ERROR_KEY, WINDOW_KEY, TIME_KEY, T0_KEY};
public TimeAnalyzer(@Nullable SignalProcessor processor) {
@ -81,7 +83,9 @@ public class TimeAnalyzer extends AbstractAnalyzer {
@Override
public Values analyzePoint(NumassPoint point, Meta config) {
//Average count rates, do not sum events
Values res = point.getBlocks().map(it -> analyze(it, config)).reduce(null, this::combineBlockResults);
Values res = point.getBlocks()
.map(it -> analyze(it, config))
.reduce(null, this::combineBlockResults);
Map<String, Value> map = new HashMap<>(res.asMap());
map.put(HV_KEY, Value.of(point.getVoltage()));
@ -127,8 +131,28 @@ public class TimeAnalyzer extends AbstractAnalyzer {
);
}
private long getT0(NumassBlock block, Meta config) {
return config.getValue("t0", 0).longValue();
@ValueDef(name = "t0", type = ValueType.NUMBER, info = "Constant t0 cut")
@ValueDef(name = "t0.crFraction", type = ValueType.NUMBER, info = "The relative fraction of events that should be removed by time cut")
@ValueDef(name = "t0.min", type = ValueType.NUMBER, def = "0", info = "Minimal t0")
private int getT0(NumassBlock block, Meta meta) {
if (meta.hasValue("t0")) {
return meta.getInt("t0");
} else if (meta.hasMeta("t0")) {
double fraction = meta.getDouble("t0.crFraction");
double cr = estimateCountRate(block);
if (cr < meta.getDouble("t0.minCR", 0)) {
return 0;
} else {
return (int) Math.max(-1e9 / cr * Math.log(1d - fraction), meta.getDouble("t0.min", 0));
}
} else {
return 0;
}
}
private double estimateCountRate(NumassBlock block) {
return (double) block.getEvents().count() / block.getLength().toMillis() * 1000;
}
/**

21
numass-core/src/main/java/inr/numass/data/storage/NumassStorageFactory.java
View File

@ -2,11 +2,12 @@ package inr.numass.data.storage;
import com.github.robtimus.filesystems.sftp.SFTPEnvironment;
import hep.dataforge.context.Context;
import hep.dataforge.context.Global;
import hep.dataforge.meta.Meta;
import hep.dataforge.meta.MetaBuilder;
import hep.dataforge.storage.api.Storage;
import hep.dataforge.storage.api.StorageType;
import hep.dataforge.storage.commons.StorageManager;
import hep.dataforge.storage.filestorage.FileStorage;
import org.jetbrains.annotations.NotNull;
import java.io.File;
@ -28,13 +29,9 @@ public class NumassStorageFactory implements StorageType {
* @return
*/
@NotNull
public static NumassStorage buildLocal(File file, boolean monitor) {
Path path = file.toPath();
Meta meta = new MetaBuilder("storage")
.setValue("path", path)
.setValue("monitor", monitor);
return new NumassStorage(Global.instance(), meta, path);
public static FileStorage buildLocal(Context context, File file, boolean readOnly, boolean monitor) {
StorageManager manager = context.loadFeature("hep.dataforge:storage", StorageManager.class);
return (FileStorage) manager.buildStorage(buildStorageMeta(file.toURI(),readOnly,monitor));
}
@Override
@ -70,4 +67,12 @@ public class NumassStorageFactory implements StorageType {
return new NumassStorage(context, meta, context.getIo().getWorkDirectory());
}
}
public static MetaBuilder buildStorageMeta(URI path, boolean readOnly, boolean monitor) {
return new MetaBuilder("storage")
.setValue("path", path.toString())
.setValue("type", "numass")
.setValue("readOnly", readOnly)
.setValue("monitor", monitor);
}
}

55
numass-main/src/main/resources/fxml/NumassWorkbench.fxml
View File

@ -1,55 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<?import javafx.scene.control.Accordion?>
<?import javafx.scene.control.Button?>
<?import javafx.scene.control.ScrollPane?>
<?import javafx.scene.control.Separator?>
<?import javafx.scene.control.SplitPane?>
<?import javafx.scene.control.Tab?>
<?import javafx.scene.control.TabPane?>
<?import javafx.scene.control.ToggleButton?>
<?import javafx.scene.control.ToolBar?>
<?import javafx.scene.layout.AnchorPane?>
<?import javafx.scene.layout.BorderPane?>
<?import javafx.scene.layout.HBox?>
<?import javafx.scene.layout.Pane?>
<?import org.controlsfx.control.StatusBar?>
<AnchorPane id="AnchorPane" prefHeight="600.0" prefWidth="800.0" xmlns="http://javafx.com/javafx/8.0.60" xmlns:fx="http://javafx.com/fxml/1" fx:controller="inr.numass.workbench.NumassWorkbenchController">
<children>
<BorderPane layoutX="-165.0" layoutY="100.0" prefHeight="200.0" prefWidth="765.0" AnchorPane.bottomAnchor="0.0" AnchorPane.leftAnchor="0.0" AnchorPane.rightAnchor="0.0" AnchorPane.topAnchor="0.0">
<bottom>
<StatusBar fx:id="statusBar" BorderPane.alignment="CENTER" />
</bottom>
<center>
<SplitPane dividerPositions="0.6177944862155389" BorderPane.alignment="CENTER">
<items>
<TabPane fx:id="stagesPane" prefHeight="200.0" prefWidth="200.0" side="LEFT" tabClosingPolicy="UNAVAILABLE">
<tabs>
<Tab fx:id="logTab" closable="false" text="Log" />
</tabs>
</TabPane>
<ScrollPane fitToHeight="true" fitToWidth="true">
<content>
<Accordion fx:id="metaContainer" prefHeight="65536.0" prefWidth="248.0" />
</content>
</ScrollPane>
</items>
</SplitPane>
</center>
<top>
<ToolBar prefHeight="40.0" prefWidth="200.0" BorderPane.alignment="CENTER">
<items>
<ToggleButton fx:id="consoleButton" mnemonicParsing="false" text="Console" />
<ToggleButton fx:id="processButton" mnemonicParsing="false" text="Processes" />
<Separator orientation="VERTICAL" />
<Pane HBox.hgrow="ALWAYS" />
<Separator orientation="VERTICAL" />
<Button mnemonicParsing="false" onAction="#onLoadConfigClick" text="Load" />
<ToggleButton fx:id="runButton" disable="true" mnemonicParsing="false" onAction="#onRunButtonClick" text="Run" />
</items>
</ToolBar>
</top>
</BorderPane>
</children>
</AnchorPane>

3
numass-server/src/main/java/inr/numass/server/ServerRunner.java
View File

@ -5,6 +5,7 @@ import hep.dataforge.context.Global;
import hep.dataforge.io.MetaFileReader;
import hep.dataforge.meta.Meta;
import hep.dataforge.meta.SimpleConfigurable;
import hep.dataforge.storage.commons.StorageManager;
import inr.numass.data.storage.NumassStorage;
import org.slf4j.LoggerFactory;
@ -57,7 +58,7 @@ public class ServerRunner extends SimpleConfigurable implements AutoCloseable {
Meta storageMeta = getMeta().getMetaOrEmpty(NUMASS_REPO_ELEMENT);
context.getLogger().info("Initializing file storage with meta: {}",storageMeta);
root = new NumassStorage(context,storageMeta);
root = (NumassStorage) StorageManager.buildStorage(context,storageMeta);
context.getLogger().info("Starting numass server");
if (root != null) {

4
numass-server/src/main/java/inr/numass/server/TestServer.java
View File

@ -10,8 +10,8 @@ import hep.dataforge.context.Global;
import hep.dataforge.server.ServerManager;
import hep.dataforge.server.storage.StorageServerUtils;
import hep.dataforge.storage.commons.StorageManager;
import hep.dataforge.storage.filestorage.FileStorageFactory;
import inr.numass.data.storage.NumassStorage;
import inr.numass.data.storage.NumassStorageFactory;
import java.io.BufferedReader;
import java.io.File;
@ -36,7 +36,7 @@ public class TestServer {
File path = new File("/D:/temp/test");
context.getLogger().info("Starting test numass storage servlet in '{}'", path);
NumassStorage storage = new NumassStorage(context, FileStorageFactory.buildStorageMeta(path, true, true));
NumassStorage storage = (NumassStorage) storageManager.buildStorage(NumassStorageFactory.buildStorageMeta(path.toURI(), true, true));
StorageServerUtils.addStorage(serverManager,storage,"numass-storage");
serverManager.startServer();

5
numass-viewer/src/main/kotlin/inr/numass/viewer/StorageView.kt
View File

@ -11,12 +11,13 @@ import hep.dataforge.meta.Metoid
import hep.dataforge.storage.api.Loader
import hep.dataforge.storage.api.Storage
import hep.dataforge.storage.api.TableLoader
import hep.dataforge.storage.filestorage.FileStorageFactory
import hep.dataforge.storage.commons.StorageManager
import inr.numass.NumassProperties
import inr.numass.data.api.NumassPoint
import inr.numass.data.api.NumassSet
import inr.numass.data.storage.NumassDataLoader
import inr.numass.data.storage.NumassStorage
import inr.numass.data.storage.NumassStorageFactory
import javafx.beans.property.SimpleBooleanProperty
import javafx.beans.property.SimpleObjectProperty
import javafx.beans.property.SimpleStringProperty
@ -260,7 +261,7 @@ class StorageView(private val context: Context = Global.instance()) : View(title
title = "Load storage ($path)"
progress = -1.0
message = "Building numass storage tree..."
NumassStorage(context, FileStorageFactory.buildStorageMeta(path, true, true)).also {
(StorageManager.buildStorage(context,NumassStorageFactory.buildStorageMeta(path, true, true))as NumassStorage).also {
progress = 1.0
}
} ui {

4
numass-viewer/src/main/kotlin/inr/numass/viewer/test/ComponentTest.kt
View File

@ -1,6 +1,7 @@
package inr.numass.viewer.test
import hep.dataforge.fx.dfIcon
import hep.dataforge.kodex.GLOBAL
import hep.dataforge.tables.Table
import inr.numass.data.api.NumassPoint
import inr.numass.data.api.NumassSet
@ -36,7 +37,8 @@ class ViewerComponentsTest : View(title = "Numass viewer test", icon = ImageView
action {
runAsync {
val rootDir = File("D:\\Work\\Numass\\data\\2017_05\\Fill_2")
val set: NumassSet = NumassStorageFactory.buildLocal(rootDir).provide("loader::set_2", NumassSet::class.java)
val set: NumassSet = NumassStorageFactory.buildLocal(GLOBAL, rootDir, true, true)
.provide("loader::set_2", NumassSet::class.java)
.orElseThrow { RuntimeException("err") }
update(set);
}