NPM/numass-framework
4
0
Fork 1
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Trying to fix dead lock in KAction. Failed for now.

Browse Source
This commit is contained in:
Alexander Nozik 2017-12-07 17:03:42 +03:00
parent d9bb3ada0a
commit 2556cf36f8
7 changed files with 88 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
numass-control/src/main/kotlin/inr/numass/control/NumassControlUtils.kt
View File

@ -10,7 +10,6 @@ import hep.dataforge.io.MetaFileReader
import hep.dataforge.io.XMLMetaReader
import hep.dataforge.meta.Meta
import hep.dataforge.storage.commons.StorageConnection
import hep.dataforge.storage.commons.StorageFactory
import hep.dataforge.storage.commons.StorageManager
import inr.numass.client.ClientUtils
import javafx.application.Application

9
numass-core/build.gradle
View File

@ -3,7 +3,7 @@ buildscript {
mavenCentral()
}
dependencies {
classpath 'com.google.protobuf:protobuf-gradle-plugin:0.8.1'
classpath 'com.google.protobuf:protobuf-gradle-plugin:0.8.3'
}
}
@ -14,14 +14,17 @@ description = "A bse package with minimal dependencies for numass"
dependencies {
compile "hep.dataforge:dataforge-storage" //project(':dataforge-storage')
compile 'com.google.protobuf:protobuf-java:3.3.0'
compile 'com.google.protobuf:protobuf-java:3.5.0'
// https://mvnrepository.com/artifact/com.github.robtimus/sftp-fs
compile group: 'com.github.robtimus', name: 'sftp-fs', version: '1.1.1'
}
protobuf {
// Configure the protoc executable
protoc {
// Download from repositories
artifact = 'com.google.protobuf:protoc:3.3.0'
artifact = 'com.google.protobuf:protoc:3.5.0'
}
generatedFilesBaseDir = "$projectDir/gen"
}

5
numass-core/src/main/java/inr/numass/data/storage/NumassDataFactory.java
View File

@ -4,6 +4,8 @@ import hep.dataforge.context.Context;
import hep.dataforge.data.DataFactory;
import hep.dataforge.data.DataTree;
import hep.dataforge.meta.Meta;
import hep.dataforge.storage.api.Storage;
import hep.dataforge.storage.commons.StorageManager;
import hep.dataforge.storage.commons.StorageUtils;
import inr.numass.data.api.NumassSet;
@ -24,7 +26,8 @@ public class NumassDataFactory extends DataFactory<NumassSet> {
@Override
protected void fill(DataTree.Builder<NumassSet> builder, Context context, Meta meta) {
NumassStorage storage = new NumassStorage(context,meta);
Meta newMeta = meta.getBuilder().setValue("type", "numass");
Storage storage = context.loadFeature("hep.dataforge:storage", StorageManager.class).buildStorage(newMeta);
StorageUtils.loaderStream(storage).forEach(loader -> {
if (loader instanceof NumassSet) {
builder.putStatic(loader.getFullName().toUnescaped(), (NumassSet) loader);

25
numass-core/src/main/java/inr/numass/data/storage/NumassStorage.java
View File

@ -52,13 +52,13 @@ public class NumassStorage extends FileStorage {
public static final String NUMASS_ZIP_EXTENSION = ".nm.zip";
public static final String NUMASS_DATA_LOADER_TYPE = "numassData";
protected NumassStorage(FileStorage parent, String path, Meta config) throws StorageException {
super(parent, path, config);
protected NumassStorage(FileStorage parent, Meta config, String shelf) throws StorageException {
super(parent, config, shelf);
super.refresh();
}
public NumassStorage(Context context, Meta config) throws StorageException {
super(context, config);
public NumassStorage(Context context, Meta config, Path path) throws StorageException {
super(context, config, path);
super.refresh();
}
@ -75,7 +75,7 @@ public class NumassStorage extends FileStorage {
NumassDataLoader.fromDir(this, file, null));
} else {
this.shelves.put(entryName(file),
new NumassStorage(this, entryName(file), getMeta()));
new NumassStorage(this, getMeta(), entryName(file)));
}
} else if (file.getFileName().endsWith(NUMASS_ZIP_EXTENSION)) {
this.loaders.put(entryName(file), NumassDataLoader.fromFile(this, file));
@ -134,8 +134,8 @@ public class NumassStorage extends FileStorage {
}
@Override
public NumassStorage createShelf(String path, Meta meta) throws StorageException {
return new NumassStorage(this, path, meta);
public NumassStorage createShelf(Meta meta, String path) throws StorageException {
return new NumassStorage(this, meta, path);
}
/**
@ -166,6 +166,17 @@ public class NumassStorage extends FileStorage {
return getMeta().getString("description", "");
}
@Override
public void close() throws Exception {
super.close();
//close remote file system after use
try {
getDataDir().getFileSystem().close();
} catch (UnsupportedOperationException ex) {
}
}
public static class NumassDataPointEvent extends Event {
public static final String FILE_NAME_KEY = "fileName";

53
numass-core/src/main/java/inr/numass/data/storage/NumassStorageFactory.java
View File

@ -1,35 +1,40 @@
package inr.numass.data.storage;
import com.github.robtimus.filesystems.sftp.SFTPEnvironment;
import hep.dataforge.context.Context;
import hep.dataforge.context.Global;
import hep.dataforge.meta.Meta;
import hep.dataforge.meta.MetaBuilder;
import hep.dataforge.storage.api.Storage;
import hep.dataforge.storage.api.StorageType;
import org.jetbrains.annotations.NotNull;
import java.io.File;
import java.net.URI;
import java.nio.file.FileSystem;
import java.nio.file.FileSystems;
import java.nio.file.Path;
import java.nio.file.Paths;
/**
* Created by darksnake on 17-May-17.
*/
public class NumassStorageFactory implements StorageType {
public static MetaBuilder buildStorageMeta(String path, boolean readOnly, boolean monitor){
return new MetaBuilder("storage")
.setValue("path", path)
.setValue("type", "numass")
.setValue("readOnly", readOnly)
.setValue("monitor", monitor);
}
/**
* Build local storage with Global context. Used for tests.
*
* @param file
* @return
*/
public static NumassStorage buildLocal(File file) {
return new NumassStorage(Global.instance(),
new MetaBuilder("storage").setValue("path", file.toPath()));
@NotNull
public static NumassStorage buildLocal(File file, boolean monitor) {
Path path = file.toPath();
Meta meta = new MetaBuilder("storage")
.setValue("path", path)
.setValue("monitor", monitor);
return new NumassStorage(Global.instance(), meta, path);
}
@Override
@ -37,8 +42,32 @@ public class NumassStorageFactory implements StorageType {
return "numass";
}
@NotNull
@Override
public Storage build(Context context, Meta meta) {
return new NumassStorage(context, meta);
if (meta.hasValue("path")) {
URI uri = URI.create(meta.getString("path"));
Path path;
if (uri.getScheme().startsWith("ssh")) {
try {
String username = meta.getString("userName", uri.getUserInfo());
//String host = meta.getString("host", uri.getHost());
int port = meta.getInt("port", 22);
SFTPEnvironment env = new SFTPEnvironment()
.withUsername(username)
.withPassword(meta.getString("password","").toCharArray());
FileSystem fs = FileSystems.newFileSystem(uri, env,context.getClassLoader());
path = fs.getPath(uri.getPath());
} catch (Exception e) {
throw new RuntimeException(e);
}
} else {
path = Paths.get(uri);
}
return new NumassStorage(context, meta, path);
} else {
context.getLogger().warn("A storage path not provided. Creating default root storage in the working directory");
return new NumassStorage(context, meta, context.getIo().getWorkDirectory());
}
}
}

17
numass-main/src/main/groovy/inr/numass/scripts/temp/RemoteStorage.groovy Normal file
View File

@ -0,0 +1,17 @@
package inr.numass.scripts.temp
import hep.dataforge.context.Context
import hep.dataforge.grind.Grind
import hep.dataforge.grind.GrindShell
import hep.dataforge.storage.api.Storage
import hep.dataforge.storage.commons.StorageManager
new GrindShell().eval {
def ctx = context as Context;
//(LoggerFactory.getLogger(Logger.ROOT_LOGGER_NAME) as Logger).setLevel(Level.INFO)
def storageMeta = Grind.buildMeta(type: "numass", path: "sftp://192.168.111.1/home/trdat/data/2017_11", userName: "trdat", password: "Anomaly")
Storage storage = ctx.loadFeature("hep.dataforge:storage", StorageManager).buildStorage(storageMeta);
}

2
numass-main/src/main/kotlin/inr/numass/tasks/NumassTasks.kt
View File

@ -103,7 +103,7 @@ val analyzeTask = task("analyze") {
}
pipe<NumassSet, Table> { set ->
SmartAnalyzer().analyzeSet(set, meta).also { res ->
context.getIo().out("numass.analyze", name).use {
context.io.out("numass.analyze", name).use {
NumassUtils.write(it, meta, res)
}
}