From 2556cf36f816a15d5db9b34408fc66f5c874eb5b Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Thu, 7 Dec 2017 17:03:42 +0300 Subject: [PATCH] Trying to fix dead lock in KAction. Failed for now. --- .../inr/numass/control/NumassControlUtils.kt | 1 - numass-core/build.gradle | 9 ++-- .../data/storage/NumassDataFactory.java | 5 +- .../numass/data/storage/NumassStorage.java | 27 +++++++--- .../data/storage/NumassStorageFactory.java | 53 ++++++++++++++----- .../numass/scripts/temp/RemoteStorage.groovy | 17 ++++++ .../kotlin/inr/numass/tasks/NumassTasks.kt | 2 +- 7 files changed, 88 insertions(+), 26 deletions(-) create mode 100644 numass-main/src/main/groovy/inr/numass/scripts/temp/RemoteStorage.groovy diff --git a/numass-control/src/main/kotlin/inr/numass/control/NumassControlUtils.kt b/numass-control/src/main/kotlin/inr/numass/control/NumassControlUtils.kt index 8e7d6809..6fa95642 100644 --- a/numass-control/src/main/kotlin/inr/numass/control/NumassControlUtils.kt +++ b/numass-control/src/main/kotlin/inr/numass/control/NumassControlUtils.kt @@ -10,7 +10,6 @@ import hep.dataforge.io.MetaFileReader import hep.dataforge.io.XMLMetaReader import hep.dataforge.meta.Meta import hep.dataforge.storage.commons.StorageConnection -import hep.dataforge.storage.commons.StorageFactory import hep.dataforge.storage.commons.StorageManager import inr.numass.client.ClientUtils import javafx.application.Application diff --git a/numass-core/build.gradle b/numass-core/build.gradle index 173e3df8..1f3200bc 100644 --- a/numass-core/build.gradle +++ b/numass-core/build.gradle @@ -3,7 +3,7 @@ buildscript { mavenCentral() } dependencies { - classpath 'com.google.protobuf:protobuf-gradle-plugin:0.8.1' + classpath 'com.google.protobuf:protobuf-gradle-plugin:0.8.3' } } @@ -14,14 +14,17 @@ description = "A bse package with minimal dependencies for numass" dependencies { compile "hep.dataforge:dataforge-storage" //project(':dataforge-storage') - compile 'com.google.protobuf:protobuf-java:3.3.0' + compile 'com.google.protobuf:protobuf-java:3.5.0' + + // https://mvnrepository.com/artifact/com.github.robtimus/sftp-fs + compile group: 'com.github.robtimus', name: 'sftp-fs', version: '1.1.1' } protobuf { // Configure the protoc executable protoc { // Download from repositories - artifact = 'com.google.protobuf:protoc:3.3.0' + artifact = 'com.google.protobuf:protoc:3.5.0' } generatedFilesBaseDir = "$projectDir/gen" } diff --git a/numass-core/src/main/java/inr/numass/data/storage/NumassDataFactory.java b/numass-core/src/main/java/inr/numass/data/storage/NumassDataFactory.java index bf2b5f70..9e792b40 100644 --- a/numass-core/src/main/java/inr/numass/data/storage/NumassDataFactory.java +++ b/numass-core/src/main/java/inr/numass/data/storage/NumassDataFactory.java @@ -4,6 +4,8 @@ import hep.dataforge.context.Context; import hep.dataforge.data.DataFactory; import hep.dataforge.data.DataTree; import hep.dataforge.meta.Meta; +import hep.dataforge.storage.api.Storage; +import hep.dataforge.storage.commons.StorageManager; import hep.dataforge.storage.commons.StorageUtils; import inr.numass.data.api.NumassSet; @@ -24,7 +26,8 @@ public class NumassDataFactory extends DataFactory { @Override protected void fill(DataTree.Builder builder, Context context, Meta meta) { - NumassStorage storage = new NumassStorage(context,meta); + Meta newMeta = meta.getBuilder().setValue("type", "numass"); + Storage storage = context.loadFeature("hep.dataforge:storage", StorageManager.class).buildStorage(newMeta); StorageUtils.loaderStream(storage).forEach(loader -> { if (loader instanceof NumassSet) { builder.putStatic(loader.getFullName().toUnescaped(), (NumassSet) loader); diff --git a/numass-core/src/main/java/inr/numass/data/storage/NumassStorage.java b/numass-core/src/main/java/inr/numass/data/storage/NumassStorage.java index fab5924c..6f8c7b95 100644 --- a/numass-core/src/main/java/inr/numass/data/storage/NumassStorage.java +++ b/numass-core/src/main/java/inr/numass/data/storage/NumassStorage.java @@ -1,4 +1,4 @@ -/* +/* * Copyright 2015 Alexander Nozik. * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -52,13 +52,13 @@ public class NumassStorage extends FileStorage { public static final String NUMASS_ZIP_EXTENSION = ".nm.zip"; public static final String NUMASS_DATA_LOADER_TYPE = "numassData"; - protected NumassStorage(FileStorage parent, String path, Meta config) throws StorageException { - super(parent, path, config); + protected NumassStorage(FileStorage parent, Meta config, String shelf) throws StorageException { + super(parent, config, shelf); super.refresh(); } - public NumassStorage(Context context, Meta config) throws StorageException { - super(context, config); + public NumassStorage(Context context, Meta config, Path path) throws StorageException { + super(context, config, path); super.refresh(); } @@ -75,7 +75,7 @@ public class NumassStorage extends FileStorage { NumassDataLoader.fromDir(this, file, null)); } else { this.shelves.put(entryName(file), - new NumassStorage(this, entryName(file), getMeta())); + new NumassStorage(this, getMeta(), entryName(file))); } } else if (file.getFileName().endsWith(NUMASS_ZIP_EXTENSION)) { this.loaders.put(entryName(file), NumassDataLoader.fromFile(this, file)); @@ -134,8 +134,8 @@ public class NumassStorage extends FileStorage { } @Override - public NumassStorage createShelf(String path, Meta meta) throws StorageException { - return new NumassStorage(this, path, meta); + public NumassStorage createShelf(Meta meta, String path) throws StorageException { + return new NumassStorage(this, meta, path); } /** @@ -166,6 +166,17 @@ public class NumassStorage extends FileStorage { return getMeta().getString("description", ""); } + @Override + public void close() throws Exception { + super.close(); + //close remote file system after use + try { + getDataDir().getFileSystem().close(); + } catch (UnsupportedOperationException ex) { + + } + } + public static class NumassDataPointEvent extends Event { public static final String FILE_NAME_KEY = "fileName"; diff --git a/numass-core/src/main/java/inr/numass/data/storage/NumassStorageFactory.java b/numass-core/src/main/java/inr/numass/data/storage/NumassStorageFactory.java index ed1ad312..bc6a7505 100644 --- a/numass-core/src/main/java/inr/numass/data/storage/NumassStorageFactory.java +++ b/numass-core/src/main/java/inr/numass/data/storage/NumassStorageFactory.java @@ -1,35 +1,40 @@ package inr.numass.data.storage; +import com.github.robtimus.filesystems.sftp.SFTPEnvironment; import hep.dataforge.context.Context; import hep.dataforge.context.Global; import hep.dataforge.meta.Meta; import hep.dataforge.meta.MetaBuilder; import hep.dataforge.storage.api.Storage; import hep.dataforge.storage.api.StorageType; +import org.jetbrains.annotations.NotNull; import java.io.File; +import java.net.URI; +import java.nio.file.FileSystem; +import java.nio.file.FileSystems; +import java.nio.file.Path; +import java.nio.file.Paths; /** * Created by darksnake on 17-May-17. */ public class NumassStorageFactory implements StorageType { - public static MetaBuilder buildStorageMeta(String path, boolean readOnly, boolean monitor){ - return new MetaBuilder("storage") - .setValue("path", path) - .setValue("type", "numass") - .setValue("readOnly", readOnly) - .setValue("monitor", monitor); - } - /** * Build local storage with Global context. Used for tests. + * * @param file * @return */ - public static NumassStorage buildLocal(File file) { - return new NumassStorage(Global.instance(), - new MetaBuilder("storage").setValue("path", file.toPath())); + @NotNull + public static NumassStorage buildLocal(File file, boolean monitor) { + Path path = file.toPath(); + Meta meta = new MetaBuilder("storage") + .setValue("path", path) + .setValue("monitor", monitor); + + return new NumassStorage(Global.instance(), meta, path); } @Override @@ -37,8 +42,32 @@ public class NumassStorageFactory implements StorageType { return "numass"; } + @NotNull @Override public Storage build(Context context, Meta meta) { - return new NumassStorage(context, meta); + if (meta.hasValue("path")) { + URI uri = URI.create(meta.getString("path")); + Path path; + if (uri.getScheme().startsWith("ssh")) { + try { + String username = meta.getString("userName", uri.getUserInfo()); + //String host = meta.getString("host", uri.getHost()); + int port = meta.getInt("port", 22); + SFTPEnvironment env = new SFTPEnvironment() + .withUsername(username) + .withPassword(meta.getString("password","").toCharArray()); + FileSystem fs = FileSystems.newFileSystem(uri, env,context.getClassLoader()); + path = fs.getPath(uri.getPath()); + } catch (Exception e) { + throw new RuntimeException(e); + } + } else { + path = Paths.get(uri); + } + return new NumassStorage(context, meta, path); + } else { + context.getLogger().warn("A storage path not provided. Creating default root storage in the working directory"); + return new NumassStorage(context, meta, context.getIo().getWorkDirectory()); + } } } diff --git a/numass-main/src/main/groovy/inr/numass/scripts/temp/RemoteStorage.groovy b/numass-main/src/main/groovy/inr/numass/scripts/temp/RemoteStorage.groovy new file mode 100644 index 00000000..f532e3cb --- /dev/null +++ b/numass-main/src/main/groovy/inr/numass/scripts/temp/RemoteStorage.groovy @@ -0,0 +1,17 @@ +package inr.numass.scripts.temp + +import hep.dataforge.context.Context +import hep.dataforge.grind.Grind +import hep.dataforge.grind.GrindShell +import hep.dataforge.storage.api.Storage +import hep.dataforge.storage.commons.StorageManager + +new GrindShell().eval { + def ctx = context as Context; + //(LoggerFactory.getLogger(Logger.ROOT_LOGGER_NAME) as Logger).setLevel(Level.INFO) + + def storageMeta = Grind.buildMeta(type: "numass", path: "sftp://192.168.111.1/home/trdat/data/2017_11", userName: "trdat", password: "Anomaly") + + Storage storage = ctx.loadFeature("hep.dataforge:storage", StorageManager).buildStorage(storageMeta); + +} \ No newline at end of file diff --git a/numass-main/src/main/kotlin/inr/numass/tasks/NumassTasks.kt b/numass-main/src/main/kotlin/inr/numass/tasks/NumassTasks.kt index 656cc029..915a74fd 100644 --- a/numass-main/src/main/kotlin/inr/numass/tasks/NumassTasks.kt +++ b/numass-main/src/main/kotlin/inr/numass/tasks/NumassTasks.kt @@ -103,7 +103,7 @@ val analyzeTask = task("analyze") { } pipe { set -> SmartAnalyzer().analyzeSet(set, meta).also { res -> - context.getIo().out("numass.analyze", name).use { + context.io.out("numass.analyze", name).use { NumassUtils.write(it, meta, res) } }