Trying to fix dead lock in KAction. Failed for now.

numass-control/src/main/kotlin/inr/numass/control/NumassControlUtils.kt

@@ -10,7 +10,6 @@ import hep.dataforge.io.MetaFileReader
 import hep.dataforge.io.XMLMetaReader
 import hep.dataforge.meta.Meta
 import hep.dataforge.storage.commons.StorageConnection
-import hep.dataforge.storage.commons.StorageFactory
 import hep.dataforge.storage.commons.StorageManager
 import inr.numass.client.ClientUtils
 import javafx.application.Application

numass-core/build.gradle

@@ -3,7 +3,7 @@ buildscript {
         mavenCentral()
     }
     dependencies {
-        classpath 'com.google.protobuf:protobuf-gradle-plugin:0.8.1'
+        classpath 'com.google.protobuf:protobuf-gradle-plugin:0.8.3'
     }
 }
 
@@ -14,14 +14,17 @@ description = "A bse package with minimal dependencies for numass"
 
 dependencies {
     compile "hep.dataforge:dataforge-storage" //project(':dataforge-storage')
-    compile 'com.google.protobuf:protobuf-java:3.3.0'
+    compile 'com.google.protobuf:protobuf-java:3.5.0'
+
+    // https://mvnrepository.com/artifact/com.github.robtimus/sftp-fs
+    compile group: 'com.github.robtimus', name: 'sftp-fs', version: '1.1.1'
 }
 
 protobuf {
     // Configure the protoc executable
     protoc {
         // Download from repositories
-        artifact = 'com.google.protobuf:protoc:3.3.0'
+        artifact = 'com.google.protobuf:protoc:3.5.0'
     }
     generatedFilesBaseDir = "$projectDir/gen"
 }

numass-core/src/main/java/inr/numass/data/storage/NumassDataFactory.java

@@ -4,6 +4,8 @@ import hep.dataforge.context.Context;
 import hep.dataforge.data.DataFactory;
 import hep.dataforge.data.DataTree;
 import hep.dataforge.meta.Meta;
+import hep.dataforge.storage.api.Storage;
+import hep.dataforge.storage.commons.StorageManager;
 import hep.dataforge.storage.commons.StorageUtils;
 import inr.numass.data.api.NumassSet;
 
@@ -24,7 +26,8 @@ public class NumassDataFactory extends DataFactory<NumassSet> {
 
     @Override
     protected void fill(DataTree.Builder<NumassSet> builder, Context context, Meta meta) {
-        NumassStorage storage = new NumassStorage(context,meta);
+        Meta newMeta = meta.getBuilder().setValue("type", "numass");
+        Storage storage = context.loadFeature("hep.dataforge:storage", StorageManager.class).buildStorage(newMeta);
         StorageUtils.loaderStream(storage).forEach(loader -> {
             if (loader instanceof NumassSet) {
                 builder.putStatic(loader.getFullName().toUnescaped(), (NumassSet) loader);

numass-core/src/main/java/inr/numass/data/storage/NumassStorage.java

@@ -1,4 +1,4 @@
-/* 
+/*
  * Copyright 2015 Alexander Nozik.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
@@ -52,13 +52,13 @@ public class NumassStorage extends FileStorage {
     public static final String NUMASS_ZIP_EXTENSION = ".nm.zip";
     public static final String NUMASS_DATA_LOADER_TYPE = "numassData";
 
-    protected NumassStorage(FileStorage parent, String path, Meta config) throws StorageException {
+    protected NumassStorage(FileStorage parent, Meta config, String shelf) throws StorageException {
-        super(parent, path, config);
+        super(parent, config, shelf);
         super.refresh();
     }
 
-    public NumassStorage(Context context, Meta config) throws StorageException {
+    public NumassStorage(Context context, Meta config, Path path) throws StorageException {
-        super(context, config);
+        super(context, config, path);
         super.refresh();
     }
 
@@ -75,7 +75,7 @@ public class NumassStorage extends FileStorage {
                                     NumassDataLoader.fromDir(this, file, null));
                         } else {
                             this.shelves.put(entryName(file),
-                                    new NumassStorage(this, entryName(file), getMeta()));
+                                    new NumassStorage(this, getMeta(), entryName(file)));
                         }
                     } else if (file.getFileName().endsWith(NUMASS_ZIP_EXTENSION)) {
                         this.loaders.put(entryName(file), NumassDataLoader.fromFile(this, file));
@@ -134,8 +134,8 @@ public class NumassStorage extends FileStorage {
     }
 
     @Override
-    public NumassStorage createShelf(String path, Meta meta) throws StorageException {
+    public NumassStorage createShelf(Meta meta, String path) throws StorageException {
-        return new NumassStorage(this, path, meta);
+        return new NumassStorage(this, meta, path);
     }
 
     /**
@@ -166,6 +166,17 @@ public class NumassStorage extends FileStorage {
         return getMeta().getString("description", "");
     }
 
+    @Override
+    public void close() throws Exception {
+        super.close();
+        //close remote file system after use
+        try {
+            getDataDir().getFileSystem().close();
+        } catch (UnsupportedOperationException ex) {
+
+        }
+    }
+
     public static class NumassDataPointEvent extends Event {
 
         public static final String FILE_NAME_KEY = "fileName";

numass-core/src/main/java/inr/numass/data/storage/NumassStorageFactory.java

@@ -1,35 +1,40 @@
 package inr.numass.data.storage;
 
+import com.github.robtimus.filesystems.sftp.SFTPEnvironment;
 import hep.dataforge.context.Context;
 import hep.dataforge.context.Global;
 import hep.dataforge.meta.Meta;
 import hep.dataforge.meta.MetaBuilder;
 import hep.dataforge.storage.api.Storage;
 import hep.dataforge.storage.api.StorageType;
+import org.jetbrains.annotations.NotNull;
 
 import java.io.File;
+import java.net.URI;
+import java.nio.file.FileSystem;
+import java.nio.file.FileSystems;
+import java.nio.file.Path;
+import java.nio.file.Paths;
 
 /**
  * Created by darksnake on 17-May-17.
  */
 public class NumassStorageFactory implements StorageType {
 
-    public static MetaBuilder buildStorageMeta(String path, boolean readOnly, boolean monitor){
-        return new MetaBuilder("storage")
-                .setValue("path", path)
-                .setValue("type", "numass")
-                .setValue("readOnly", readOnly)
-                .setValue("monitor", monitor);
-    }
-
     /**
      * Build local storage with Global context. Used for tests.
+     *
      * @param file
      * @return
      */
-    public static NumassStorage buildLocal(File file) {
+    @NotNull
-        return new NumassStorage(Global.instance(),
+    public static NumassStorage buildLocal(File file, boolean monitor) {
-                new MetaBuilder("storage").setValue("path", file.toPath()));
+        Path path = file.toPath();
+        Meta meta = new MetaBuilder("storage")
+                .setValue("path", path)
+                .setValue("monitor", monitor);
+
+        return new NumassStorage(Global.instance(), meta, path);
     }
 
     @Override
@@ -37,8 +42,32 @@ public class NumassStorageFactory implements StorageType {
         return "numass";
     }
 
+    @NotNull
     @Override
     public Storage build(Context context, Meta meta) {
-        return new NumassStorage(context, meta);
+        if (meta.hasValue("path")) {
+            URI uri = URI.create(meta.getString("path"));
+            Path path;
+            if (uri.getScheme().startsWith("ssh")) {
+                try {
+                    String username = meta.getString("userName", uri.getUserInfo());
+                    //String host = meta.getString("host", uri.getHost());
+                    int port = meta.getInt("port", 22);
+                    SFTPEnvironment env = new SFTPEnvironment()
+                            .withUsername(username)
+                            .withPassword(meta.getString("password","").toCharArray());
+                    FileSystem fs = FileSystems.newFileSystem(uri, env,context.getClassLoader());
+                    path = fs.getPath(uri.getPath());
+                } catch (Exception e) {
+                    throw new RuntimeException(e);
+                }
+            } else {
+                path = Paths.get(uri);
+            }
+            return new NumassStorage(context, meta, path);
+        } else {
+            context.getLogger().warn("A storage path not provided. Creating default root storage in the working directory");
+            return new NumassStorage(context, meta, context.getIo().getWorkDirectory());
+        }
     }
 }

numass-main/src/main/groovy/inr/numass/scripts/temp/RemoteStorage.groovy

@@ -0,0 +1,17 @@
+package inr.numass.scripts.temp
+
+import hep.dataforge.context.Context
+import hep.dataforge.grind.Grind
+import hep.dataforge.grind.GrindShell
+import hep.dataforge.storage.api.Storage
+import hep.dataforge.storage.commons.StorageManager
+
+new GrindShell().eval {
+    def ctx = context as Context;
+    //(LoggerFactory.getLogger(Logger.ROOT_LOGGER_NAME) as Logger).setLevel(Level.INFO)
+
+    def storageMeta = Grind.buildMeta(type: "numass", path: "sftp://192.168.111.1/home/trdat/data/2017_11", userName: "trdat", password: "Anomaly")
+
+    Storage storage = ctx.loadFeature("hep.dataforge:storage", StorageManager).buildStorage(storageMeta);
+
+}

numass-main/src/main/kotlin/inr/numass/tasks/NumassTasks.kt

@@ -103,7 +103,7 @@ val analyzeTask = task("analyze") {
     }
     pipe<NumassSet, Table> { set ->
         SmartAnalyzer().analyzeSet(set, meta).also { res ->
-            context.getIo().out("numass.analyze", name).use {
+            context.io.out("numass.analyze", name).use {
                 NumassUtils.write(it, meta, res)
             }
         }