Moved everything to storage2. A lot of fixes in plots. Storage1 deprecated.

numass-core/build.gradle

@@ -14,6 +14,7 @@ description = "A bse package with minimal dependencies for numass"
 
 dependencies {
     compile "hep.dataforge:dataforge-storage2"
+    compile "hep.dataforge:dataforge-json"
     compile 'com.google.protobuf:protobuf-java:3.5.0'
 
     // https://mvnrepository.com/artifact/com.github.robtimus/sftp-fs

numass-core/src/main/kotlin/inr/numass/NumassEnvelopeType.kt

@@ -3,11 +3,12 @@ package inr.numass
 import hep.dataforge.io.envelopes.*
 import hep.dataforge.values.Value
 import hep.dataforge.values.parseValue
-import inr.numass.data.legacy.NumassFileEnvelope.Companion.LEGACY_END_SEQUENCE
-import inr.numass.data.legacy.NumassFileEnvelope.Companion.LEGACY_START_SEQUENCE
 import org.slf4j.LoggerFactory
 import java.io.IOException
 import java.nio.ByteBuffer
+import java.nio.file.Files
+import java.nio.file.Path
+import java.nio.file.StandardOpenOption
 import java.util.*
 
 /**
@@ -21,10 +22,16 @@ class NumassEnvelopeType : EnvelopeType {
 
     override fun description(): String = "Numass legacy envelope"
 
+    /**
+     * Read as legacy
+     */
     override fun getReader(properties: Map<String, String>): EnvelopeReader {
         return NumassEnvelopeReader()
     }
 
+    /**
+     * Write as default
+     */
     override fun getWriter(properties: Map<String, String>): EnvelopeWriter {
         return DefaultEnvelopeWriter(this, MetaType.resolve(properties))
     }
@@ -80,4 +87,39 @@ class NumassEnvelopeType : EnvelopeType {
         }
     }
 
+    companion object {
+        val INSTANCE = NumassEnvelopeType()
+
+        val LEGACY_START_SEQUENCE = byteArrayOf('#'.toByte(), '!'.toByte())
+        val LEGACY_END_SEQUENCE = byteArrayOf('!'.toByte(), '#'.toByte(), '\r'.toByte(), '\n'.toByte())
+
+        /**
+         * Replacement for standard type infer to include legacy type
+         *
+         * @param path
+         * @return
+         */
+        fun infer(path: Path): EnvelopeType? {
+            return try {
+                Files.newInputStream(path, StandardOpenOption.READ).use {
+                    val buffer = ByteArray(6)
+                    it.read(buffer)
+                    val header = String(buffer)
+                    when {
+                        //TODO use templates from appropriate types
+                        header.startsWith("#!") -> NumassEnvelopeType.INSTANCE
+                        header.startsWith("#~DFTL") -> TaglessEnvelopeType.INSTANCE
+                        header.startsWith("#~") -> DefaultEnvelopeType.INSTANCE
+                        else -> null
+                    }
+                }
+            } catch (ex: Exception) {
+                LoggerFactory.getLogger(EnvelopeType::class.java).warn("Could not infer envelope type of file {} due to exception: {}", path, ex)
+                null
+            }
+
+        }
+
+    }
+
 }

numass-core/src/main/kotlin/inr/numass/data/legacy/NumassFileEnvelope.kt

@@ -1,43 +1,70 @@
 package inr.numass.data.legacy
 
-import hep.dataforge.io.envelopes.EnvelopeTag
+import hep.dataforge.meta.Meta
 import hep.dataforge.storage.files.FileEnvelope
 import inr.numass.NumassEnvelopeType
-import java.io.IOException
 import java.nio.ByteBuffer
-import java.nio.channels.FileChannel
+import java.nio.file.Files
 import java.nio.file.Path
-import java.nio.file.StandardOpenOption.READ
+import java.nio.file.StandardOpenOption
 
-class NumassFileEnvelope private constructor(path: Path, readOnly: Boolean) : FileEnvelope(path, readOnly) {
+class NumassFileEnvelope(path: Path) : FileEnvelope(path) {
 
-    override fun buildTag(): EnvelopeTag {
+    private val tag by lazy { Files.newByteChannel(path, StandardOpenOption.READ).use { NumassEnvelopeType.LegacyTag().read(it) } }
-        return NumassEnvelopeType.LegacyTag()
+
+    override val dataOffset: Long by lazy { (tag.length + tag.metaSize).toLong() }
+
+    override var dataLength: Int
+        get() = tag.dataSize
+        set(value) {
+            if (value > Int.MAX_VALUE) {
+                throw RuntimeException("Too large data block")
+            }
+            tag.dataSize = value
+            if (channel.write(tag.toBytes(), 0L) < tag.length) {
+                throw error("Tag is not overwritten.")
+            }
         }
 
-    companion object {
 
-        val LEGACY_START_SEQUENCE = byteArrayOf('#'.toByte(), '!'.toByte())
+    override val meta: Meta by lazy {
-        val LEGACY_END_SEQUENCE = byteArrayOf('!'.toByte(), '#'.toByte(), '\r'.toByte(), '\n'.toByte())
+        val buffer = ByteBuffer.allocate(tag.metaSize).also {
+            channel.read(it, tag.length.toLong())
+        }
+        tag.metaType.reader.readBuffer(buffer)
+    }
 
-        fun open(path: Path, readOnly: Boolean): FileEnvelope {
+
-            //        if (!Files.exists(path)) {
+}
-            //            throw new RuntimeException("File envelope does not exist");
+
+//
+//    override fun buildTag(): EnvelopeTag {
+//        return NumassEnvelopeType.LegacyTag()
+//    }
+//
+//    companion object {
+//
+//        val LEGACY_START_SEQUENCE = byteArrayOf('#'.toByte(), '!'.toByte())
+//        val LEGACY_END_SEQUENCE = byteArrayOf('!'.toByte(), '#'.toByte(), '\r'.toByte(), '\n'.toByte())
+//
+//        fun open(path: Path, readOnly: Boolean): FileEnvelope {
+//            //        if (!Files.exists(path)) {
+//            //            throw new RuntimeException("File envelope does not exist");
+//            //        }
+//
+//            try {
+//                FileChannel.open(path, READ).use { channel ->
+//                    val buffer = channel.map(FileChannel.MapMode.READ_ONLY, 0, 2)
+//                    return if (buffer.compareTo(ByteBuffer.wrap(LEGACY_START_SEQUENCE)) == 0) {
+//                        NumassFileEnvelope(path, readOnly)
+//                    } else {
+//                        FileEnvelope.open(path, readOnly)
+//                    }
+//                }
+//            } catch (e: IOException) {
+//                throw RuntimeException("Failed to open file envelope", e)
+//            }
+//
+//        }
 //    }
 
-            try {
-                FileChannel.open(path, READ).use { channel ->
-                    val buffer = channel.map(FileChannel.MapMode.READ_ONLY, 0, 2)
-                    return if (buffer.compareTo(ByteBuffer.wrap(LEGACY_START_SEQUENCE)) == 0) {
-                        NumassFileEnvelope(path, readOnly)
-                    } else {
-                        FileEnvelope.open(path, readOnly)
-                    }
-                }
-            } catch (e: IOException) {
-                throw RuntimeException("Failed to open file envelope", e)
-            }
-
-        }
-    }
-}

numass-core/src/main/kotlin/inr/numass/data/storage/ClassicNumassPoint.kt

@@ -104,7 +104,7 @@ class ClassicNumassPoint(private val envelope: Envelope) : NumassPoint {
 
     companion object {
         fun readFile(path: Path): ClassicNumassPoint {
-            return ClassicNumassPoint(NumassFileEnvelope.open(path, true))
+            return ClassicNumassPoint(NumassFileEnvelope(path))
         }
     }
 }

numass-core/src/main/kotlin/inr/numass/data/storage/NumassDataFactory.kt

@@ -4,8 +4,13 @@ import hep.dataforge.context.Context
 import hep.dataforge.data.DataFactory
 import hep.dataforge.data.DataNodeEditor
 import hep.dataforge.meta.Meta
+import hep.dataforge.names.Name
+import hep.dataforge.storage.Storage
+import hep.dataforge.storage.StorageElement
 import hep.dataforge.storage.StorageManager
 import inr.numass.data.api.NumassSet
+import kotlinx.coroutines.experimental.runBlocking
+import kotlin.coroutines.experimental.buildSequence
 
 /**
  * Created by darksnake on 03-Feb-17.
@@ -14,13 +19,31 @@ class NumassDataFactory : DataFactory<NumassSet>(NumassSet::class.java) {
 
     override val name = "numass"
 
+    /**
+     * Build the sequence of name
+     */
+    private fun Storage.sequence(prefix: Name = Name.empty()): Sequence<Pair<Name, StorageElement>> {
+        return buildSequence {
+            runBlocking { getChildren() }.forEach {
+                val newName = prefix + it.name
+                yield(Pair(newName, it))
+                if (it is Storage) {
+                    yieldAll(it.sequence(newName))
+                }
+            }
+        }
+
+    }
 
     override fun fill(builder: DataNodeEditor<NumassSet>, context: Context, meta: Meta) {
         val newMeta = meta.builder.setValue("type", "numass")
-        val storage = context.load(StorageManager::class.java, Meta.empty()).buildStorage(newMeta)
+        runBlocking {
-        StorageUtils.loaderStream(storage).forEach { loader ->
+            val storage = context.load(StorageManager::class.java, Meta.empty()).create(newMeta) as Storage
-            if (loader is NumassSet) {
+            storage.sequence().forEach { pair ->
-                builder.putStatic(loader.fullName.unescaped, loader as NumassSet)
+                val value = pair.second
+                if (value is NumassSet) {
+                    builder.putStatic(pair.first.unescaped, value)
+                }
             }
         }
     }

numass-core/src/main/kotlin/inr/numass/data/storage/NumassDataLoader.kt

@@ -19,14 +19,13 @@ import hep.dataforge.connections.ConnectionHelper
 import hep.dataforge.context.Context
 import hep.dataforge.io.ColumnedDataReader
 import hep.dataforge.io.envelopes.Envelope
-import hep.dataforge.io.envelopes.EnvelopeReader
 import hep.dataforge.meta.Meta
 import hep.dataforge.providers.Provider
 import hep.dataforge.storage.Loader
 import hep.dataforge.storage.StorageElement
-import hep.dataforge.storage.files.FileStorage
 import hep.dataforge.storage.files.FileStorageElement
 import hep.dataforge.tables.Table
+import inr.numass.NumassEnvelopeType
 import inr.numass.data.api.NumassPoint
 import inr.numass.data.api.NumassSet
 import org.slf4j.LoggerFactory
@@ -58,11 +57,14 @@ class NumassDataLoader(
 
 
     override val meta: Meta by lazy {
-        FileStorage.resolveMeta(path) ?: Meta.empty()
+        val metaPath = path.resolve("meta")
+         NumassEnvelopeType.infer(metaPath)?.reader?.read(metaPath)?.meta?: Meta.empty()
     }
 
     override suspend fun getHvData(): Table? {
-        val hvEnvelope = path.resolve(HV_FRAGMENT_NAME)?.let { EnvelopeReader.readFile(it) }
+        val hvEnvelope = path.resolve(HV_FRAGMENT_NAME)?.let {
+            NumassEnvelopeType.infer(it)?.reader?.read(it) ?: error("Can't read hv file")
+        }
         return hvEnvelope?.let {
             try {
                 ColumnedDataReader(it.data.stream, "timestamp", "block", "value").toTable()
@@ -77,7 +79,9 @@ class NumassDataLoader(
     private val pointEnvelopes: List<Envelope>
         get() = Files.list(path)
                 .filter { it.fileName.toString().startsWith(POINT_FRAGMENT_NAME) }
-                .map { EnvelopeReader.readFile(it) }.toList()
+                .map {
+                    NumassEnvelopeType.infer(it)?.reader?.read(it) ?: error("Can't read point file")
+                }.toList()
 
     val isReversed: Boolean
         get() = this.meta.getBoolean("iteration_info.reverse", false)
@@ -95,10 +99,6 @@ class NumassDataLoader(
     override val startTime: Instant
         get() = meta.optValue("start_time").map<Instant> { it.time }.orElseGet { super.startTime }
 
-    override suspend fun open() {
-
-    }
-
     override fun close() {
         //do nothing
     }

numass-core/src/main/kotlin/inr/numass/data/storage/NumassStorage.kt

@@ -15,47 +15,35 @@
  */
 package inr.numass.data.storage
 
+import hep.dataforge.context.Context
 import hep.dataforge.events.Event
 import hep.dataforge.events.EventBuilder
-import hep.dataforge.io.envelopes.TaglessEnvelopeType
 import hep.dataforge.meta.Meta
-import hep.dataforge.nullable
+import hep.dataforge.storage.StorageElement
 import hep.dataforge.storage.files.FileStorage
 import hep.dataforge.storage.files.FileStorageElement
-import hep.dataforge.storage.files.FileStorageElementType
+import inr.numass.NumassEnvelopeType
 import java.nio.file.Files
 import java.nio.file.Path
-import java.nio.file.StandardOpenOption
 
 /**
  * Numass storage directory. Works as a normal directory, but creates a numass loader from each directory with meta
  */
-class NumassDirectory : FileStorageElementType {
+class NumassDirectory : FileStorage.Directory() {
-    override val name: String = "inr.numass.storage.directory"
+    override val name: String = NUMASS_DIRECTORY_TYPE
 
-
+    override suspend fun read(context: Context, path: Path, parent: StorageElement?): FileStorageElement? {
-
+        val meta = FileStorage.resolveMeta(path){ NumassEnvelopeType.infer(it)?.reader?.read(it)?.meta }
-    //TODO create mutable loader
+        return if (Files.isDirectory(path) && meta != null) {
-    override suspend fun create(parent: FileStorage, meta: Meta): FileStorageElement {
+            NumassDataLoader(context, parent, path.fileName.toString(), path)
-        val fileName = meta.getString("name")
+        } else {
-        val path: Path = parent.path.resolve(fileName)
+            super.read(context, path, parent)
-        Files.createDirectory(path)
-        //writing meta to directory
-        val metaFile = path.resolve("meta.df")
-        Files.newOutputStream(metaFile, StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE).use {
-            TaglessEnvelopeType.INSTANCE.writer.write(it, FileStorage.createMetaEnvelope(meta))
         }
-        return FileStorage(parent.context, meta, path, parent, this)
     }
 
-    override suspend fun read(parent: FileStorage, path: Path): FileStorageElement? {
+    companion object {
-        val meta = FileStorage.resolveMeta(path)
+        val INSTANCE = NumassDirectory()
-        val type = meta?.optString("type").nullable?.let { type -> parent.types.find { it.name == type } }
+        const val NUMASS_DIRECTORY_TYPE = "inr.numass.storage.directory"
-        return when {
-            type == this || Files.isDirectory(path) && meta != null -> NumassDataLoader(parent.context, parent, path.fileName.toString(), path)
-            Files.isDirectory(path) -> FileStorage(parent.context, meta ?: Meta.empty(), path, parent, this)
-            else -> type?.read(parent, path)
-        }
     }
 }
 
@@ -72,6 +60,7 @@ class NumassDataPointEvent(meta: Meta) : Event(meta) {
 
     companion object {
 
+
         const val FILE_NAME_KEY = "fileName"
         const val FILE_SIZE_KEY = "fileSize"
 
@@ -229,3 +218,69 @@ class NumassDataPointEvent(meta: Meta) : Event(meta) {
 //    }
 //
 //}
+
+//class NumassStorageFactory : StorageType {
+//
+//    override fun type(): String {
+//        return "numass"
+//    }
+//
+//    override fun build(context: Context, meta: Meta): Storage {
+//        if (meta.hasValue("path")) {
+//            val uri = URI.create(meta.getString("path"))
+//            val path: Path
+//            if (uri.scheme.startsWith("ssh")) {
+//                try {
+//                    val username = meta.getString("userName", uri.userInfo)
+//                    //String host = meta.getString("host", uri.getHost());
+//                    val port = meta.getInt("port", 22)
+//                    val env = SFTPEnvironment()
+//                            .withUsername(username)
+//                            .withPassword(meta.getString("password", "").toCharArray())
+//                    val fs = FileSystems.newFileSystem(uri, env, context.classLoader)
+//                    path = fs.getPath(uri.path)
+//                } catch (e: Exception) {
+//                    throw RuntimeException(e)
+//                }
+//
+//            } else {
+//                path = Paths.get(uri)
+//            }
+//            if(!Files.exists(path)){
+//                context.logger.info("File $path does not exist. Creating a new storage directory.")
+//                Files.createDirectories(path)
+//            }
+//            return NumassStorage(context, meta, path)
+//        } else {
+//            context.logger.warn("A storage path not provided. Creating default root storage in the working directory")
+//            return NumassStorage(context, meta, context.workDir)
+//        }
+//    }
+//
+//    companion object {
+//
+//        /**
+//         * Build local storage with Global context. Used for tests.
+//         *
+//         * @param file
+//         * @return
+//         */
+//        fun buildLocal(context: Context, file: Path, readOnly: Boolean, monitor: Boolean): FileStorage {
+//            val manager = context.load(StorageManager::class.java, Meta.empty())
+//            return manager.buildStorage(buildStorageMeta(file.toUri(), readOnly, monitor)) as FileStorage
+//        }
+//
+//        fun buildLocal(context: Context, path: String, readOnly: Boolean, monitor: Boolean): FileStorage {
+//            val file = context.dataDir.resolve(path)
+//            return buildLocal(context, file, readOnly, monitor)
+//        }
+//
+//        fun buildStorageMeta(path: URI, readOnly: Boolean, monitor: Boolean): MetaBuilder {
+//            return MetaBuilder("storage")
+//                    .setValue("path", path.toString())
+//                    .setValue("type", "numass")
+//                    .setValue("readOnly", readOnly)
+//                    .setValue("monitor", monitor)
+//        }
+//    }
+//}

numass-core/src/main/kotlin/inr/numass/data/storage/NumassStorageFactory.kt

@@ -1,84 +0,0 @@
-package inr.numass.data.storage
-
-import com.github.robtimus.filesystems.sftp.SFTPEnvironment
-import hep.dataforge.context.Context
-import hep.dataforge.meta.Meta
-import hep.dataforge.meta.MetaBuilder
-import hep.dataforge.storage.api.Storage
-import hep.dataforge.storage.api.StorageType
-import hep.dataforge.storage.commons.StorageManager
-import hep.dataforge.storage.filestorage.FileStorage
-import java.net.URI
-import java.nio.file.FileSystems
-import java.nio.file.Files
-import java.nio.file.Path
-import java.nio.file.Paths
-
-/**
- * Created by darksnake on 17-May-17.
- */
-class NumassStorageFactory : StorageType {
-
-    override fun type(): String {
-        return "numass"
-    }
-
-    override fun build(context: Context, meta: Meta): Storage {
-        if (meta.hasValue("path")) {
-            val uri = URI.create(meta.getString("path"))
-            val path: Path
-            if (uri.scheme.startsWith("ssh")) {
-                try {
-                    val username = meta.getString("userName", uri.userInfo)
-                    //String host = meta.getString("host", uri.getHost());
-                    val port = meta.getInt("port", 22)
-                    val env = SFTPEnvironment()
-                            .withUsername(username)
-                            .withPassword(meta.getString("password", "").toCharArray())
-                    val fs = FileSystems.newFileSystem(uri, env, context.classLoader)
-                    path = fs.getPath(uri.path)
-                } catch (e: Exception) {
-                    throw RuntimeException(e)
-                }
-
-            } else {
-                path = Paths.get(uri)
-            }
-            if(!Files.exists(path)){
-                context.logger.info("File $path does not exist. Creating a new storage directory.")
-                Files.createDirectories(path)
-            }
-            return NumassStorage(context, meta, path)
-        } else {
-            context.logger.warn("A storage path not provided. Creating default root storage in the working directory")
-            return NumassStorage(context, meta, context.workDir)
-        }
-    }
-
-    companion object {
-
-        /**
-         * Build local storage with Global context. Used for tests.
-         *
-         * @param file
-         * @return
-         */
-        fun buildLocal(context: Context, file: Path, readOnly: Boolean, monitor: Boolean): FileStorage {
-            val manager = context.load(StorageManager::class.java, Meta.empty())
-            return manager.buildStorage(buildStorageMeta(file.toUri(), readOnly, monitor)) as FileStorage
-        }
-
-        fun buildLocal(context: Context, path: String, readOnly: Boolean, monitor: Boolean): FileStorage {
-            val file = context.dataDir.resolve(path)
-            return buildLocal(context, file, readOnly, monitor)
-        }
-
-        fun buildStorageMeta(path: URI, readOnly: Boolean, monitor: Boolean): MetaBuilder {
-            return MetaBuilder("storage")
-                    .setValue("path", path.toString())
-                    .setValue("type", "numass")
-                    .setValue("readOnly", readOnly)
-                    .setValue("monitor", monitor)
-        }
-    }
-}

numass-core/src/main/kotlin/inr/numass/data/storage/ProtoNumassPoint.kt

@@ -58,7 +58,7 @@ class ProtoNumassPoint(override val meta: Meta, val protoBuilder: () -> NumassPr
 
     companion object {
         fun readFile(path: Path): ProtoNumassPoint {
-            return fromEnvelope(NumassFileEnvelope.open(path, true))
+            return fromEnvelope(NumassFileEnvelope(path))
         }
 
         fun fromEnvelope(envelope: Envelope): ProtoNumassPoint {

numass-core/src/main/resources/META-INF/services/hep.dataforge.storage.StorageElementType

@@ -0,0 +1 @@
+inr.numass.data.storage.NumassDirectory

numass-core/src/main/resources/META-INF/services/hep.dataforge.storage.api.StorageType

@@ -1 +0,0 @@
-inr.numass.data.storage.NumassStorageFactory

numass-main/src/main/kotlin/inr/numass/scripts/Correlation.kt

@@ -23,7 +23,6 @@ import inr.numass.data.NumassDataUtils
 import inr.numass.data.analyzers.SmartAnalyzer
 import inr.numass.data.api.NumassEvent
 import inr.numass.data.api.NumassSet
-import inr.numass.data.storage.NumassStorageFactory
 import org.apache.commons.math3.stat.correlation.PearsonsCorrelation
 import java.util.stream.Stream
 

numass-main/src/main/kotlin/inr/numass/scripts/InversedChain.kt

@@ -26,7 +26,6 @@ import inr.numass.data.analyzers.NumassAnalyzer.Companion.AMPLITUDE_ADAPTER
 import inr.numass.data.analyzers.SmartAnalyzer
 import inr.numass.data.analyzers.withBinning
 import inr.numass.data.api.NumassSet
-import inr.numass.data.storage.NumassStorageFactory
 import inr.numass.displayChart
 
 

numass-main/src/main/kotlin/inr/numass/scripts/models/DifferentialSpectrum.kt

@@ -28,7 +28,6 @@ import inr.numass.data.analyzers.SmartAnalyzer
 import inr.numass.data.analyzers.subtractAmplitudeSpectrum
 import inr.numass.data.analyzers.withBinning
 import inr.numass.data.api.NumassSet
-import inr.numass.data.storage.NumassStorageFactory
 import inr.numass.displayChart
 
 fun main(args: Array<String>) {

numass-main/src/main/kotlin/inr/numass/scripts/timeanalysis/AnalyzePoint.kt

@@ -13,7 +13,6 @@ import inr.numass.data.api.NumassPoint
 import inr.numass.data.api.NumassSet
 import inr.numass.data.api.SimpleNumassPoint
 import inr.numass.data.channel
-import inr.numass.data.storage.NumassStorageFactory
 
 fun main(args: Array<String>) {
 

numass-main/src/main/kotlin/inr/numass/scripts/timeanalysis/Histogram.kt

@@ -23,7 +23,6 @@ import inr.numass.NumassPlugin
 import inr.numass.data.NumassDataUtils
 import inr.numass.data.analyzers.TimeAnalyzer
 import inr.numass.data.api.NumassSet
-import inr.numass.data.storage.NumassStorageFactory
 import kotlin.streams.asStream
 
 fun main(args: Array<String>) {

numass-main/src/main/kotlin/inr/numass/scripts/tristan/ProtoPoint.kt

@@ -21,7 +21,6 @@ import hep.dataforge.toList
 import inr.numass.data.api.NumassPoint
 import inr.numass.data.channel
 import inr.numass.data.storage.NumassDataLoader
-import inr.numass.data.storage.NumassStorageFactory
 
 fun main(args: Array<String>) {
     val storage = NumassStorageFactory.buildLocal(Global, "D:\\Work\\Numass\\data\\2018_04\\Adiabacity_19\\", true, false)

numass-main/src/main/kotlin/inr/numass/scripts/utils/ScanTree.kt

@@ -6,10 +6,7 @@ import hep.dataforge.meta.Meta
 import hep.dataforge.meta.MetaBuilder
 import hep.dataforge.meta.MetaUtils
 import hep.dataforge.meta.buildMeta
-import hep.dataforge.storage.api.Storage
-import hep.dataforge.useValue
 import inr.numass.data.storage.NumassDataLoader
-import inr.numass.data.storage.NumassStorageFactory
 import java.io.File
 import java.nio.file.Paths
 

numass-main/src/main/kotlin/inr/numass/subthreshold/Threshold.kt

@@ -6,7 +6,6 @@ import hep.dataforge.data.DataNode
 import hep.dataforge.data.DataSet
 import hep.dataforge.meta.Meta
 import hep.dataforge.meta.buildMeta
-import hep.dataforge.storage.commons.StorageUtils
 import hep.dataforge.tables.ListTable
 import hep.dataforge.tables.Table
 import hep.dataforge.values.ValueMap
@@ -18,7 +17,6 @@ import inr.numass.data.analyzers.withBinning
 import inr.numass.data.api.NumassPoint
 import inr.numass.data.api.NumassSet
 import inr.numass.data.api.SimpleNumassPoint
-import inr.numass.data.storage.NumassStorageFactory
 import org.apache.commons.math3.analysis.ParametricUnivariateFunction
 import org.apache.commons.math3.exception.DimensionMismatchException
 import org.apache.commons.math3.fitting.SimpleCurveFitter

numass-viewer/build.gradle

@@ -12,7 +12,7 @@ if (!hasProperty('mainClass')) {
 
 mainClassName = mainClass
 
-version = "0.5.3"
+version = "0.5.4"
 
 description = "The viewer for numass data"
 

numass-viewer/src/main/kotlin/inr/numass/viewer/AmplitudeView.kt

@@ -7,6 +7,7 @@ import hep.dataforge.fx.plots.PlotContainer
 import hep.dataforge.fx.runGoal
 import hep.dataforge.fx.ui
 import hep.dataforge.goals.Goal
+import hep.dataforge.names.Name
 import hep.dataforge.plots.PlotFrame
 import hep.dataforge.plots.PlotGroup
 import hep.dataforge.plots.Plottable
@@ -122,9 +123,8 @@ class AmplitudeView : View(title = "Numass amplitude spectrum plot", icon = Imag
     }
 
     private fun invalidate() {
-        isEmpty.invalidate()
         data.forEach { key, point ->
-            plots.computeIfAbsent(key) {
+            plots.getOrPut(key) {
                 runGoal<Plottable>("loadAmplitudeSpectrum_$key") {
                     val valueAxis = if (normalize) {
                         NumassAnalyzer.COUNT_RATE_KEY
@@ -157,7 +157,7 @@ class AmplitudeView : View(title = "Numass amplitude spectrum plot", icon = Imag
                     frame.add(plot)
                     progress.invalidate()
                 } except {
-                    throw it
+                    progress.invalidate()
                 }
             }
             plots.keys.filter { !data.containsKey(it) }.forEach { remove(it) }
@@ -177,10 +177,11 @@ class AmplitudeView : View(title = "Numass amplitude spectrum plot", icon = Imag
      * Remove the plot and cancel loading task if it is in progress.
      */
     fun remove(name: String) {
-        frame.remove(name);
+        frame.plots.remove(Name.ofSingle(name))
-        plots[name]?.cancel();
+        plots[name]?.cancel()
-        plots.remove(name);
+        plots.remove(name)
         data.remove(name)
+        progress.invalidate()
     }
 
     /**

numass-viewer/src/main/kotlin/inr/numass/viewer/HVView.kt

@@ -4,10 +4,13 @@ import hep.dataforge.configure
 import hep.dataforge.fx.dfIcon
 import hep.dataforge.fx.plots.PlotContainer
 import hep.dataforge.fx.runGoal
+import hep.dataforge.fx.ui
+import hep.dataforge.names.Name
 import hep.dataforge.plots.PlotFrame
 import hep.dataforge.plots.data.DataPlot
 import hep.dataforge.plots.data.TimePlot
 import hep.dataforge.plots.jfreechart.JFreeChartFrame
+import hep.dataforge.tables.Adapters
 import inr.numass.data.api.NumassSet
 import javafx.collections.FXCollections
 import javafx.collections.MapChangeListener
@@ -33,7 +36,7 @@ class HVView : View(title = "High voltage time plot", icon = ImageView(dfIcon))
             "showSymbol" to false
             "showErrors" to false
         }
-        plots.setType<DataPlot>()
+        plots.setType<TimePlot>()
     }
     private val container = PlotContainer(frame);
 
@@ -48,20 +51,19 @@ class HVView : View(title = "High voltage time plot", icon = ImageView(dfIcon))
         data.addListener { change: MapChangeListener.Change<out String, out NumassSet> ->
             isEmpty.invalidate()
             if (change.wasRemoved()) {
-                frame.remove(change.key)
+                frame.plots.remove(Name.ofSingle(change.key))
             }
             if (change.wasAdded()) {
                 runLater { container.progress = -1.0 }
                 runGoal("hvData[${change.key}]") {
-                    change.valueAdded.getHvData().await()
+                    change.valueAdded.getHvData()
-                } ui { hvData ->
+                } ui {table->
-                    hvData?.let {
+                    if(table!= null) {
-                        for (dp in it) {
+                        ((frame[change.key] as? DataPlot)
-                            val plot: TimePlot = frame[change.key] as TimePlot?
+                                ?: DataPlot(change.key, adapter = Adapters.buildXYAdapter("timestamp", "value")).also { frame.add(it) })
-                                    ?: TimePlot(change.key).apply { frame.add(this) }
+                                .fillData(table)
-                            plot.put(dp.getValue("timestamp").time, dp.getValue("value"))
-                        }
                     }
+
                     container.progress = 1.0;
                 }
             }

numass-viewer/src/main/kotlin/inr/numass/viewer/MainView.kt

@@ -5,12 +5,12 @@ import hep.dataforge.context.Global
 import hep.dataforge.fx.*
 import hep.dataforge.fx.fragments.LogFragment
 import hep.dataforge.fx.meta.MetaViewer
-import hep.dataforge.storage.commons.StorageManager
+import hep.dataforge.storage.Storage
 import inr.numass.NumassProperties
 import inr.numass.data.api.NumassPoint
 import inr.numass.data.legacy.NumassFileEnvelope
 import inr.numass.data.storage.NumassDataLoader
-import inr.numass.data.storage.NumassStorageFactory
+import inr.numass.data.storage.NumassDirectory
 import javafx.beans.property.SimpleObjectProperty
 import javafx.geometry.Insets
 import javafx.scene.control.Alert
@@ -18,7 +18,6 @@ import javafx.scene.layout.Priority
 import javafx.scene.text.Font
 import javafx.stage.DirectoryChooser
 import javafx.stage.FileChooser
-import kotlinx.coroutines.experimental.async
 import org.controlsfx.control.StatusBar
 import tornadofx.*
 import java.io.File
@@ -68,7 +67,7 @@ class MainView(val context: Context = Global.getContext("viewer")) : View(title
 
                             if (rootDir != null) {
                                 NumassProperties.setNumassProperty("numass.viewer.lastPath", rootDir.absolutePath)
-                                async {
+                                kotlinx.coroutines.experimental.launch {
                                     runLater {
                                         path = rootDir.toPath()
                                     }
@@ -97,7 +96,7 @@ class MainView(val context: Context = Global.getContext("viewer")) : View(title
                             val file = chooser.showOpenDialog(primaryStage.scene.window)
                             if (file != null) {
                                 NumassProperties.setNumassProperty("numass.viewer.lastPath", file.parentFile.absolutePath)
-                                async {
+                                kotlinx.coroutines.experimental.launch {
                                     runLater {
                                         path = file.toPath()
                                     }
@@ -149,7 +148,7 @@ class MainView(val context: Context = Global.getContext("viewer")) : View(title
                 runGoal("viewer.load.set[$path]") {
                     title = "Load set ($path)"
                     message = "Building numass set..."
-                    NumassDataLoader.fromDir(context, path)
+                    NumassDataLoader(context, null, path.fileName.toString(), path)
                 } ui {
                     contentView = SpectrumView().apply {
                         clear()
@@ -168,25 +167,16 @@ class MainView(val context: Context = Global.getContext("viewer")) : View(title
                 runGoal("viewer.load.storage[$path]") {
                     title = "Load storage ($path)"
                     message = "Building numass storage tree..."
-                    StorageManager.buildStorage(
+                    NumassDirectory.INSTANCE.read(context, path)
-                            context,
-                            NumassStorageFactory.buildStorageMeta(path.toUri(), true, false)
-                    )
                 } ui {
-                    contentView = StorageView(it)
+                    contentView = StorageView(it as Storage)
                     infoView = MetaViewer(it.meta)
-                } except {
-                    alert(
-                            type = Alert.AlertType.ERROR,
-                            header = "Error during storage loading",
-                            content = it.toString()
-                    ).show()
                 }
             }
         } else {
             //Reading individual file
             val envelope = try {
-                NumassFileEnvelope.open(path, true)
+                NumassFileEnvelope(path)
             } catch (ex: Exception) {
                 runLater {
                     alert(

numass-viewer/src/main/kotlin/inr/numass/viewer/SlowControlView.kt

@@ -5,17 +5,12 @@ import hep.dataforge.fx.dfIcon
 import hep.dataforge.fx.plots.PlotContainer
 import hep.dataforge.fx.runGoal
 import hep.dataforge.fx.ui
-import hep.dataforge.meta.Meta
 import hep.dataforge.plots.PlotGroup
 import hep.dataforge.plots.data.DataPlot
 import hep.dataforge.plots.jfreechart.JFreeChartFrame
-import hep.dataforge.storage.api.TableLoader
+import hep.dataforge.storage.TableLoader
-import hep.dataforge.storage.api.ValueIndex
 import hep.dataforge.tables.Adapters
-import hep.dataforge.tables.ListTable
 import hep.dataforge.tables.Table
-import hep.dataforge.toList
-import hep.dataforge.values.Values
 import javafx.collections.FXCollections
 import javafx.collections.MapChangeListener
 import javafx.collections.ObservableMap
@@ -71,20 +66,9 @@ class SlowControlView : View(title = "Numass slow control view", icon = ImageVie
         }
     }
 
-    private fun getData(loader: TableLoader, query: Meta = Meta.empty()): Table {
+    private suspend fun getData(loader: TableLoader): Table {
-        val index: ValueIndex<Values> =
+        //TODO add query
-                if (query.hasValue("index")) {
+        return loader.asTable()
-                    //use custom index if needed
-                    loader.getIndex(query.getString("index"))
-                } else {
-                    //use loader default one otherwise
-                    loader.index
-                }
-        try {
-            return ListTable(loader.format, index.query(query).toList())
-        } catch (e: Exception) {
-            throw RuntimeException(e)
-        }
     }
 
     operator fun set(id: String, loader: TableLoader) {

numass-viewer/src/main/kotlin/inr/numass/viewer/SpectrumView.kt

@@ -5,6 +5,7 @@ import hep.dataforge.fx.dfIcon
 import hep.dataforge.fx.plots.PlotContainer
 import hep.dataforge.fx.runGoal
 import hep.dataforge.fx.ui
+import hep.dataforge.names.Name
 import hep.dataforge.plots.PlotFrame
 import hep.dataforge.plots.data.DataPlot
 import hep.dataforge.plots.jfreechart.JFreeChartFrame
@@ -100,7 +101,7 @@ class SpectrumView : View(title = "Numass spectrum plot", icon = ImageView(dfIco
     init {
         data.addListener { change: MapChangeListener.Change<out String, out NumassSet> ->
             if (change.wasRemoved()) {
-                frame.remove(change.key);
+                frame.plots.remove(Name.ofSingle(change.key));
             }
 
             if (change.wasAdded()) {
@@ -116,7 +117,7 @@ class SpectrumView : View(title = "Numass spectrum plot", icon = ImageView(dfIco
         val totalProgress = data.values.stream().mapToInt { it.points.size }.sum()
 
         data.forEach { name, set ->
-            val plot: DataPlot = frame[name] as DataPlot? ?: DataPlot(name).apply { frame.add(this) }
+            val plot: DataPlot = frame.plots[Name.ofSingle(name)] as DataPlot? ?: DataPlot(name).apply { frame.add(this) }
 
             runGoal("spectrumData[$name]") {
                 set.points.forEach { it.spectrum.start() }

numass-viewer/src/main/kotlin/inr/numass/viewer/StorageView.kt

@@ -2,17 +2,18 @@ package inr.numass.viewer
 
 import hep.dataforge.fx.dfIconView
 import hep.dataforge.fx.meta.MetaViewer
-import hep.dataforge.fx.runGoal
+import hep.dataforge.meta.Meta
 import hep.dataforge.meta.Metoid
-import hep.dataforge.storage.api.Loader
+import hep.dataforge.storage.Storage
-import hep.dataforge.storage.api.Storage
+import hep.dataforge.storage.TableLoader
-import hep.dataforge.storage.api.TableLoader
+import hep.dataforge.storage.files.FileTableLoader
 import inr.numass.data.api.NumassPoint
 import inr.numass.data.api.NumassSet
 import inr.numass.data.storage.NumassDataLoader
 import javafx.beans.property.SimpleBooleanProperty
 import javafx.scene.control.ContextMenu
 import javafx.scene.control.TreeItem
+import kotlinx.coroutines.experimental.runBlocking
 import tornadofx.*
 
 class StorageView(val storage: Storage) : View(title = "Numass storage", icon = dfIconView) {
@@ -33,11 +34,11 @@ class StorageView(val storage: Storage) : View(title = "Numass storage", icon =
         val checkedProperty = SimpleBooleanProperty(false)
         var checked by checkedProperty
 
-        val infoView: UIComponent? by lazy {
+        val infoView: UIComponent by lazy {
             when (content) {
                 is CachedPoint -> PointInfoView(content)
                 is Metoid -> MetaViewer(content.meta, title = "Meta view: $id")
-                else -> null
+                else -> MetaViewer(Meta.empty(), title = "Meta view: $id")
             }
         }
 
@@ -73,8 +74,8 @@ class StorageView(val storage: Storage) : View(title = "Numass storage", icon =
 
         val children: List<Container>? by lazy {
             when (content) {
-                is Storage -> (content.shelves().sorted() + content.loaders().sorted()).map { buildContainer(it, this) }
+                is Storage -> (runBlocking { content.getChildren() }.sortedBy { it.name }).map { buildContainer(it, this) }
-                is CachedSet -> content.points
+                is NumassSet -> content.points
                         .sortedBy { it.index }
                         .map { buildContainer(it, this) }
                         .toList()
@@ -82,9 +83,7 @@ class StorageView(val storage: Storage) : View(title = "Numass storage", icon =
             }
         }
 
-        val hasChildren: Boolean
+        val hasChildren: Boolean = (content is Storage) || (content is NumassSet)
-            get() = (content is Storage) || (content is NumassPoint)
-
     }
 
 
@@ -93,9 +92,7 @@ class StorageView(val storage: Storage) : View(title = "Numass storage", icon =
             //isShowRoot = false
             root = TreeItem(Container(storage.name, storage))
             root.isExpanded = true
-            runGoal("viewer.storage.populateTree") {
+            lazyPopulate(leafCheck = { !it.value.hasChildren }) { it.value.children }
-                populate { parent -> parent.value.children }
-            }
             cellFormat { value ->
                 when (value.content) {
                     is Storage -> {
@@ -131,11 +128,9 @@ class StorageView(val storage: Storage) : View(title = "Numass storage", icon =
                             this@cellFormat.treeItem.uncheckAll()
                         }
                     }
-                    value.infoView?.let {
                     item("Info") {
                         action {
-                                it.openModal(escapeClosesWindow = true)
+                            value.infoView.openModal(escapeClosesWindow = true)
-                            }
                         }
                     }
                 }
@@ -181,19 +176,19 @@ class StorageView(val storage: Storage) : View(title = "Numass storage", icon =
                 }
                 is NumassSet -> {
                     val id: String = if (content is NumassDataLoader) {
-                        content.path.toString()
+                        content.fullName.unescaped
                     } else {
                         content.name
                     }
                     Container(id, content as? CachedSet ?: CachedSet(content))
                 }
                 is NumassPoint -> {
-                    Container("${parent.id}/${content.voltage}[${content.index}]", content as? CachedPoint ?: CachedPoint(content))
+                    Container("${parent.id}/${content.voltage}[${content.index}]", content as? CachedPoint
+                            ?: CachedPoint(content))
                 }
-                is Loader -> {
+                is FileTableLoader -> {
                     Container(content.path.toString(), content);
                 }
                 else -> throw IllegalArgumentException("Unknown content type: ${content::class.java}");
             }
-
 }

numass-viewer/src/main/kotlin/inr/numass/viewer/test/ComponentTest.kt

@@ -2,14 +2,16 @@ package inr.numass.viewer.test
 
 import hep.dataforge.context.Global
 import hep.dataforge.fx.dfIcon
+import hep.dataforge.nullable
 import hep.dataforge.tables.Table
 import inr.numass.data.api.NumassPoint
 import inr.numass.data.api.NumassSet
-import inr.numass.data.storage.NumassStorageFactory
+import inr.numass.data.storage.NumassDirectory
 import inr.numass.viewer.*
 import javafx.application.Application
 import javafx.scene.image.ImageView
 import tornadofx.*
+import java.io.File
 import java.util.concurrent.ConcurrentHashMap
 
 class ViewerComponentsTestApp : App(ViewerComponentsTest::class)
@@ -31,10 +33,10 @@ class ViewerComponentsTest : View(title = "Numass viewer test", icon = ImageView
         top {
             button("Click me!") {
                 action {
-                    runAsync {
+                    kotlinx.coroutines.experimental.launch {
-                        val set: NumassSet = NumassStorageFactory.buildLocal(Global, "D:\\Work\\Numass\\data\\2017_05\\Fill_2", true, true)
+                        val set: NumassSet = NumassDirectory.INSTANCE.read(Global, File("D:\\Work\\Numass\\data\\2017_05\\Fill_2").toPath())
-                                .provide("loader::set_2", NumassSet::class.java)
+                                ?.provide("loader::set_2", NumassSet::class.java).nullable
-                                .orElseThrow { RuntimeException("err") }
+                                ?: kotlin.error("Error")
                         update(set);
                     }
                 }