NPM/numass-framework
4
0
Fork 1
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Moved everything to storage2. A lot of fixes in plots. Storage1 deprecated.

Browse Source
This commit is contained in:
Alexander Nozik 2018-08-05 21:01:05 +03:00
parent 0d1c027820
commit e2de03fdcc
27 changed files with 283 additions and 255 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
numass-core/build.gradle
View File

@ -14,6 +14,7 @@ description = "A bse package with minimal dependencies for numass"
dependencies {
compile "hep.dataforge:dataforge-storage2"
compile "hep.dataforge:dataforge-json"
compile 'com.google.protobuf:protobuf-java:3.5.0'
// https://mvnrepository.com/artifact/com.github.robtimus/sftp-fs

46
numass-core/src/main/kotlin/inr/numass/NumassEnvelopeType.kt
View File

@ -3,11 +3,12 @@ package inr.numass
import hep.dataforge.io.envelopes.*
import hep.dataforge.values.Value
import hep.dataforge.values.parseValue
import inr.numass.data.legacy.NumassFileEnvelope.Companion.LEGACY_END_SEQUENCE
import inr.numass.data.legacy.NumassFileEnvelope.Companion.LEGACY_START_SEQUENCE
import org.slf4j.LoggerFactory
import java.io.IOException
import java.nio.ByteBuffer
import java.nio.file.Files
import java.nio.file.Path
import java.nio.file.StandardOpenOption
import java.util.*
/**
@ -21,10 +22,16 @@ class NumassEnvelopeType : EnvelopeType {
override fun description(): String = "Numass legacy envelope"
/**
* Read as legacy
*/
override fun getReader(properties: Map<String, String>): EnvelopeReader {
return NumassEnvelopeReader()
}
/**
* Write as default
*/
override fun getWriter(properties: Map<String, String>): EnvelopeWriter {
return DefaultEnvelopeWriter(this, MetaType.resolve(properties))
}
@ -80,4 +87,39 @@ class NumassEnvelopeType : EnvelopeType {
}
}
companion object {
val INSTANCE = NumassEnvelopeType()
val LEGACY_START_SEQUENCE = byteArrayOf('#'.toByte(), '!'.toByte())
val LEGACY_END_SEQUENCE = byteArrayOf('!'.toByte(), '#'.toByte(), '\r'.toByte(), '\n'.toByte())
/**
* Replacement for standard type infer to include legacy type
*
* @param path
* @return
*/
fun infer(path: Path): EnvelopeType? {
return try {
Files.newInputStream(path, StandardOpenOption.READ).use {
val buffer = ByteArray(6)
it.read(buffer)
val header = String(buffer)
when {
//TODO use templates from appropriate types
header.startsWith("#!") -> NumassEnvelopeType.INSTANCE
header.startsWith("#~DFTL") -> TaglessEnvelopeType.INSTANCE
header.startsWith("#~") -> DefaultEnvelopeType.INSTANCE
else -> null
}
}
} catch (ex: Exception) {
LoggerFactory.getLogger(EnvelopeType::class.java).warn("Could not infer envelope type of file {} due to exception: {}", path, ex)
null
}
}
}
}

85
numass-core/src/main/kotlin/inr/numass/data/legacy/NumassFileEnvelope.kt
View File

@ -1,43 +1,70 @@
package inr.numass.data.legacy
import hep.dataforge.io.envelopes.EnvelopeTag
import hep.dataforge.meta.Meta
import hep.dataforge.storage.files.FileEnvelope
import inr.numass.NumassEnvelopeType
import java.io.IOException
import java.nio.ByteBuffer
import java.nio.channels.FileChannel
import java.nio.file.Files
import java.nio.file.Path
import java.nio.file.StandardOpenOption.READ
import java.nio.file.StandardOpenOption
class NumassFileEnvelope private constructor(path: Path, readOnly: Boolean) : FileEnvelope(path, readOnly) {
class NumassFileEnvelope(path: Path) : FileEnvelope(path) {
override fun buildTag(): EnvelopeTag {
return NumassEnvelopeType.LegacyTag()
}
private val tag by lazy { Files.newByteChannel(path, StandardOpenOption.READ).use { NumassEnvelopeType.LegacyTag().read(it) } }
companion object {
override val dataOffset: Long by lazy { (tag.length + tag.metaSize).toLong() }
val LEGACY_START_SEQUENCE = byteArrayOf('#'.toByte(), '!'.toByte())
val LEGACY_END_SEQUENCE = byteArrayOf('!'.toByte(), '#'.toByte(), '\r'.toByte(), '\n'.toByte())
fun open(path: Path, readOnly: Boolean): FileEnvelope {
// if (!Files.exists(path)) {
// throw new RuntimeException("File envelope does not exist");
// }
try {
FileChannel.open(path, READ).use { channel ->
val buffer = channel.map(FileChannel.MapMode.READ_ONLY, 0, 2)
return if (buffer.compareTo(ByteBuffer.wrap(LEGACY_START_SEQUENCE)) == 0) {
NumassFileEnvelope(path, readOnly)
} else {
FileEnvelope.open(path, readOnly)
}
}
} catch (e: IOException) {
throw RuntimeException("Failed to open file envelope", e)
override var dataLength: Int
get() = tag.dataSize
set(value) {
if (value > Int.MAX_VALUE) {
throw RuntimeException("Too large data block")
}
tag.dataSize = value
if (channel.write(tag.toBytes(), 0L) < tag.length) {
throw error("Tag is not overwritten.")
}
}
override val meta: Meta by lazy {
val buffer = ByteBuffer.allocate(tag.metaSize).also {
channel.read(it, tag.length.toLong())
}
tag.metaType.reader.readBuffer(buffer)
}
}
//
// override fun buildTag(): EnvelopeTag {
// return NumassEnvelopeType.LegacyTag()
// }
//
// companion object {
//
// val LEGACY_START_SEQUENCE = byteArrayOf('#'.toByte(), '!'.toByte())
// val LEGACY_END_SEQUENCE = byteArrayOf('!'.toByte(), '#'.toByte(), '\r'.toByte(), '\n'.toByte())
//
// fun open(path: Path, readOnly: Boolean): FileEnvelope {
// // if (!Files.exists(path)) {
// // throw new RuntimeException("File envelope does not exist");
// // }
//
// try {
// FileChannel.open(path, READ).use { channel ->
// val buffer = channel.map(FileChannel.MapMode.READ_ONLY, 0, 2)
// return if (buffer.compareTo(ByteBuffer.wrap(LEGACY_START_SEQUENCE)) == 0) {
// NumassFileEnvelope(path, readOnly)
// } else {
// FileEnvelope.open(path, readOnly)
// }
// }
// } catch (e: IOException) {
// throw RuntimeException("Failed to open file envelope", e)
// }
//
// }
// }

2
numass-core/src/main/kotlin/inr/numass/data/storage/ClassicNumassPoint.kt
View File

@ -104,7 +104,7 @@ class ClassicNumassPoint(private val envelope: Envelope) : NumassPoint {
companion object {
fun readFile(path: Path): ClassicNumassPoint {
return ClassicNumassPoint(NumassFileEnvelope.open(path, true))
return ClassicNumassPoint(NumassFileEnvelope(path))
}
}
}

33
numass-core/src/main/kotlin/inr/numass/data/storage/NumassDataFactory.kt
View File

@ -4,23 +4,46 @@ import hep.dataforge.context.Context
import hep.dataforge.data.DataFactory
import hep.dataforge.data.DataNodeEditor
import hep.dataforge.meta.Meta
import hep.dataforge.names.Name
import hep.dataforge.storage.Storage
import hep.dataforge.storage.StorageElement
import hep.dataforge.storage.StorageManager
import inr.numass.data.api.NumassSet
import kotlinx.coroutines.experimental.runBlocking
import kotlin.coroutines.experimental.buildSequence
/**
* Created by darksnake on 03-Feb-17.
*/
class NumassDataFactory : DataFactory<NumassSet>(NumassSet::class.java) {
override val name= "numass"
override val name = "numass"
/**
* Build the sequence of name
*/
private fun Storage.sequence(prefix: Name = Name.empty()): Sequence<Pair<Name, StorageElement>> {
return buildSequence {
runBlocking { getChildren() }.forEach {
val newName = prefix + it.name
yield(Pair(newName, it))
if (it is Storage) {
yieldAll(it.sequence(newName))
}
}
}
}
override fun fill(builder: DataNodeEditor<NumassSet>, context: Context, meta: Meta) {
val newMeta = meta.builder.setValue("type", "numass")
val storage = context.load(StorageManager::class.java, Meta.empty()).buildStorage(newMeta)
StorageUtils.loaderStream(storage).forEach { loader ->
if (loader is NumassSet) {
builder.putStatic(loader.fullName.unescaped, loader as NumassSet)
runBlocking {
val storage = context.load(StorageManager::class.java, Meta.empty()).create(newMeta) as Storage
storage.sequence().forEach { pair ->
val value = pair.second
if (value is NumassSet) {
builder.putStatic(pair.first.unescaped, value)
}
}
}
}

20
numass-core/src/main/kotlin/inr/numass/data/storage/NumassDataLoader.kt
View File

@ -19,14 +19,13 @@ import hep.dataforge.connections.ConnectionHelper
import hep.dataforge.context.Context
import hep.dataforge.io.ColumnedDataReader
import hep.dataforge.io.envelopes.Envelope
import hep.dataforge.io.envelopes.EnvelopeReader
import hep.dataforge.meta.Meta
import hep.dataforge.providers.Provider
import hep.dataforge.storage.Loader
import hep.dataforge.storage.StorageElement
import hep.dataforge.storage.files.FileStorage
import hep.dataforge.storage.files.FileStorageElement
import hep.dataforge.tables.Table
import inr.numass.NumassEnvelopeType
import inr.numass.data.api.NumassPoint
import inr.numass.data.api.NumassSet
import org.slf4j.LoggerFactory
@ -54,15 +53,18 @@ class NumassDataLoader(
private val _connectionHelper = ConnectionHelper(this)
override fun getConnectionHelper(): ConnectionHelper =_connectionHelper
override fun getConnectionHelper(): ConnectionHelper = _connectionHelper
override val meta: Meta by lazy {
FileStorage.resolveMeta(path) ?: Meta.empty()
val metaPath = path.resolve("meta")
NumassEnvelopeType.infer(metaPath)?.reader?.read(metaPath)?.meta?: Meta.empty()
}
override suspend fun getHvData(): Table? {
val hvEnvelope = path.resolve(HV_FRAGMENT_NAME)?.let { EnvelopeReader.readFile(it) }
val hvEnvelope = path.resolve(HV_FRAGMENT_NAME)?.let {
NumassEnvelopeType.infer(it)?.reader?.read(it) ?: error("Can't read hv file")
}
return hvEnvelope?.let {
try {
ColumnedDataReader(it.data.stream, "timestamp", "block", "value").toTable()
@ -77,7 +79,9 @@ class NumassDataLoader(
private val pointEnvelopes: List<Envelope>
get() = Files.list(path)
.filter { it.fileName.toString().startsWith(POINT_FRAGMENT_NAME) }
.map { EnvelopeReader.readFile(it) }.toList()
.map {
NumassEnvelopeType.infer(it)?.reader?.read(it) ?: error("Can't read point file")
}.toList()
val isReversed: Boolean
get() = this.meta.getBoolean("iteration_info.reverse", false)
@ -95,10 +99,6 @@ class NumassDataLoader(
override val startTime: Instant
get() = meta.optValue("start_time").map<Instant> { it.time }.orElseGet { super.startTime }
override suspend fun open() {
}
override fun close() {
//do nothing
}

107
numass-core/src/main/kotlin/inr/numass/data/storage/NumassStorage.kt
View File

@ -15,47 +15,35 @@
*/
package inr.numass.data.storage
import hep.dataforge.context.Context
import hep.dataforge.events.Event
import hep.dataforge.events.EventBuilder
import hep.dataforge.io.envelopes.TaglessEnvelopeType
import hep.dataforge.meta.Meta
import hep.dataforge.nullable
import hep.dataforge.storage.StorageElement
import hep.dataforge.storage.files.FileStorage
import hep.dataforge.storage.files.FileStorageElement
import hep.dataforge.storage.files.FileStorageElementType
import inr.numass.NumassEnvelopeType
import java.nio.file.Files
import java.nio.file.Path
import java.nio.file.StandardOpenOption
/**
* Numass storage directory. Works as a normal directory, but creates a numass loader from each directory with meta
*/
class NumassDirectory : FileStorageElementType {
override val name: String = "inr.numass.storage.directory"
class NumassDirectory : FileStorage.Directory() {
override val name: String = NUMASS_DIRECTORY_TYPE
//TODO create mutable loader
override suspend fun create(parent: FileStorage, meta: Meta): FileStorageElement {
val fileName = meta.getString("name")
val path: Path = parent.path.resolve(fileName)
Files.createDirectory(path)
//writing meta to directory
val metaFile = path.resolve("meta.df")
Files.newOutputStream(metaFile, StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE).use {
TaglessEnvelopeType.INSTANCE.writer.write(it, FileStorage.createMetaEnvelope(meta))
override suspend fun read(context: Context, path: Path, parent: StorageElement?): FileStorageElement? {
val meta = FileStorage.resolveMeta(path){ NumassEnvelopeType.infer(it)?.reader?.read(it)?.meta }
return if (Files.isDirectory(path) && meta != null) {
NumassDataLoader(context, parent, path.fileName.toString(), path)
} else {
super.read(context, path, parent)
}
return FileStorage(parent.context, meta, path, parent, this)
}
override suspend fun read(parent: FileStorage, path: Path): FileStorageElement? {
val meta = FileStorage.resolveMeta(path)
val type = meta?.optString("type").nullable?.let { type -> parent.types.find { it.name == type } }
return when {
type == this || Files.isDirectory(path) && meta != null -> NumassDataLoader(parent.context, parent, path.fileName.toString(), path)
Files.isDirectory(path) -> FileStorage(parent.context, meta ?: Meta.empty(), path, parent, this)
else -> type?.read(parent, path)
}
companion object {
val INSTANCE = NumassDirectory()
const val NUMASS_DIRECTORY_TYPE = "inr.numass.storage.directory"
}
}
@ -72,6 +60,7 @@ class NumassDataPointEvent(meta: Meta) : Event(meta) {
companion object {
const val FILE_NAME_KEY = "fileName"
const val FILE_SIZE_KEY = "fileSize"
@ -229,3 +218,69 @@ class NumassDataPointEvent(meta: Meta) : Event(meta) {
// }
//
//}
//class NumassStorageFactory : StorageType {
//
// override fun type(): String {
// return "numass"
// }
//
// override fun build(context: Context, meta: Meta): Storage {
// if (meta.hasValue("path")) {
// val uri = URI.create(meta.getString("path"))
// val path: Path
// if (uri.scheme.startsWith("ssh")) {
// try {
// val username = meta.getString("userName", uri.userInfo)
// //String host = meta.getString("host", uri.getHost());
// val port = meta.getInt("port", 22)
// val env = SFTPEnvironment()
// .withUsername(username)
// .withPassword(meta.getString("password", "").toCharArray())
// val fs = FileSystems.newFileSystem(uri, env, context.classLoader)
// path = fs.getPath(uri.path)
// } catch (e: Exception) {
// throw RuntimeException(e)
// }
//
// } else {
// path = Paths.get(uri)
// }
// if(!Files.exists(path)){
// context.logger.info("File $path does not exist. Creating a new storage directory.")
// Files.createDirectories(path)
// }
// return NumassStorage(context, meta, path)
// } else {
// context.logger.warn("A storage path not provided. Creating default root storage in the working directory")
// return NumassStorage(context, meta, context.workDir)
// }
// }
//
// companion object {
//
// /**
// * Build local storage with Global context. Used for tests.
// *
// * @param file
// * @return
// */
// fun buildLocal(context: Context, file: Path, readOnly: Boolean, monitor: Boolean): FileStorage {
// val manager = context.load(StorageManager::class.java, Meta.empty())
// return manager.buildStorage(buildStorageMeta(file.toUri(), readOnly, monitor)) as FileStorage
// }
//
// fun buildLocal(context: Context, path: String, readOnly: Boolean, monitor: Boolean): FileStorage {
// val file = context.dataDir.resolve(path)
// return buildLocal(context, file, readOnly, monitor)
// }
//
// fun buildStorageMeta(path: URI, readOnly: Boolean, monitor: Boolean): MetaBuilder {
// return MetaBuilder("storage")
// .setValue("path", path.toString())
// .setValue("type", "numass")
// .setValue("readOnly", readOnly)
// .setValue("monitor", monitor)
// }
// }
//}

84
numass-core/src/main/kotlin/inr/numass/data/storage/NumassStorageFactory.kt
View File

@ -1,84 +0,0 @@
package inr.numass.data.storage
import com.github.robtimus.filesystems.sftp.SFTPEnvironment
import hep.dataforge.context.Context
import hep.dataforge.meta.Meta
import hep.dataforge.meta.MetaBuilder
import hep.dataforge.storage.api.Storage
import hep.dataforge.storage.api.StorageType
import hep.dataforge.storage.commons.StorageManager
import hep.dataforge.storage.filestorage.FileStorage
import java.net.URI
import java.nio.file.FileSystems
import java.nio.file.Files
import java.nio.file.Path
import java.nio.file.Paths
/**
* Created by darksnake on 17-May-17.
*/
class NumassStorageFactory : StorageType {
override fun type(): String {
return "numass"
}
override fun build(context: Context, meta: Meta): Storage {
if (meta.hasValue("path")) {
val uri = URI.create(meta.getString("path"))
val path: Path
if (uri.scheme.startsWith("ssh")) {
try {
val username = meta.getString("userName", uri.userInfo)
//String host = meta.getString("host", uri.getHost());
val port = meta.getInt("port", 22)
val env = SFTPEnvironment()
.withUsername(username)
.withPassword(meta.getString("password", "").toCharArray())
val fs = FileSystems.newFileSystem(uri, env, context.classLoader)
path = fs.getPath(uri.path)
} catch (e: Exception) {
throw RuntimeException(e)
}
} else {
path = Paths.get(uri)
}
if(!Files.exists(path)){
context.logger.info("File $path does not exist. Creating a new storage directory.")
Files.createDirectories(path)
}
return NumassStorage(context, meta, path)
} else {
context.logger.warn("A storage path not provided. Creating default root storage in the working directory")
return NumassStorage(context, meta, context.workDir)
}
}
companion object {
/**
* Build local storage with Global context. Used for tests.
*
* @param file
* @return
*/
fun buildLocal(context: Context, file: Path, readOnly: Boolean, monitor: Boolean): FileStorage {
val manager = context.load(StorageManager::class.java, Meta.empty())
return manager.buildStorage(buildStorageMeta(file.toUri(), readOnly, monitor)) as FileStorage
}
fun buildLocal(context: Context, path: String, readOnly: Boolean, monitor: Boolean): FileStorage {
val file = context.dataDir.resolve(path)
return buildLocal(context, file, readOnly, monitor)
}
fun buildStorageMeta(path: URI, readOnly: Boolean, monitor: Boolean): MetaBuilder {
return MetaBuilder("storage")
.setValue("path", path.toString())
.setValue("type", "numass")
.setValue("readOnly", readOnly)
.setValue("monitor", monitor)
}
}
}

2
numass-core/src/main/kotlin/inr/numass/data/storage/ProtoNumassPoint.kt
View File

@ -58,7 +58,7 @@ class ProtoNumassPoint(override val meta: Meta, val protoBuilder: () -> NumassPr
companion object {
fun readFile(path: Path): ProtoNumassPoint {
return fromEnvelope(NumassFileEnvelope.open(path, true))
return fromEnvelope(NumassFileEnvelope(path))
}
fun fromEnvelope(envelope: Envelope): ProtoNumassPoint {

1
numass-core/src/main/resources/META-INF/services/hep.dataforge.storage.StorageElementType Normal file
View File

@ -0,0 +1 @@
inr.numass.data.storage.NumassDirectory

1
numass-core/src/main/resources/META-INF/services/hep.dataforge.storage.api.StorageType
View File

@ -1 +0,0 @@
inr.numass.data.storage.NumassStorageFactory

1
numass-main/src/main/kotlin/inr/numass/scripts/Correlation.kt
View File

@ -23,7 +23,6 @@ import inr.numass.data.NumassDataUtils
import inr.numass.data.analyzers.SmartAnalyzer
import inr.numass.data.api.NumassEvent
import inr.numass.data.api.NumassSet
import inr.numass.data.storage.NumassStorageFactory
import org.apache.commons.math3.stat.correlation.PearsonsCorrelation
import java.util.stream.Stream

1
numass-main/src/main/kotlin/inr/numass/scripts/InversedChain.kt
View File

@ -26,7 +26,6 @@ import inr.numass.data.analyzers.NumassAnalyzer.Companion.AMPLITUDE_ADAPTER
import inr.numass.data.analyzers.SmartAnalyzer
import inr.numass.data.analyzers.withBinning
import inr.numass.data.api.NumassSet
import inr.numass.data.storage.NumassStorageFactory
import inr.numass.displayChart

1
numass-main/src/main/kotlin/inr/numass/scripts/models/DifferentialSpectrum.kt
View File

@ -28,7 +28,6 @@ import inr.numass.data.analyzers.SmartAnalyzer
import inr.numass.data.analyzers.subtractAmplitudeSpectrum
import inr.numass.data.analyzers.withBinning
import inr.numass.data.api.NumassSet
import inr.numass.data.storage.NumassStorageFactory
import inr.numass.displayChart
fun main(args: Array<String>) {

1
numass-main/src/main/kotlin/inr/numass/scripts/timeanalysis/AnalyzePoint.kt
View File

@ -13,7 +13,6 @@ import inr.numass.data.api.NumassPoint
import inr.numass.data.api.NumassSet
import inr.numass.data.api.SimpleNumassPoint
import inr.numass.data.channel
import inr.numass.data.storage.NumassStorageFactory
fun main(args: Array<String>) {

1
numass-main/src/main/kotlin/inr/numass/scripts/timeanalysis/Histogram.kt
View File

@ -23,7 +23,6 @@ import inr.numass.NumassPlugin
import inr.numass.data.NumassDataUtils
import inr.numass.data.analyzers.TimeAnalyzer
import inr.numass.data.api.NumassSet
import inr.numass.data.storage.NumassStorageFactory
import kotlin.streams.asStream
fun main(args: Array<String>) {

1
numass-main/src/main/kotlin/inr/numass/scripts/tristan/ProtoPoint.kt
View File

@ -21,7 +21,6 @@ import hep.dataforge.toList
import inr.numass.data.api.NumassPoint
import inr.numass.data.channel
import inr.numass.data.storage.NumassDataLoader
import inr.numass.data.storage.NumassStorageFactory
fun main(args: Array<String>) {
val storage = NumassStorageFactory.buildLocal(Global, "D:\\Work\\Numass\\data\\2018_04\\Adiabacity_19\\", true, false)

3
numass-main/src/main/kotlin/inr/numass/scripts/utils/ScanTree.kt
View File

@ -6,10 +6,7 @@ import hep.dataforge.meta.Meta
import hep.dataforge.meta.MetaBuilder
import hep.dataforge.meta.MetaUtils
import hep.dataforge.meta.buildMeta
import hep.dataforge.storage.api.Storage
import hep.dataforge.useValue
import inr.numass.data.storage.NumassDataLoader
import inr.numass.data.storage.NumassStorageFactory
import java.io.File
import java.nio.file.Paths

2
numass-main/src/main/kotlin/inr/numass/subthreshold/Threshold.kt
View File

@ -6,7 +6,6 @@ import hep.dataforge.data.DataNode
import hep.dataforge.data.DataSet
import hep.dataforge.meta.Meta
import hep.dataforge.meta.buildMeta
import hep.dataforge.storage.commons.StorageUtils
import hep.dataforge.tables.ListTable
import hep.dataforge.tables.Table
import hep.dataforge.values.ValueMap
@ -18,7 +17,6 @@ import inr.numass.data.analyzers.withBinning
import inr.numass.data.api.NumassPoint
import inr.numass.data.api.NumassSet
import inr.numass.data.api.SimpleNumassPoint
import inr.numass.data.storage.NumassStorageFactory
import org.apache.commons.math3.analysis.ParametricUnivariateFunction
import org.apache.commons.math3.exception.DimensionMismatchException
import org.apache.commons.math3.fitting.SimpleCurveFitter

2
numass-viewer/build.gradle
View File

@ -12,7 +12,7 @@ if (!hasProperty('mainClass')) {
mainClassName = mainClass
version = "0.5.3"
version = "0.5.4"
description = "The viewer for numass data"

13
numass-viewer/src/main/kotlin/inr/numass/viewer/AmplitudeView.kt
View File

@ -7,6 +7,7 @@ import hep.dataforge.fx.plots.PlotContainer
import hep.dataforge.fx.runGoal
import hep.dataforge.fx.ui
import hep.dataforge.goals.Goal
import hep.dataforge.names.Name
import hep.dataforge.plots.PlotFrame
import hep.dataforge.plots.PlotGroup
import hep.dataforge.plots.Plottable
@ -122,9 +123,8 @@ class AmplitudeView : View(title = "Numass amplitude spectrum plot", icon = Imag
}
private fun invalidate() {
isEmpty.invalidate()
data.forEach { key, point ->
plots.computeIfAbsent(key) {
plots.getOrPut(key) {
runGoal<Plottable>("loadAmplitudeSpectrum_$key") {
val valueAxis = if (normalize) {
NumassAnalyzer.COUNT_RATE_KEY
@ -157,7 +157,7 @@ class AmplitudeView : View(title = "Numass amplitude spectrum plot", icon = Imag
frame.add(plot)
progress.invalidate()
} except {
throw it
progress.invalidate()
}
}
plots.keys.filter { !data.containsKey(it) }.forEach { remove(it) }
@ -177,10 +177,11 @@ class AmplitudeView : View(title = "Numass amplitude spectrum plot", icon = Imag
* Remove the plot and cancel loading task if it is in progress.
*/
fun remove(name: String) {
frame.remove(name);
plots[name]?.cancel();
plots.remove(name);
frame.plots.remove(Name.ofSingle(name))
plots[name]?.cancel()
plots.remove(name)
data.remove(name)
progress.invalidate()
}
/**

22
numass-viewer/src/main/kotlin/inr/numass/viewer/HVView.kt
View File

@ -4,10 +4,13 @@ import hep.dataforge.configure
import hep.dataforge.fx.dfIcon
import hep.dataforge.fx.plots.PlotContainer
import hep.dataforge.fx.runGoal
import hep.dataforge.fx.ui
import hep.dataforge.names.Name
import hep.dataforge.plots.PlotFrame
import hep.dataforge.plots.data.DataPlot
import hep.dataforge.plots.data.TimePlot
import hep.dataforge.plots.jfreechart.JFreeChartFrame
import hep.dataforge.tables.Adapters
import inr.numass.data.api.NumassSet
import javafx.collections.FXCollections
import javafx.collections.MapChangeListener
@ -33,7 +36,7 @@ class HVView : View(title = "High voltage time plot", icon = ImageView(dfIcon))
"showSymbol" to false
"showErrors" to false
}
plots.setType<DataPlot>()
plots.setType<TimePlot>()
}
private val container = PlotContainer(frame);
@ -48,20 +51,19 @@ class HVView : View(title = "High voltage time plot", icon = ImageView(dfIcon))
data.addListener { change: MapChangeListener.Change<out String, out NumassSet> ->
isEmpty.invalidate()
if (change.wasRemoved()) {
frame.remove(change.key)
frame.plots.remove(Name.ofSingle(change.key))
}
if (change.wasAdded()) {
runLater { container.progress = -1.0 }
runGoal("hvData[${change.key}]") {
change.valueAdded.getHvData().await()
} ui { hvData ->
hvData?.let {
for (dp in it) {
val plot: TimePlot = frame[change.key] as TimePlot?
?: TimePlot(change.key).apply { frame.add(this) }
plot.put(dp.getValue("timestamp").time, dp.getValue("value"))
}
change.valueAdded.getHvData()
} ui {table->
if(table!= null) {
((frame[change.key] as? DataPlot)
?: DataPlot(change.key, adapter = Adapters.buildXYAdapter("timestamp", "value")).also { frame.add(it) })
.fillData(table)
}
container.progress = 1.0;
}
}

26
numass-viewer/src/main/kotlin/inr/numass/viewer/MainView.kt
View File

@ -5,12 +5,12 @@ import hep.dataforge.context.Global
import hep.dataforge.fx.*
import hep.dataforge.fx.fragments.LogFragment
import hep.dataforge.fx.meta.MetaViewer
import hep.dataforge.storage.commons.StorageManager
import hep.dataforge.storage.Storage
import inr.numass.NumassProperties
import inr.numass.data.api.NumassPoint
import inr.numass.data.legacy.NumassFileEnvelope
import inr.numass.data.storage.NumassDataLoader
import inr.numass.data.storage.NumassStorageFactory
import inr.numass.data.storage.NumassDirectory
import javafx.beans.property.SimpleObjectProperty
import javafx.geometry.Insets
import javafx.scene.control.Alert
@ -18,7 +18,6 @@ import javafx.scene.layout.Priority
import javafx.scene.text.Font
import javafx.stage.DirectoryChooser
import javafx.stage.FileChooser
import kotlinx.coroutines.experimental.async
import org.controlsfx.control.StatusBar
import tornadofx.*
import java.io.File
@ -68,7 +67,7 @@ class MainView(val context: Context = Global.getContext("viewer")) : View(title
if (rootDir != null) {
NumassProperties.setNumassProperty("numass.viewer.lastPath", rootDir.absolutePath)
async {
kotlinx.coroutines.experimental.launch {
runLater {
path = rootDir.toPath()
}
@ -97,7 +96,7 @@ class MainView(val context: Context = Global.getContext("viewer")) : View(title
val file = chooser.showOpenDialog(primaryStage.scene.window)
if (file != null) {
NumassProperties.setNumassProperty("numass.viewer.lastPath", file.parentFile.absolutePath)
async {
kotlinx.coroutines.experimental.launch {
runLater {
path = file.toPath()
}
@ -149,7 +148,7 @@ class MainView(val context: Context = Global.getContext("viewer")) : View(title
runGoal("viewer.load.set[$path]") {
title = "Load set ($path)"
message = "Building numass set..."
NumassDataLoader.fromDir(context, path)
NumassDataLoader(context, null, path.fileName.toString(), path)
} ui {
contentView = SpectrumView().apply {
clear()
@ -168,25 +167,16 @@ class MainView(val context: Context = Global.getContext("viewer")) : View(title
runGoal("viewer.load.storage[$path]") {
title = "Load storage ($path)"
message = "Building numass storage tree..."
StorageManager.buildStorage(
context,
NumassStorageFactory.buildStorageMeta(path.toUri(), true, false)
)
NumassDirectory.INSTANCE.read(context, path)
} ui {
contentView = StorageView(it)
contentView = StorageView(it as Storage)
infoView = MetaViewer(it.meta)
} except {
alert(
type = Alert.AlertType.ERROR,
header = "Error during storage loading",
content = it.toString()
).show()
}
}
} else {
//Reading individual file
val envelope = try {
NumassFileEnvelope.open(path, true)
NumassFileEnvelope(path)
} catch (ex: Exception) {
runLater {
alert(

24
numass-viewer/src/main/kotlin/inr/numass/viewer/SlowControlView.kt
View File

@ -5,17 +5,12 @@ import hep.dataforge.fx.dfIcon
import hep.dataforge.fx.plots.PlotContainer
import hep.dataforge.fx.runGoal
import hep.dataforge.fx.ui
import hep.dataforge.meta.Meta
import hep.dataforge.plots.PlotGroup
import hep.dataforge.plots.data.DataPlot
import hep.dataforge.plots.jfreechart.JFreeChartFrame
import hep.dataforge.storage.api.TableLoader
import hep.dataforge.storage.api.ValueIndex
import hep.dataforge.storage.TableLoader
import hep.dataforge.tables.Adapters
import hep.dataforge.tables.ListTable
import hep.dataforge.tables.Table
import hep.dataforge.toList
import hep.dataforge.values.Values
import javafx.collections.FXCollections
import javafx.collections.MapChangeListener
import javafx.collections.ObservableMap
@ -71,20 +66,9 @@ class SlowControlView : View(title = "Numass slow control view", icon = ImageVie
}
}
private fun getData(loader: TableLoader, query: Meta = Meta.empty()): Table {
val index: ValueIndex<Values> =
if (query.hasValue("index")) {
//use custom index if needed
loader.getIndex(query.getString("index"))
} else {
//use loader default one otherwise
loader.index
}
try {
return ListTable(loader.format, index.query(query).toList())
} catch (e: Exception) {
throw RuntimeException(e)
}
private suspend fun getData(loader: TableLoader): Table {
//TODO add query
return loader.asTable()
}
operator fun set(id: String, loader: TableLoader) {

5
numass-viewer/src/main/kotlin/inr/numass/viewer/SpectrumView.kt
View File

@ -5,6 +5,7 @@ import hep.dataforge.fx.dfIcon
import hep.dataforge.fx.plots.PlotContainer
import hep.dataforge.fx.runGoal
import hep.dataforge.fx.ui
import hep.dataforge.names.Name
import hep.dataforge.plots.PlotFrame
import hep.dataforge.plots.data.DataPlot
import hep.dataforge.plots.jfreechart.JFreeChartFrame
@ -100,7 +101,7 @@ class SpectrumView : View(title = "Numass spectrum plot", icon = ImageView(dfIco
init {
data.addListener { change: MapChangeListener.Change<out String, out NumassSet> ->
if (change.wasRemoved()) {
frame.remove(change.key);
frame.plots.remove(Name.ofSingle(change.key));
}
if (change.wasAdded()) {
@ -116,7 +117,7 @@ class SpectrumView : View(title = "Numass spectrum plot", icon = ImageView(dfIco
val totalProgress = data.values.stream().mapToInt { it.points.size }.sum()
data.forEach { name, set ->
val plot: DataPlot = frame[name] as DataPlot? ?: DataPlot(name).apply { frame.add(this) }
val plot: DataPlot = frame.plots[Name.ofSingle(name)] as DataPlot? ?: DataPlot(name).apply { frame.add(this) }
runGoal("spectrumData[$name]") {
set.points.forEach { it.spectrum.start() }

41
numass-viewer/src/main/kotlin/inr/numass/viewer/StorageView.kt
View File

@ -2,17 +2,18 @@ package inr.numass.viewer
import hep.dataforge.fx.dfIconView
import hep.dataforge.fx.meta.MetaViewer
import hep.dataforge.fx.runGoal
import hep.dataforge.meta.Meta
import hep.dataforge.meta.Metoid
import hep.dataforge.storage.api.Loader
import hep.dataforge.storage.api.Storage
import hep.dataforge.storage.api.TableLoader
import hep.dataforge.storage.Storage
import hep.dataforge.storage.TableLoader
import hep.dataforge.storage.files.FileTableLoader
import inr.numass.data.api.NumassPoint
import inr.numass.data.api.NumassSet
import inr.numass.data.storage.NumassDataLoader
import javafx.beans.property.SimpleBooleanProperty
import javafx.scene.control.ContextMenu
import javafx.scene.control.TreeItem
import kotlinx.coroutines.experimental.runBlocking
import tornadofx.*
class StorageView(val storage: Storage) : View(title = "Numass storage", icon = dfIconView) {
@ -33,11 +34,11 @@ class StorageView(val storage: Storage) : View(title = "Numass storage", icon =
val checkedProperty = SimpleBooleanProperty(false)
var checked by checkedProperty
val infoView: UIComponent? by lazy {
val infoView: UIComponent by lazy {
when (content) {
is CachedPoint -> PointInfoView(content)
is Metoid -> MetaViewer(content.meta, title = "Meta view: $id")
else -> null
else -> MetaViewer(Meta.empty(), title = "Meta view: $id")
}
}
@ -73,8 +74,8 @@ class StorageView(val storage: Storage) : View(title = "Numass storage", icon =
val children: List<Container>? by lazy {
when (content) {
is Storage -> (content.shelves().sorted() + content.loaders().sorted()).map { buildContainer(it, this) }
is CachedSet -> content.points
is Storage -> (runBlocking { content.getChildren() }.sortedBy { it.name }).map { buildContainer(it, this) }
is NumassSet -> content.points
.sortedBy { it.index }
.map { buildContainer(it, this) }
.toList()
@ -82,9 +83,7 @@ class StorageView(val storage: Storage) : View(title = "Numass storage", icon =
}
}
val hasChildren: Boolean
get() = (content is Storage) || (content is NumassPoint)
val hasChildren: Boolean = (content is Storage) || (content is NumassSet)
}
@ -93,9 +92,7 @@ class StorageView(val storage: Storage) : View(title = "Numass storage", icon =
//isShowRoot = false
root = TreeItem(Container(storage.name, storage))
root.isExpanded = true
runGoal("viewer.storage.populateTree") {
populate { parent -> parent.value.children }
}
lazyPopulate(leafCheck = { !it.value.hasChildren }) { it.value.children }
cellFormat { value ->
when (value.content) {
is Storage -> {
@ -131,11 +128,9 @@ class StorageView(val storage: Storage) : View(title = "Numass storage", icon =
this@cellFormat.treeItem.uncheckAll()
}
}
value.infoView?.let {
item("Info") {
action {
it.openModal(escapeClosesWindow = true)
}
item("Info") {
action {
value.infoView.openModal(escapeClosesWindow = true)
}
}
}
@ -181,19 +176,19 @@ class StorageView(val storage: Storage) : View(title = "Numass storage", icon =
}
is NumassSet -> {
val id: String = if (content is NumassDataLoader) {
content.path.toString()
content.fullName.unescaped
} else {
content.name
}
Container(id, content as? CachedSet ?: CachedSet(content))
}
is NumassPoint -> {
Container("${parent.id}/${content.voltage}[${content.index}]", content as? CachedPoint ?: CachedPoint(content))
Container("${parent.id}/${content.voltage}[${content.index}]", content as? CachedPoint
?: CachedPoint(content))
}
is Loader -> {
is FileTableLoader -> {
Container(content.path.toString(), content);
}
else -> throw IllegalArgumentException("Unknown content type: ${content::class.java}");
}
}

12
numass-viewer/src/main/kotlin/inr/numass/viewer/test/ComponentTest.kt
View File

@ -2,14 +2,16 @@ package inr.numass.viewer.test
import hep.dataforge.context.Global
import hep.dataforge.fx.dfIcon
import hep.dataforge.nullable
import hep.dataforge.tables.Table
import inr.numass.data.api.NumassPoint
import inr.numass.data.api.NumassSet
import inr.numass.data.storage.NumassStorageFactory
import inr.numass.data.storage.NumassDirectory
import inr.numass.viewer.*
import javafx.application.Application
import javafx.scene.image.ImageView
import tornadofx.*
import java.io.File
import java.util.concurrent.ConcurrentHashMap
class ViewerComponentsTestApp : App(ViewerComponentsTest::class)
@ -31,10 +33,10 @@ class ViewerComponentsTest : View(title = "Numass viewer test", icon = ImageView
top {
button("Click me!") {
action {
runAsync {
val set: NumassSet = NumassStorageFactory.buildLocal(Global, "D:\\Work\\Numass\\data\\2017_05\\Fill_2", true, true)
.provide("loader::set_2", NumassSet::class.java)
.orElseThrow { RuntimeException("err") }
kotlinx.coroutines.experimental.launch {
val set: NumassSet = NumassDirectory.INSTANCE.read(Global, File("D:\\Work\\Numass\\data\\2017_05\\Fill_2").toPath())
?.provide("loader::set_2", NumassSet::class.java).nullable
?: kotlin.error("Error")
update(set);
}
}