NPM/numass-framework
4
0
Fork 1
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Faster file storage file type inference

Browse Source
This commit is contained in:
Alexander Nozik 2018-08-14 10:43:23 +03:00
parent e2de03fdcc
commit 2e312047cf
3 changed files with 17 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
numass-core/src/main/kotlin/inr/numass/NumassEnvelopeType.kt
View File

@ -6,7 +6,7 @@ import hep.dataforge.values.parseValue
import org.slf4j.LoggerFactory import org.slf4j.LoggerFactory
import java.io.IOException import java.io.IOException
import java.nio.ByteBuffer import java.nio.ByteBuffer
import java.nio.file.Files import java.nio.channels.FileChannel
import java.nio.file.Path import java.nio.file.Path
import java.nio.file.StandardOpenOption import java.nio.file.StandardOpenOption
import java.util.* import java.util.*
@ -101,10 +101,11 @@ class NumassEnvelopeType : EnvelopeType {
*/ */
fun infer(path: Path): EnvelopeType? { fun infer(path: Path): EnvelopeType? {
return try { return try {
Files.newInputStream(path, StandardOpenOption.READ).use { FileChannel.open(path, StandardOpenOption.READ).use {
val buffer = ByteArray(6) val buffer = it.map(FileChannel.MapMode.READ_ONLY, 0, 6)
it.read(buffer) val array = ByteArray(6)
val header = String(buffer) buffer.get(array)
val header = String(array)
when { when {
//TODO use templates from appropriate types //TODO use templates from appropriate types
header.startsWith("#!") -> NumassEnvelopeType.INSTANCE header.startsWith("#!") -> NumassEnvelopeType.INSTANCE

7
numass-core/src/main/kotlin/inr/numass/data/storage/NumassDataLoader.kt
View File

@ -58,7 +58,7 @@ class NumassDataLoader(
override val meta: Meta by lazy { override val meta: Meta by lazy {
val metaPath = path.resolve("meta") val metaPath = path.resolve("meta")
NumassEnvelopeType.infer(metaPath)?.reader?.read(metaPath)?.meta?: Meta.empty() NumassEnvelopeType.infer(metaPath)?.reader?.read(metaPath)?.meta ?: Meta.empty()
} }
override suspend fun getHvData(): Table? { override suspend fun getHvData(): Table? {
@ -76,12 +76,13 @@ class NumassDataLoader(
} }
private val pointEnvelopes: List<Envelope> private val pointEnvelopes: List<Envelope> by lazy {
get() = Files.list(path) Files.list(path)
.filter { it.fileName.toString().startsWith(POINT_FRAGMENT_NAME) } .filter { it.fileName.toString().startsWith(POINT_FRAGMENT_NAME) }
.map { .map {
NumassEnvelopeType.infer(it)?.reader?.read(it) ?: error("Can't read point file") NumassEnvelopeType.infer(it)?.reader?.read(it) ?: error("Can't read point file")
}.toList() }.toList()
}
val isReversed: Boolean val isReversed: Boolean
get() = this.meta.getBoolean("iteration_info.reverse", false) get() = this.meta.getBoolean("iteration_info.reverse", false)

8
numass-viewer/src/main/kotlin/inr/numass/viewer/StorageView.kt
View File

@ -4,6 +4,7 @@ import hep.dataforge.fx.dfIconView
import hep.dataforge.fx.meta.MetaViewer import hep.dataforge.fx.meta.MetaViewer
import hep.dataforge.meta.Meta import hep.dataforge.meta.Meta
import hep.dataforge.meta.Metoid import hep.dataforge.meta.Metoid
import hep.dataforge.names.AlphanumComparator
import hep.dataforge.storage.Storage import hep.dataforge.storage.Storage
import hep.dataforge.storage.TableLoader import hep.dataforge.storage.TableLoader
import hep.dataforge.storage.files.FileTableLoader import hep.dataforge.storage.files.FileTableLoader
@ -74,7 +75,12 @@ class StorageView(val storage: Storage) : View(title = "Numass storage", icon =
val children: List<Container>? by lazy { val children: List<Container>? by lazy {
when (content) { when (content) {
is Storage -> (runBlocking { content.getChildren() }.sortedBy { it.name }).map { buildContainer(it, this) } is Storage -> runBlocking { content.getChildren() }.map { buildContainer(it, this) }.sortedWith(
object : Comparator<Container> {
private val alphanumComparator = AlphanumComparator()
override fun compare(o1: Container, o2: Container): Int = alphanumComparator.compare(o1.id, o2.id)
}
)
is NumassSet -> content.points is NumassSet -> content.points
.sortedBy { it.index } .sortedBy { it.index }
.map { buildContainer(it, this) } .map { buildContainer(it, this) }