NPM/numass
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Working again!

Browse Source
This commit is contained in:
Alexander Nozik 2021-03-08 22:58:28 +03:00
parent 1df14dd2fa
commit a85fddb5f1
22 changed files with 180 additions and 117 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
build.gradle.kts
View File

@ -1,24 +1,25 @@
plugins {
id("ru.mipt.npm.project")
id("ru.mipt.npm.gradle.project")
}
allprojects {
repositories{
mavenLocal()
maven("https://repo.kotlin.link")
}
group = "ru.inr.mass"
version = "0.1.0-SNAPSHOT"
}
val dataforgeVersion by extra("0.3.0-dev-3")
val dataforgeVersion by extra("0.4.0-dev-2")
val kmathVersion by extra("0.2.1")
apiValidation{
validationDisabled = true
}
val vcs by project.extra("https://mipt-npm.jetbrains.space/p/numass/code/numass/")
ksciencePublish{
spaceRepo = "https://maven.pkg.jetbrains.space/mipt-npm/p/numass/maven"
configurePublications("https://mipt-npm.jetbrains.space/p/numass/code/numass/")
space("https://maven.pkg.jetbrains.space/mipt-npm/p/numass/maven")
}

11
numass-data-model/build.gradle.kts
View File

@ -1,6 +1,6 @@
plugins {
kotlin("multiplatform")
id("ru.mipt.npm.kscience")
id("ru.mipt.npm.gradle.common")
}
kscience {
@ -12,11 +12,16 @@ val dataforgeVersion: String by rootProject.extra
kotlin.sourceSets {
commonMain {
dependencies {
api("hep.dataforge:dataforge-context:$dataforgeVersion")
api("hep.dataforge:dataforge-data:$dataforgeVersion")
api("space.kscience:dataforge-context:$dataforgeVersion")
api("space.kscience:dataforge-data:$dataforgeVersion")
api("org.jetbrains.kotlinx:kotlinx-datetime:0.1.1")
}
}
jvmMain{
dependencies{
api("ch.qos.logback:logback-classic:1.2.3")
}
}
}

8
numass-data-model/src/commonMain/kotlin/ru/inr/mass/data/api/NumassPoint.kt
View File

@ -16,11 +16,11 @@
package ru.inr.mass.data.api
import hep.dataforge.meta.Meta
import hep.dataforge.meta.double
import hep.dataforge.meta.get
import hep.dataforge.meta.int
import kotlinx.coroutines.flow.*
import space.kscience.dataforge.meta.Meta
import space.kscience.dataforge.meta.double
import space.kscience.dataforge.meta.get
import space.kscience.dataforge.meta.int
import kotlin.time.Duration
import kotlin.time.nanoseconds

12
numass-data-model/src/commonMain/kotlin/ru/inr/mass/data/api/NumassSet.kt
View File

@ -5,13 +5,13 @@
*/
package ru.inr.mass.data.api
import hep.dataforge.meta.Meta
import hep.dataforge.meta.get
import hep.dataforge.meta.long
import hep.dataforge.names.Name
import hep.dataforge.names.toName
import hep.dataforge.provider.Provider
import kotlinx.datetime.Instant
import space.kscience.dataforge.meta.Meta
import space.kscience.dataforge.meta.get
import space.kscience.dataforge.meta.long
import space.kscience.dataforge.names.Name
import space.kscience.dataforge.names.toName
import space.kscience.dataforge.provider.Provider
/**
* A single set of numass measurements together with metadata.

2
numass-data-model/src/commonMain/kotlin/ru/inr/mass/data/api/SimpleNumassPoint.kt
View File

@ -1,9 +1,9 @@
package ru.inr.mass.data.api
import hep.dataforge.meta.Meta
import kotlinx.coroutines.flow.Flow
import kotlinx.coroutines.flow.asFlow
import kotlinx.datetime.Instant
import space.kscience.dataforge.meta.Meta
/**
* A simple static implementation of NumassPoint

4
numass-data-proto/build.gradle.kts
View File

@ -1,6 +1,6 @@
plugins {
kotlin("jvm")
id("ru.mipt.npm.kscience")
id("ru.mipt.npm.gradle.common")
id("com.squareup.wire") version "3.5.0"
}
@ -12,7 +12,7 @@ val dataforgeVersion: String by rootProject.extra
dependencies {
api(project(":numass-data-model"))
api("hep.dataforge:dataforge-io:$dataforgeVersion")
api("space.kscience:dataforge-io:$dataforgeVersion")
}
wire{

13
numass-data-proto/src/main/kotlin/ru/inr/mass/data/proto/HVEntry.kt
View File

@ -1,25 +1,24 @@
package ru.inr.mass.data.proto
import hep.dataforge.io.Envelope
import hep.dataforge.meta.get
import hep.dataforge.meta.string
import kotlinx.datetime.Instant
import kotlinx.datetime.toInstant
import kotlinx.io.asInputStream
import space.kscience.dataforge.io.Envelope
import space.kscience.dataforge.meta.get
import space.kscience.dataforge.meta.string
public data class HVEntry(val timestamp: Instant, val value: Double, val channel: Int = 1) {
public companion object {
public fun readString(line: String): HVEntry {
val (timeStr, channelStr, valueStr) = line.split(' ')
return HVEntry((timeStr+"Z").toInstant(), valueStr.toDouble(), channelStr.toInt())
return HVEntry((timeStr + "Z").toInstant(), valueStr.toDouble(), channelStr.toInt())
}
public fun readEnvelope(envelope: Envelope): List<HVEntry> {
check(envelope.meta["type"].string == "voltage"){"Expecting voltage type envelope"}
check(envelope.meta["type"].string == "voltage") { "Expecting voltage type envelope" }
return buildList {
envelope.data?.read {
//Some problems with readLines
asInputStream().bufferedReader().lines().forEach { str->
lines().forEach { str ->
add(readString(str))
}
}

14
numass-data-proto/src/main/kotlin/ru/inr/mass/data/proto/NumassDirectorySet.kt
View File

@ -1,13 +1,15 @@
package ru.inr.mass.data.proto
import hep.dataforge.context.Context
import hep.dataforge.context.logger
import hep.dataforge.io.io
import hep.dataforge.io.readEnvelopeFile
import hep.dataforge.meta.DFExperimental
import hep.dataforge.meta.Meta
import ru.inr.mass.data.api.NumassPoint
import ru.inr.mass.data.api.NumassSet
import space.kscience.dataforge.context.Context
import space.kscience.dataforge.context.error
import space.kscience.dataforge.context.logger
import space.kscience.dataforge.context.warn
import space.kscience.dataforge.io.io
import space.kscience.dataforge.io.readEnvelopeFile
import space.kscience.dataforge.meta.Meta
import space.kscience.dataforge.misc.DFExperimental
import java.nio.file.Files
import java.nio.file.Path
import kotlin.io.path.*

16
numass-data-proto/src/main/kotlin/ru/inr/mass/data/proto/NumassProtoPlugin.kt
View File

@ -1,13 +1,13 @@
package ru.inr.mass.data.proto
import hep.dataforge.context.AbstractPlugin
import hep.dataforge.context.Context
import hep.dataforge.context.PluginFactory
import hep.dataforge.context.PluginTag
import hep.dataforge.io.EnvelopeFormatFactory
import hep.dataforge.io.IOPlugin
import hep.dataforge.meta.Meta
import hep.dataforge.names.Name
import space.kscience.dataforge.context.AbstractPlugin
import space.kscience.dataforge.context.Context
import space.kscience.dataforge.context.PluginFactory
import space.kscience.dataforge.context.PluginTag
import space.kscience.dataforge.io.EnvelopeFormatFactory
import space.kscience.dataforge.io.IOPlugin
import space.kscience.dataforge.meta.Meta
import space.kscience.dataforge.names.Name
import kotlin.reflect.KClass
public class NumassProtoPlugin : AbstractPlugin() {

9
numass-data-proto/src/main/kotlin/ru/inr/mass/data/proto/ProtoNumassPoint.kt
View File

@ -16,18 +16,17 @@
package ru.inr.mass.data.proto
import hep.dataforge.io.Envelope
import hep.dataforge.meta.*
import io.ktor.utils.io.core.readBytes
import kotlinx.coroutines.flow.*
import kotlinx.coroutines.runBlocking
import kotlinx.datetime.DateTimeUnit
import kotlinx.datetime.Instant
import kotlinx.datetime.plus
import kotlinx.io.asInputStream
import kotlinx.io.readByteArray
import okio.ByteString
import org.slf4j.LoggerFactory
import ru.inr.mass.data.api.*
import space.kscience.dataforge.io.Envelope
import space.kscience.dataforge.meta.*
import java.io.ByteArrayInputStream
import java.io.ByteArrayOutputStream
import java.io.InputStream
@ -85,7 +84,7 @@ internal class ProtoNumassPoint(
val inflater = Inflater()
val array: ByteArray = data?.read {
readByteArray()
readBytes()
} ?: ByteArray(0)
inflater.setInput(array)

26
numass-data-proto/src/main/kotlin/ru/inr/mass/data/proto/TaggedNumassEnvelopeFormat.kt
View File

@ -16,15 +16,15 @@
package ru.inr.mass.data.proto
import hep.dataforge.context.Context
import hep.dataforge.io.*
import hep.dataforge.meta.Meta
import hep.dataforge.meta.get
import hep.dataforge.meta.string
import hep.dataforge.names.Name
import hep.dataforge.names.plus
import hep.dataforge.names.toName
import kotlinx.io.*
import io.ktor.utils.io.core.*
import space.kscience.dataforge.context.Context
import space.kscience.dataforge.io.*
import space.kscience.dataforge.meta.Meta
import space.kscience.dataforge.meta.get
import space.kscience.dataforge.meta.string
import space.kscience.dataforge.names.Name
import space.kscience.dataforge.names.plus
import space.kscience.dataforge.names.toName
import java.util.*
@ -71,7 +71,7 @@ internal class TaggedNumassEnvelopeFormat(private val io: IOPlugin) : EnvelopeFo
val metaFormat = io.resolveMetaFormat(tag.metaFormatKey)
?: error("Meta format with key ${tag.metaFormatKey} not found")
val meta: Meta = metaFormat.readObject(input.limit(tag.metaSize.toInt()))
val meta: Meta = metaFormat.readObject(input.readBinary(tag.metaSize.toInt()))
val data = input.readBinary(tag.dataSize.toInt())
@ -88,7 +88,7 @@ internal class TaggedNumassEnvelopeFormat(private val io: IOPlugin) : EnvelopeFo
?: error("Meta format with key ${tag.metaFormatKey} not found")
}
val meta: Meta = metaFormat.readObject(input.limit(tag.metaSize.toInt()))
val meta: Meta = metaFormat.readObject(input.readBinary(tag.metaSize.toInt()))
return PartialEnvelope(meta, 30u + tag.metaSize, tag.dataSize)
@ -134,9 +134,9 @@ internal class TaggedNumassEnvelopeFormat(private val io: IOPlugin) : EnvelopeFo
return Tag(metaFormatKey, metaLength, dataLength)
}
override fun peekFormat(io: IOPlugin, input: Input): EnvelopeFormat? {
override fun peekFormat(io: IOPlugin, binary: Binary): EnvelopeFormat? {
return try {
input.preview {
binary.read {
val header = readRawString(30)
if (header.startsWith(START_SEQUENCE) && header.endsWith(END_SEQUENCE)) {
TaggedNumassEnvelopeFormat(io)

37
numass-data-proto/src/main/kotlin/ru/inr/mass/data/proto/io.kt Normal file
View File

@ -0,0 +1,37 @@
package ru.inr.mass.data.proto
import io.ktor.utils.io.core.*
import java.io.InputStream
// TODO move to dataforge-io
/**
* Sequentially read Utf8 lines from the input until it is exhausted
*/
public fun Input.lines(): Sequence<String> = sequence {
while (!endOfInput) {
readUTF8Line()?.let { yield(it) }
}
}
private class InputAsInputStream(val input: Input) : InputStream() {
override fun read(): Int = input.run {
if (endOfInput) {
-1
} else {
readUByte().toInt()
}
}
override fun readAllBytes(): ByteArray = input.readBytes()
override fun read(b: ByteArray): Int = input.readAvailable(b)
override fun close() {
input.close()
}
}
public fun Input.asInputStream(): InputStream = InputAsInputStream(this)

10
numass-data-proto/src/test/kotlin/ru/inr/mass/data/proto/TestNumassDirectory.kt
View File

@ -1,11 +1,11 @@
package ru.inr.mass.data.proto
import hep.dataforge.context.Context
import hep.dataforge.meta.get
import hep.dataforge.meta.string
import hep.dataforge.meta.value
import hep.dataforge.values.ListValue
import org.junit.jupiter.api.Test
import space.kscience.dataforge.context.Context
import space.kscience.dataforge.meta.get
import space.kscience.dataforge.meta.string
import space.kscience.dataforge.meta.value
import space.kscience.dataforge.values.ListValue
import java.nio.file.Path
import kotlin.test.assertEquals

19
numass-workspace/build.gradle.kts
View File

@ -1,11 +1,8 @@
plugins {
kotlin("jvm")
id("ru.mipt.npm.kscience")
id("ru.mipt.npm.gradle.common")
id("com.github.johnrengelman.shadow") version "6.1.0"
}
kscience {
publish()
`maven-publish`
}
kotlin {
@ -13,13 +10,13 @@ kotlin {
}
val dataforgeVersion: String by rootProject.extra
val plotlyVersion: String by rootProject.extra("0.3.1-dev-5")
val kmathVersion: String by rootProject.extra("0.2.0-dev-6")
val plotlyVersion: String by rootProject.extra("0.4.0-dev-1")
val kmathVersion: String by rootProject.extra
dependencies {
implementation(project(":numass-data-proto"))
implementation("hep.dataforge:dataforge-workspace:$dataforgeVersion")
implementation("kscience.plotlykt:plotlykt-core:$plotlyVersion")
implementation("kscience.kmath:kmath-histograms:$kmathVersion")
implementation("kscience.kmath:kmath-for-real:$kmathVersion")
implementation("space.kscience:dataforge-workspace:$dataforgeVersion")
implementation("space.kscience:plotlykt-core:$plotlyVersion")
implementation("space.kscience:kmath-histograms:$kmathVersion")
implementation("space.kscience:kmath-for-real:$kmathVersion")
}

2
numass-workspace/numass.json
View File

@ -3,7 +3,7 @@
"kscience.plotly.*",
"kscience.plotly.models.*",
"kscience.plotly.JupyterPlotly",
"hep.dataforge.meta.*",
"space.kscience.dataforge.meta.*",
"kotlinx.html.*",
"ru.inr.mass.workspace.*"
],

14
numass-workspace/src/main/kotlin/ru/inr/mass/scripts/demo.kt
View File

@ -1,14 +1,16 @@
package ru.inr.mass.workspace
import hep.dataforge.data.await
import hep.dataforge.names.toName
import kscience.plotly.Plotly
import kscience.plotly.makeFile
import ru.inr.mass.data.proto.NumassDirectorySet
import space.kscience.dataforge.data.DataTree
import space.kscience.dataforge.data.await
import space.kscience.dataforge.data.getData
import space.kscience.plotly.Plotly
import space.kscience.plotly.makeFile
suspend fun main() {
val repo = readNumassRepository("D:\\Work\\Numass\\data\\2018_04")
val repo: DataTree<NumassDirectorySet> = readNumassRepository("D:\\Work\\Numass\\data\\2018_04")
//val dataPath = Path.of("D:\\Work\\Numass\\data\\2018_04\\Adiabacity_19\\set_4\\")
//val testSet = NUMASS.context.readNumassDirectory(dataPath)
val testSet = repo.getData("Adiabacity_19.set_4".toName())!!.await()
val testSet = repo.getData("Adiabacity_19.set_4")?.await() ?: error("Not found")
Plotly.numassDirectory(testSet).makeFile()
}

20
numass-workspace/src/main/kotlin/ru/inr/mass/scripts/run_2020_12.kt Normal file
View File

@ -0,0 +1,20 @@
package ru.inr.mass.scripts
import kotlinx.coroutines.flow.collect
import ru.inr.mass.data.proto.NumassDirectorySet
import ru.inr.mass.workspace.readNumassRepository
import space.kscience.dataforge.data.DataTree
import space.kscience.dataforge.data.filter
import space.kscience.dataforge.meta.get
import space.kscience.dataforge.meta.string
suspend fun main() {
val repo: DataTree<NumassDirectorySet> = readNumassRepository("D:\\Work\\Numass\\data\\2018_04")
val filtered = repo.filter { _, data ->
data.meta["operator"].string?.startsWith("Vas") ?: false
}
filtered.flow().collect {
println(it)
}
}

24
numass-workspace/src/main/kotlin/ru/inr/mass/workspace/amplitudeSpectrum.kt
View File

@ -1,14 +1,16 @@
@file:Suppress("EXPERIMENTAL_API_USAGE")
package ru.inr.mass.workspace
import hep.dataforge.context.logger
import kotlinx.coroutines.flow.collect
import kotlinx.coroutines.runBlocking
import kscience.kmath.histogram.UnivariateHistogram
import kscience.kmath.structures.RealBuffer
import kscience.kmath.structures.asBuffer
import ru.inr.mass.data.api.NumassPoint
import space.kscience.dataforge.context.logger
import space.kscience.dataforge.context.warn
import space.kscience.kmath.histogram.UnivariateHistogram
import space.kscience.kmath.histogram.center
import space.kscience.kmath.histogram.put
import space.kscience.kmath.structures.RealBuffer
import space.kscience.kmath.structures.asBuffer
/**
* Build an amplitude spectrum
@ -19,8 +21,8 @@ fun NumassPoint.spectrum(): UnivariateHistogram = UnivariateHistogram.uniform(1.
}
}
operator fun UnivariateHistogram.component1(): RealBuffer = map {it.position}.toDoubleArray().asBuffer()
operator fun UnivariateHistogram.component2(): RealBuffer = map { it.value }.toDoubleArray().asBuffer()
operator fun UnivariateHistogram.component1(): RealBuffer = bins.map { it.domain.center }.toDoubleArray().asBuffer()
operator fun UnivariateHistogram.component2(): RealBuffer = bins.map { it.value }.toDoubleArray().asBuffer()
fun Collection<NumassPoint>.spectrum(): UnivariateHistogram {
if (distinctBy { it.voltage }.size != 1) {
@ -43,8 +45,8 @@ fun UnivariateHistogram.reShape(
binSize: Int,
channelRange: IntRange,
): UnivariateHistogram = UnivariateHistogram.uniform(binSize.toDouble()) {
this@reShape.filter { it.position.toInt() in channelRange }.forEach { bin ->
if(bin.size > binSize.toDouble()) error("Can't reShape the spectrum with increased binning")
putMany(bin.position, bin.value.toInt())
this@reShape.bins.filter { it.domain.center.toInt() in channelRange }.forEach { bin ->
if (bin.domain.volume() > binSize.toDouble()) error("Can't reShape the spectrum with increased binning")
putValue(bin.domain.center, bin.value)
}
}

12
numass-workspace/src/main/kotlin/ru/inr/mass/workspace/files.kt
View File

@ -1,14 +1,14 @@
package ru.inr.mass.workspace
import hep.dataforge.data.ActiveDataTree
import hep.dataforge.data.DataTree
import hep.dataforge.data.emitStatic
import hep.dataforge.names.Name
import hep.dataforge.names.NameToken
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.withContext
import ru.inr.mass.data.proto.NumassDirectorySet
import ru.inr.mass.data.proto.readNumassDirectory
import space.kscience.dataforge.data.ActiveDataTree
import space.kscience.dataforge.data.DataTree
import space.kscience.dataforge.data.static
import space.kscience.dataforge.names.Name
import space.kscience.dataforge.names.NameToken
import java.nio.file.Files
import java.nio.file.Path
import kotlin.io.path.ExperimentalPathApi
@ -31,7 +31,7 @@ suspend fun readNumassRepository(path: String): DataTree<NumassDirectorySet> = A
NameToken(segment.fileName.toString())
})
val value = NUMASS.context.readNumassDirectory(childPath)
emitStatic(name, value, value.meta)
static(name, value, value.meta)
}
}
//TODO add file watcher

13
numass-workspace/src/main/kotlin/ru/inr/mass/workspace/plots.kt
View File

@ -2,18 +2,19 @@ package ru.inr.mass.workspace
import kotlinx.html.h1
import kotlinx.html.h2
import kscience.kmath.histogram.UnivariateHistogram
import kscience.kmath.misc.UnstableKMathAPI
import kscience.plotly.*
import kscience.plotly.models.Trace
import ru.inr.mass.data.api.NumassPoint
import ru.inr.mass.data.proto.HVEntry
import ru.inr.mass.data.proto.NumassDirectorySet
import space.kscience.kmath.histogram.UnivariateHistogram
import space.kscience.kmath.histogram.center
import space.kscience.kmath.misc.UnstableKMathAPI
import space.kscience.plotly.*
import space.kscience.plotly.models.Trace
@OptIn(UnstableKMathAPI::class)
fun Trace.fromSpectrum(histogram: UnivariateHistogram) {
x.numbers = histogram.map { it.position }
y.numbers = histogram.map { it.value }
x.numbers = histogram.bins.map { it.domain.center }
y.numbers = histogram.bins.map { it.value }
}
@OptIn(UnstableKMathAPI::class)

2
numass-workspace/src/main/kotlin/ru/inr/mass/workspace/workspace.kt
View File

@ -1,7 +1,7 @@
package ru.inr.mass.workspace
import hep.dataforge.workspace.Workspace
import ru.inr.mass.data.proto.NumassProtoPlugin
import space.kscience.dataforge.workspace.Workspace
val NUMASS = Workspace {
context("NUMASS") {

18
settings.gradle.kts
View File

@ -1,24 +1,22 @@
pluginManagement {
repositories {
maven("https://repo.kotlin.link")
gradlePluginPortal()
mavenCentral()
jcenter()
maven("https://dl.bintray.com/kotlin/kotlin-eap")
maven("https://dl.bintray.com/kotlin/kotlin-dev")
maven("https://dl.bintray.com/mipt-npm/dataforge")
maven("https://dl.bintray.com/mipt-npm/kscience")
maven("https://dl.bintray.com/mipt-npm/dev")
}
val toolsVersion = "0.7.3-1.4.30-RC"
val kotlinVersion = "1.4.30-RC"
val toolsVersion = "0.9.2"
val kotlinVersion = "1.4.31"
plugins {
id("ru.mipt.npm.project") version toolsVersion
id("ru.mipt.npm.mpp") version toolsVersion
id("ru.mipt.npm.jvm") version toolsVersion
id("ru.mipt.npm.js") version toolsVersion
id("ru.mipt.npm.publish") version toolsVersion
id("ru.mipt.npm.gradle.project") version toolsVersion
id("ru.mipt.npm.gradle.mpp") version toolsVersion
id("ru.mipt.npm.gradle.jvm") version toolsVersion
id("ru.mipt.npm.gradle.js") version toolsVersion
id("ru.mipt.npm.gradle.publish") version toolsVersion
kotlin("jvm") version kotlinVersion
kotlin("js") version kotlinVersion
}