Working again!

build.gradle.kts

@@ -1,24 +1,25 @@
 plugins {
-    id("ru.mipt.npm.project")
+    id("ru.mipt.npm.gradle.project")
 }
 
 allprojects {
     repositories{
         mavenLocal()
+        maven("https://repo.kotlin.link")
     }
 
     group = "ru.inr.mass"
     version = "0.1.0-SNAPSHOT"
 }
 
-val dataforgeVersion by extra("0.3.0-dev-3")
+val dataforgeVersion by extra("0.4.0-dev-2")
+val kmathVersion by extra("0.2.1")
 
 apiValidation{
     validationDisabled = true
 }
 
-val vcs by project.extra("https://mipt-npm.jetbrains.space/p/numass/code/numass/")
-
 ksciencePublish{
-    spaceRepo = "https://maven.pkg.jetbrains.space/mipt-npm/p/numass/maven"
+    configurePublications("https://mipt-npm.jetbrains.space/p/numass/code/numass/")
+    space("https://maven.pkg.jetbrains.space/mipt-npm/p/numass/maven")
 }

numass-data-model/build.gradle.kts

@@ -1,6 +1,6 @@
 plugins {
     kotlin("multiplatform")
-    id("ru.mipt.npm.kscience")
+    id("ru.mipt.npm.gradle.common")
 }
 
 kscience {
@@ -12,11 +12,16 @@ val dataforgeVersion: String by rootProject.extra
 kotlin.sourceSets {
     commonMain {
         dependencies {
-            api("hep.dataforge:dataforge-context:$dataforgeVersion")
+            api("space.kscience:dataforge-context:$dataforgeVersion")
-            api("hep.dataforge:dataforge-data:$dataforgeVersion")
+            api("space.kscience:dataforge-data:$dataforgeVersion")
             api("org.jetbrains.kotlinx:kotlinx-datetime:0.1.1")
         }
     }
+    jvmMain{
+        dependencies{
+            api("ch.qos.logback:logback-classic:1.2.3")
+        }
+    }
 }
 
 

numass-data-model/src/commonMain/kotlin/ru/inr/mass/data/api/NumassPoint.kt

@@ -16,11 +16,11 @@
 
 package ru.inr.mass.data.api
 
-import hep.dataforge.meta.Meta
-import hep.dataforge.meta.double
-import hep.dataforge.meta.get
-import hep.dataforge.meta.int
 import kotlinx.coroutines.flow.*
+import space.kscience.dataforge.meta.Meta
+import space.kscience.dataforge.meta.double
+import space.kscience.dataforge.meta.get
+import space.kscience.dataforge.meta.int
 import kotlin.time.Duration
 import kotlin.time.nanoseconds
 

numass-data-model/src/commonMain/kotlin/ru/inr/mass/data/api/NumassSet.kt

@@ -5,13 +5,13 @@
  */
 package ru.inr.mass.data.api
 
-import hep.dataforge.meta.Meta
-import hep.dataforge.meta.get
-import hep.dataforge.meta.long
-import hep.dataforge.names.Name
-import hep.dataforge.names.toName
-import hep.dataforge.provider.Provider
 import kotlinx.datetime.Instant
+import space.kscience.dataforge.meta.Meta
+import space.kscience.dataforge.meta.get
+import space.kscience.dataforge.meta.long
+import space.kscience.dataforge.names.Name
+import space.kscience.dataforge.names.toName
+import space.kscience.dataforge.provider.Provider
 
 /**
  * A single set of numass measurements together with metadata.

numass-data-model/src/commonMain/kotlin/ru/inr/mass/data/api/SimpleNumassPoint.kt

@@ -1,9 +1,9 @@
 package ru.inr.mass.data.api
 
-import hep.dataforge.meta.Meta
 import kotlinx.coroutines.flow.Flow
 import kotlinx.coroutines.flow.asFlow
 import kotlinx.datetime.Instant
+import space.kscience.dataforge.meta.Meta
 
 /**
  * A simple static implementation of NumassPoint

numass-data-proto/build.gradle.kts

@@ -1,6 +1,6 @@
 plugins {
     kotlin("jvm")
-    id("ru.mipt.npm.kscience")
+    id("ru.mipt.npm.gradle.common")
     id("com.squareup.wire") version "3.5.0"
 }
 
@@ -12,7 +12,7 @@ val dataforgeVersion: String by rootProject.extra
 
 dependencies {
     api(project(":numass-data-model"))
-    api("hep.dataforge:dataforge-io:$dataforgeVersion")
+    api("space.kscience:dataforge-io:$dataforgeVersion")
 }
 
 wire{

numass-data-proto/src/main/kotlin/ru/inr/mass/data/proto/HVEntry.kt

@@ -1,11 +1,10 @@
 package ru.inr.mass.data.proto
 
-import hep.dataforge.io.Envelope
-import hep.dataforge.meta.get
-import hep.dataforge.meta.string
 import kotlinx.datetime.Instant
 import kotlinx.datetime.toInstant
-import kotlinx.io.asInputStream
+import space.kscience.dataforge.io.Envelope
+import space.kscience.dataforge.meta.get
+import space.kscience.dataforge.meta.string
 
 public data class HVEntry(val timestamp: Instant, val value: Double, val channel: Int = 1) {
     public companion object {
@@ -19,7 +18,7 @@ public data class HVEntry(val timestamp: Instant, val value: Double, val channel
             return buildList {
                 envelope.data?.read {
                     //Some problems with readLines
-                    asInputStream().bufferedReader().lines().forEach { str->
+                    lines().forEach { str ->
                         add(readString(str))
                     }
                 }

numass-data-proto/src/main/kotlin/ru/inr/mass/data/proto/NumassDirectorySet.kt

@@ -1,13 +1,15 @@
 package ru.inr.mass.data.proto
 
-import hep.dataforge.context.Context
-import hep.dataforge.context.logger
-import hep.dataforge.io.io
-import hep.dataforge.io.readEnvelopeFile
-import hep.dataforge.meta.DFExperimental
-import hep.dataforge.meta.Meta
 import ru.inr.mass.data.api.NumassPoint
 import ru.inr.mass.data.api.NumassSet
+import space.kscience.dataforge.context.Context
+import space.kscience.dataforge.context.error
+import space.kscience.dataforge.context.logger
+import space.kscience.dataforge.context.warn
+import space.kscience.dataforge.io.io
+import space.kscience.dataforge.io.readEnvelopeFile
+import space.kscience.dataforge.meta.Meta
+import space.kscience.dataforge.misc.DFExperimental
 import java.nio.file.Files
 import java.nio.file.Path
 import kotlin.io.path.*

numass-data-proto/src/main/kotlin/ru/inr/mass/data/proto/NumassProtoPlugin.kt

@@ -1,13 +1,13 @@
 package ru.inr.mass.data.proto
 
-import hep.dataforge.context.AbstractPlugin
+import space.kscience.dataforge.context.AbstractPlugin
-import hep.dataforge.context.Context
+import space.kscience.dataforge.context.Context
-import hep.dataforge.context.PluginFactory
+import space.kscience.dataforge.context.PluginFactory
-import hep.dataforge.context.PluginTag
+import space.kscience.dataforge.context.PluginTag
-import hep.dataforge.io.EnvelopeFormatFactory
+import space.kscience.dataforge.io.EnvelopeFormatFactory
-import hep.dataforge.io.IOPlugin
+import space.kscience.dataforge.io.IOPlugin
-import hep.dataforge.meta.Meta
+import space.kscience.dataforge.meta.Meta
-import hep.dataforge.names.Name
+import space.kscience.dataforge.names.Name
 import kotlin.reflect.KClass
 
 public class NumassProtoPlugin : AbstractPlugin() {

numass-data-proto/src/main/kotlin/ru/inr/mass/data/proto/ProtoNumassPoint.kt

@@ -16,18 +16,17 @@
 
 package ru.inr.mass.data.proto
 
-import hep.dataforge.io.Envelope
+import io.ktor.utils.io.core.readBytes
-import hep.dataforge.meta.*
 import kotlinx.coroutines.flow.*
 import kotlinx.coroutines.runBlocking
 import kotlinx.datetime.DateTimeUnit
 import kotlinx.datetime.Instant
 import kotlinx.datetime.plus
-import kotlinx.io.asInputStream
-import kotlinx.io.readByteArray
 import okio.ByteString
 import org.slf4j.LoggerFactory
 import ru.inr.mass.data.api.*
+import space.kscience.dataforge.io.Envelope
+import space.kscience.dataforge.meta.*
 import java.io.ByteArrayInputStream
 import java.io.ByteArrayOutputStream
 import java.io.InputStream
@@ -85,7 +84,7 @@ internal class ProtoNumassPoint(
                 val inflater = Inflater()
 
                 val array: ByteArray = data?.read {
-                    readByteArray()
+                    readBytes()
                 } ?: ByteArray(0)
 
                 inflater.setInput(array)

numass-data-proto/src/main/kotlin/ru/inr/mass/data/proto/TaggedNumassEnvelopeFormat.kt

@@ -16,15 +16,15 @@
 
 package ru.inr.mass.data.proto
 
-import hep.dataforge.context.Context
+import io.ktor.utils.io.core.*
-import hep.dataforge.io.*
+import space.kscience.dataforge.context.Context
-import hep.dataforge.meta.Meta
+import space.kscience.dataforge.io.*
-import hep.dataforge.meta.get
+import space.kscience.dataforge.meta.Meta
-import hep.dataforge.meta.string
+import space.kscience.dataforge.meta.get
-import hep.dataforge.names.Name
+import space.kscience.dataforge.meta.string
-import hep.dataforge.names.plus
+import space.kscience.dataforge.names.Name
-import hep.dataforge.names.toName
+import space.kscience.dataforge.names.plus
-import kotlinx.io.*
+import space.kscience.dataforge.names.toName
 import java.util.*
 
 
@@ -71,7 +71,7 @@ internal class TaggedNumassEnvelopeFormat(private val io: IOPlugin) : EnvelopeFo
         val metaFormat = io.resolveMetaFormat(tag.metaFormatKey)
             ?: error("Meta format with key ${tag.metaFormatKey} not found")
 
-        val meta: Meta = metaFormat.readObject(input.limit(tag.metaSize.toInt()))
+        val meta: Meta = metaFormat.readObject(input.readBinary(tag.metaSize.toInt()))
 
         val data = input.readBinary(tag.dataSize.toInt())
 
@@ -88,7 +88,7 @@ internal class TaggedNumassEnvelopeFormat(private val io: IOPlugin) : EnvelopeFo
                 ?: error("Meta format with key ${tag.metaFormatKey} not found")
         }
 
-        val meta: Meta = metaFormat.readObject(input.limit(tag.metaSize.toInt()))
+        val meta: Meta = metaFormat.readObject(input.readBinary(tag.metaSize.toInt()))
 
 
         return PartialEnvelope(meta, 30u + tag.metaSize, tag.dataSize)
@@ -134,9 +134,9 @@ internal class TaggedNumassEnvelopeFormat(private val io: IOPlugin) : EnvelopeFo
             return Tag(metaFormatKey, metaLength, dataLength)
         }
 
-        override fun peekFormat(io: IOPlugin, input: Input): EnvelopeFormat? {
+        override fun peekFormat(io: IOPlugin, binary: Binary): EnvelopeFormat? {
             return try {
-                input.preview {
+                binary.read {
                     val header = readRawString(30)
                     if (header.startsWith(START_SEQUENCE) && header.endsWith(END_SEQUENCE)) {
                         TaggedNumassEnvelopeFormat(io)

numass-data-proto/src/main/kotlin/ru/inr/mass/data/proto/io.kt

@@ -0,0 +1,37 @@
+package ru.inr.mass.data.proto
+
+import io.ktor.utils.io.core.*
+import java.io.InputStream
+
+// TODO move to dataforge-io
+
+/**
+ * Sequentially read Utf8 lines from the input until it is exhausted
+ */
+public fun Input.lines(): Sequence<String> = sequence {
+    while (!endOfInput) {
+        readUTF8Line()?.let { yield(it) }
+    }
+}
+
+private class InputAsInputStream(val input: Input) : InputStream() {
+
+
+    override fun read(): Int = input.run {
+        if (endOfInput) {
+            -1
+        } else {
+            readUByte().toInt()
+        }
+    }
+
+    override fun readAllBytes(): ByteArray = input.readBytes()
+
+    override fun read(b: ByteArray): Int = input.readAvailable(b)
+
+    override fun close() {
+        input.close()
+    }
+}
+
+public fun Input.asInputStream(): InputStream = InputAsInputStream(this)

numass-data-proto/src/test/kotlin/ru/inr/mass/data/proto/TestNumassDirectory.kt

@@ -1,11 +1,11 @@
 package ru.inr.mass.data.proto
 
-import hep.dataforge.context.Context
-import hep.dataforge.meta.get
-import hep.dataforge.meta.string
-import hep.dataforge.meta.value
-import hep.dataforge.values.ListValue
 import org.junit.jupiter.api.Test
+import space.kscience.dataforge.context.Context
+import space.kscience.dataforge.meta.get
+import space.kscience.dataforge.meta.string
+import space.kscience.dataforge.meta.value
+import space.kscience.dataforge.values.ListValue
 import java.nio.file.Path
 import kotlin.test.assertEquals
 

numass-workspace/build.gradle.kts

@@ -1,11 +1,8 @@
 plugins {
     kotlin("jvm")
-    id("ru.mipt.npm.kscience")
+    id("ru.mipt.npm.gradle.common")
     id("com.github.johnrengelman.shadow") version "6.1.0"
-}
+    `maven-publish`
-
-kscience {
-    publish()
 }
 
 kotlin {
@@ -13,13 +10,13 @@ kotlin {
 }
 
 val dataforgeVersion: String by rootProject.extra
-val plotlyVersion: String by rootProject.extra("0.3.1-dev-5")
+val plotlyVersion: String by rootProject.extra("0.4.0-dev-1")
-val kmathVersion: String by rootProject.extra("0.2.0-dev-6")
+val kmathVersion: String by rootProject.extra
 
 dependencies {
     implementation(project(":numass-data-proto"))
-    implementation("hep.dataforge:dataforge-workspace:$dataforgeVersion")
+    implementation("space.kscience:dataforge-workspace:$dataforgeVersion")
-    implementation("kscience.plotlykt:plotlykt-core:$plotlyVersion")
+    implementation("space.kscience:plotlykt-core:$plotlyVersion")
-    implementation("kscience.kmath:kmath-histograms:$kmathVersion")
+    implementation("space.kscience:kmath-histograms:$kmathVersion")
-    implementation("kscience.kmath:kmath-for-real:$kmathVersion")
+    implementation("space.kscience:kmath-for-real:$kmathVersion")
 }

numass-workspace/numass.json

@@ -3,7 +3,7 @@
         "kscience.plotly.*",
         "kscience.plotly.models.*",
         "kscience.plotly.JupyterPlotly",
-        "hep.dataforge.meta.*",
+        "space.kscience.dataforge.meta.*",
         "kotlinx.html.*",
         "ru.inr.mass.workspace.*"
     ],

numass-workspace/src/main/kotlin/ru/inr/mass/scripts/demo.kt

@@ -1,14 +1,16 @@
 package ru.inr.mass.workspace
 
-import hep.dataforge.data.await
+import ru.inr.mass.data.proto.NumassDirectorySet
-import hep.dataforge.names.toName
+import space.kscience.dataforge.data.DataTree
-import kscience.plotly.Plotly
+import space.kscience.dataforge.data.await
-import kscience.plotly.makeFile
+import space.kscience.dataforge.data.getData
+import space.kscience.plotly.Plotly
+import space.kscience.plotly.makeFile
 
 suspend fun main() {
-    val repo = readNumassRepository("D:\\Work\\Numass\\data\\2018_04")
+    val repo: DataTree<NumassDirectorySet> = readNumassRepository("D:\\Work\\Numass\\data\\2018_04")
     //val dataPath = Path.of("D:\\Work\\Numass\\data\\2018_04\\Adiabacity_19\\set_4\\")
     //val testSet = NUMASS.context.readNumassDirectory(dataPath)
-    val testSet = repo.getData("Adiabacity_19.set_4".toName())!!.await()
+    val testSet = repo.getData("Adiabacity_19.set_4")?.await() ?: error("Not found")
     Plotly.numassDirectory(testSet).makeFile()
 }

numass-workspace/src/main/kotlin/ru/inr/mass/scripts/run_2020_12.kt

@@ -0,0 +1,20 @@
+package ru.inr.mass.scripts
+
+import kotlinx.coroutines.flow.collect
+import ru.inr.mass.data.proto.NumassDirectorySet
+import ru.inr.mass.workspace.readNumassRepository
+import space.kscience.dataforge.data.DataTree
+import space.kscience.dataforge.data.filter
+import space.kscience.dataforge.meta.get
+import space.kscience.dataforge.meta.string
+
+suspend fun main() {
+    val repo: DataTree<NumassDirectorySet> = readNumassRepository("D:\\Work\\Numass\\data\\2018_04")
+    val filtered = repo.filter { _, data ->
+        data.meta["operator"].string?.startsWith("Vas") ?: false
+    }
+
+    filtered.flow().collect {
+        println(it)
+    }
+}

numass-workspace/src/main/kotlin/ru/inr/mass/workspace/amplitudeSpectrum.kt

@@ -1,14 +1,16 @@
-@file:Suppress("EXPERIMENTAL_API_USAGE")
-
 package ru.inr.mass.workspace
 
-import hep.dataforge.context.logger
 import kotlinx.coroutines.flow.collect
 import kotlinx.coroutines.runBlocking
-import kscience.kmath.histogram.UnivariateHistogram
-import kscience.kmath.structures.RealBuffer
-import kscience.kmath.structures.asBuffer
 import ru.inr.mass.data.api.NumassPoint
+import space.kscience.dataforge.context.logger
+import space.kscience.dataforge.context.warn
+import space.kscience.kmath.histogram.UnivariateHistogram
+import space.kscience.kmath.histogram.center
+import space.kscience.kmath.histogram.put
+import space.kscience.kmath.structures.RealBuffer
+import space.kscience.kmath.structures.asBuffer
+
 
 /**
  * Build an amplitude spectrum
@@ -19,8 +21,8 @@ fun NumassPoint.spectrum(): UnivariateHistogram = UnivariateHistogram.uniform(1.
     }
 }
 
-operator fun UnivariateHistogram.component1(): RealBuffer = map {it.position}.toDoubleArray().asBuffer()
+operator fun UnivariateHistogram.component1(): RealBuffer = bins.map { it.domain.center }.toDoubleArray().asBuffer()
-operator fun UnivariateHistogram.component2(): RealBuffer = map { it.value }.toDoubleArray().asBuffer()
+operator fun UnivariateHistogram.component2(): RealBuffer = bins.map { it.value }.toDoubleArray().asBuffer()
 
 fun Collection<NumassPoint>.spectrum(): UnivariateHistogram {
     if (distinctBy { it.voltage }.size != 1) {
@@ -43,8 +45,8 @@ fun UnivariateHistogram.reShape(
     binSize: Int,
     channelRange: IntRange,
 ): UnivariateHistogram = UnivariateHistogram.uniform(binSize.toDouble()) {
-    this@reShape.filter { it.position.toInt() in channelRange }.forEach { bin ->
+    this@reShape.bins.filter { it.domain.center.toInt() in channelRange }.forEach { bin ->
-        if(bin.size > binSize.toDouble()) error("Can't reShape the spectrum with increased binning")
+        if (bin.domain.volume() > binSize.toDouble()) error("Can't reShape the spectrum with increased binning")
-        putMany(bin.position, bin.value.toInt())
+        putValue(bin.domain.center, bin.value)
     }
 }

numass-workspace/src/main/kotlin/ru/inr/mass/workspace/files.kt

@@ -1,14 +1,14 @@
 package ru.inr.mass.workspace
 
-import hep.dataforge.data.ActiveDataTree
-import hep.dataforge.data.DataTree
-import hep.dataforge.data.emitStatic
-import hep.dataforge.names.Name
-import hep.dataforge.names.NameToken
 import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.withContext
 import ru.inr.mass.data.proto.NumassDirectorySet
 import ru.inr.mass.data.proto.readNumassDirectory
+import space.kscience.dataforge.data.ActiveDataTree
+import space.kscience.dataforge.data.DataTree
+import space.kscience.dataforge.data.static
+import space.kscience.dataforge.names.Name
+import space.kscience.dataforge.names.NameToken
 import java.nio.file.Files
 import java.nio.file.Path
 import kotlin.io.path.ExperimentalPathApi
@@ -31,7 +31,7 @@ suspend fun readNumassRepository(path: String): DataTree<NumassDirectorySet> = A
                 NameToken(segment.fileName.toString())
             })
             val value = NUMASS.context.readNumassDirectory(childPath)
-            emitStatic(name, value, value.meta)
+            static(name, value, value.meta)
         }
     }
     //TODO add file watcher

numass-workspace/src/main/kotlin/ru/inr/mass/workspace/plots.kt

@@ -2,18 +2,19 @@ package ru.inr.mass.workspace
 
 import kotlinx.html.h1
 import kotlinx.html.h2
-import kscience.kmath.histogram.UnivariateHistogram
-import kscience.kmath.misc.UnstableKMathAPI
-import kscience.plotly.*
-import kscience.plotly.models.Trace
 import ru.inr.mass.data.api.NumassPoint
 import ru.inr.mass.data.proto.HVEntry
 import ru.inr.mass.data.proto.NumassDirectorySet
+import space.kscience.kmath.histogram.UnivariateHistogram
+import space.kscience.kmath.histogram.center
+import space.kscience.kmath.misc.UnstableKMathAPI
+import space.kscience.plotly.*
+import space.kscience.plotly.models.Trace
 
 @OptIn(UnstableKMathAPI::class)
 fun Trace.fromSpectrum(histogram: UnivariateHistogram) {
-    x.numbers = histogram.map { it.position }
+    x.numbers = histogram.bins.map { it.domain.center }
-    y.numbers = histogram.map { it.value }
+    y.numbers = histogram.bins.map { it.value }
 }
 
 @OptIn(UnstableKMathAPI::class)

numass-workspace/src/main/kotlin/ru/inr/mass/workspace/workspace.kt

@@ -1,7 +1,7 @@
 package ru.inr.mass.workspace
 
-import hep.dataforge.workspace.Workspace
 import ru.inr.mass.data.proto.NumassProtoPlugin
+import space.kscience.dataforge.workspace.Workspace
 
 val NUMASS = Workspace {
     context("NUMASS") {

settings.gradle.kts

@@ -1,24 +1,22 @@
 pluginManagement {
     repositories {
+        maven("https://repo.kotlin.link")
         gradlePluginPortal()
         mavenCentral()
         jcenter()
         maven("https://dl.bintray.com/kotlin/kotlin-eap")
         maven("https://dl.bintray.com/kotlin/kotlin-dev")
-        maven("https://dl.bintray.com/mipt-npm/dataforge")
-        maven("https://dl.bintray.com/mipt-npm/kscience")
-        maven("https://dl.bintray.com/mipt-npm/dev")
     }
 
-    val toolsVersion = "0.7.3-1.4.30-RC"
+    val toolsVersion = "0.9.2"
-    val kotlinVersion = "1.4.30-RC"
+    val kotlinVersion = "1.4.31"
 
     plugins {
-        id("ru.mipt.npm.project") version toolsVersion
+        id("ru.mipt.npm.gradle.project") version toolsVersion
-        id("ru.mipt.npm.mpp") version toolsVersion
+        id("ru.mipt.npm.gradle.mpp") version toolsVersion
-        id("ru.mipt.npm.jvm") version toolsVersion
+        id("ru.mipt.npm.gradle.jvm") version toolsVersion
-        id("ru.mipt.npm.js") version toolsVersion
+        id("ru.mipt.npm.gradle.js") version toolsVersion
-        id("ru.mipt.npm.publish") version toolsVersion
+        id("ru.mipt.npm.gradle.publish") version toolsVersion
         kotlin("jvm") version kotlinVersion
         kotlin("js") version kotlinVersion
     }