Fixed legacy file reading

build.gradle.kts

@@ -3,11 +3,15 @@ plugins {
 }
 
 allprojects {
+    repositories{
+        mavenLocal()
+    }
+
     group = "ru.inr.mass"
     version = "0.1.0"
 }
 
-val dataforgeVersion by extra("0.3.0-dev-1")
+val dataforgeVersion by extra("0.3.0-dev-2")
 
 apiValidation{
     validationDisabled = true

numass-data-model/src/main/kotlin/ru/inr/mass/data/api/NumassSet.kt

@@ -40,20 +40,19 @@ public interface NumassSet : Iterable<NumassPoint>, Provider {
         return points.iterator()
     }
 
-
-    override val defaultTarget: String get() = NUMASS_POINT_PROVIDER_KEY
+    override val defaultTarget: String get() = NUMASS_POINT_TARGET
 
     override fun content(target: String): Map<Name, Any> {
-        return if (target == NUMASS_POINT_PROVIDER_KEY) {
+        return if (target == NUMASS_POINT_TARGET) {
             points.associateBy { "point[${it.voltage}]".toName() }
         } else {
             super.content(target)
         }
     }
 
-    companion object {
-        const val DESCRIPTION_KEY = "info"
-        const val NUMASS_POINT_PROVIDER_KEY = "point"
+    public companion object {
+        //public const val DESCRIPTION_KEY = "info"
+        public const val NUMASS_POINT_TARGET: String = "point"
     }
 }
 

numass-data-proto/src/main/kotlin/ru/inr/mass/data/proto/NumassDirectorySet.kt

@@ -10,10 +10,7 @@ import ru.inr.mass.data.api.NumassPoint
 import ru.inr.mass.data.api.NumassSet
 import java.nio.file.Files
 import java.nio.file.Path
-import kotlin.io.path.ExperimentalPathApi
-import kotlin.io.path.div
-import kotlin.io.path.exists
-import kotlin.io.path.isDirectory
+import kotlin.io.path.*
 import kotlin.streams.toList
 
 @OptIn(ExperimentalPathApi::class)
@@ -35,7 +32,9 @@ public class NumassDirectorySet internal constructor(
     }
 
     override val points: List<NumassPoint> by lazy<List<NumassPoint>> {
-        Files.list(path).filter { it.fileName.startsWith("p") }.map { pointPath ->
+        Files.list(path).filter {
+            it.fileName.name.startsWith("p")
+        }.map { pointPath ->
             try {
                 context.readNumassFile(pointPath)
             } catch (e: Exception) {

numass-data-proto/src/main/kotlin/ru/inr/mass/data/proto/NumassEnvelopeType.kt

@@ -1,139 +0,0 @@
-///*
-// * Copyright  2018 Alexander Nozik.
-// *
-// * Licensed under the Apache License, Version 2.0 (the "License");
-// * you may not use this file except in compliance with the License.
-// * You may obtain a copy of the License at
-// *
-// *      http://www.apache.org/licenses/LICENSE-2.0
-// *
-// *  Unless required by applicable law or agreed to in writing, software
-// *  distributed under the License is distributed on an "AS IS" BASIS,
-// *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// *  See the License for the specific language governing permissions and
-// *  limitations under the License.
-// */
-//
-//package ru.inr.mass.data.proto
-//
-//import hep.dataforge.io.envelopes.*
-//import hep.dataforge.values.Value
-//import org.slf4j.LoggerFactory
-//import java.io.IOException
-//import java.nio.ByteBuffer
-//import java.nio.channels.FileChannel
-//import java.nio.file.Path
-//import java.nio.file.StandardOpenOption
-//import java.util.*
-//
-///**
-// * An envelope type for legacy numass tags. Reads legacy tag and writes DF02 tags
-// */
-//class NumassEnvelopeType : EnvelopeType {
-//
-//    override val code: Int = DefaultEnvelopeType.DEFAULT_ENVELOPE_CODE
-//
-//    override val name: String = "numass"
-//
-//    override fun description(): String = "Numass legacy envelope"
-//
-//    /**
-//     * Read as legacy
-//     */
-//    override fun getReader(properties: Map<String, String>): EnvelopeReader {
-//        return NumassEnvelopeReader()
-//    }
-//
-//    /**
-//     * Write as default
-//     */
-//    override fun getWriter(properties: Map<String, String>): EnvelopeWriter {
-//        return DefaultEnvelopeWriter(this, MetaType.resolve(properties))
-//    }
-//
-//    class LegacyTag : EnvelopeTag() {
-//        override val startSequence: ByteArray
-//            get() = LEGACY_START_SEQUENCE
-//
-//        override val endSequence: ByteArray
-//            get() = LEGACY_END_SEQUENCE
-//
-//        /**
-//         * Get the length of tag in bytes. -1 means undefined size in case tag was modified
-//         *
-//         * @return
-//         */
-//        override val length: Int
-//            get() = 30
-//
-//        /**
-//         * Read leagscy version 1 tag without leading tag head
-//         *
-//         * @param buffer
-//         * @return
-//         * @throws IOException
-//         */
-//        override fun readHeader(buffer: ByteBuffer): Map<String, Value> {
-//            val res = HashMap<String, Value>()
-//
-//            val type = buffer.getInt(2)
-//            res[Envelope.TYPE_PROPERTY] = Value.of(type)
-//
-//            val metaTypeCode = buffer.getShort(10)
-//            val metaType = MetaType.resolve(metaTypeCode)
-//
-//            if (metaType != null) {
-//                res[Envelope.META_TYPE_PROPERTY] = metaType.name.parseValue()
-//            } else {
-//                LoggerFactory.getLogger(EnvelopeTag::class.java).warn("Could not resolve meta type. Using default")
-//            }
-//
-//            val metaLength = Integer.toUnsignedLong(buffer.getInt(14))
-//            res[Envelope.META_LENGTH_PROPERTY] = Value.of(metaLength)
-//            val dataLength = Integer.toUnsignedLong(buffer.getInt(22))
-//            res[Envelope.DATA_LENGTH_PROPERTY] = Value.of(dataLength)
-//            return res
-//        }
-//    }
-//
-//    private class NumassEnvelopeReader : DefaultEnvelopeReader() {
-//        override fun newTag(): EnvelopeTag {
-//            return LegacyTag()
-//        }
-//    }
-//
-//    companion object {
-//        val INSTANCE = NumassEnvelopeType()
-//
-//        val LEGACY_START_SEQUENCE = byteArrayOf('#'.toByte(), '!'.toByte())
-//        val LEGACY_END_SEQUENCE = byteArrayOf('!'.toByte(), '#'.toByte(), '\r'.toByte(), '\n'.toByte())
-//
-//        /**
-//         * Replacement for standard type infer to include legacy type
-//         *
-//         * @param path
-//         * @return
-//         */
-//        fun infer(path: Path): EnvelopeType? {
-//            return try {
-//                FileChannel.open(path, StandardOpenOption.READ).use {
-//                    val buffer = it.map(FileChannel.MapMode.READ_ONLY, 0, 6)
-//                    when {
-//                        //TODO use templates from appropriate types
-//                        buffer.get(0) == '#'.toByte() && buffer.get(1) == '!'.toByte() -> INSTANCE
-//                        buffer.get(0) == '#'.toByte() && buffer.get(1) == '!'.toByte() &&
-//                                buffer.get(4) == 'T'.toByte() && buffer.get(5) == 'L'.toByte() -> TaglessEnvelopeType.INSTANCE
-//                        buffer.get(0) == '#'.toByte() && buffer.get(1) == '~'.toByte() -> DefaultEnvelopeType.INSTANCE
-//                        else -> null
-//                    }
-//                }
-//            } catch (ex: Exception) {
-//                LoggerFactory.getLogger(EnvelopeType::class.java).warn("Could not infer envelope type of file {} due to exception: {}", path, ex)
-//                null
-//            }
-//
-//        }
-//
-//    }
-//
-//}

numass-data-proto/src/main/kotlin/ru/inr/mass/data/proto/NumassProtoPlugin.kt

@@ -4,14 +4,24 @@ import hep.dataforge.context.AbstractPlugin
 import hep.dataforge.context.Context
 import hep.dataforge.context.PluginFactory
 import hep.dataforge.context.PluginTag
+import hep.dataforge.io.EnvelopeFormatFactory
 import hep.dataforge.io.IOPlugin
 import hep.dataforge.meta.Meta
+import hep.dataforge.names.Name
 import kotlin.reflect.KClass
 
 public class NumassProtoPlugin : AbstractPlugin() {
     public val io: IOPlugin by require(IOPlugin)
     override val tag: PluginTag get() = Companion.tag
 
+    override fun content(target: String): Map<Name, Any> {
+        return if(target== EnvelopeFormatFactory.ENVELOPE_FORMAT_TYPE){
+            mapOf(TaggedNumassEnvelopeFormat.name to TaggedNumassEnvelopeFormat)
+        } else{
+            super.content(target)
+        }
+    }
+
     public companion object : PluginFactory<NumassProtoPlugin> {
         override fun invoke(meta: Meta, context: Context): NumassProtoPlugin = NumassProtoPlugin()
         override val tag: PluginTag = PluginTag("numass-proto", group = "ru.inr.mass")

numass-data-proto/src/main/kotlin/ru/inr/mass/data/proto/TaggedNumassEnvelopeFormat.kt

@@ -0,0 +1,295 @@
+/*
+ * Copyright  2018 Alexander Nozik.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package ru.inr.mass.data.proto
+
+import hep.dataforge.context.Context
+import hep.dataforge.io.*
+import hep.dataforge.meta.Meta
+import hep.dataforge.meta.get
+import hep.dataforge.meta.string
+import hep.dataforge.names.Name
+import hep.dataforge.names.plus
+import hep.dataforge.names.toName
+import kotlinx.io.*
+import java.util.*
+
+
+/**
+ * A streaming-friendly envelope format with a short binary tag.
+ * TODO add description
+ */
+public class TaggedNumassEnvelopeFormat(private val io: IOPlugin) : EnvelopeFormat {
+
+//    private val metaFormat = io.metaFormat(metaFormatKey)
+//        ?: error("Meta format with key $metaFormatKey could not be resolved in $io")
+
+
+    private fun Tag.toBinary() = Binary(24) {
+        writeRawString(START_SEQUENCE)
+        writeRawString("DFNU")
+        writeShort(metaFormatKey)
+        writeUInt(metaSize)
+        writeUInt(dataSize.toUInt())
+        writeRawString(END_SEQUENCE)
+    }
+
+    override fun writeEnvelope(
+        output: Output,
+        envelope: Envelope,
+        metaFormatFactory: MetaFormatFactory,
+        formatMeta: Meta,
+    ) {
+        error("Don't write legacy formats")
+//        val metaFormat = metaFormatFactory.invoke(formatMeta, io.context)
+//        val metaBytes = metaFormat.toBinary(envelope.meta)
+//        val actualSize: ULong = (envelope.data?.size ?: 0).toULong()
+//        val tag = Tag(metaFormatFactory.key, metaBytes.size.toUInt() + 2u, actualSize)
+//        output.writeBinary(tag.toBinary())
+//        output.writeBinary(metaBytes)
+//        output.writeRawString("\r\n")
+//        envelope.data?.let {
+//            output.writeBinary(it)
+//        }
+//        output.flush()
+    }
+
+    /**
+     * Read an envelope from input into memory
+     *
+     * @param input an input to read from
+     * @param formats a collection of meta formats to resolve
+     */
+    override fun readObject(input: Input): Envelope {
+        val tag = input.readTag()
+
+        val metaFormat = io.resolveMetaFormat(tag.metaFormatKey)
+            ?: error("Meta format with key ${tag.metaFormatKey} not found")
+
+        val meta: Meta = metaFormat.readObject(input.limit(tag.metaSize.toInt()))
+
+        val data = input.readBinary(tag.dataSize.toInt())
+
+        return SimpleEnvelope(meta, data)
+    }
+
+    override fun readPartial(input: Input): PartialEnvelope {
+        val tag = input.readTag()
+
+        val metaFormat = if (tag.metaFormatKey == 1.toShort()) {
+            JsonMetaFormat
+        } else {
+            io.resolveMetaFormat(tag.metaFormatKey)
+                ?: error("Meta format with key ${tag.metaFormatKey} not found")
+        }
+
+        val meta: Meta = metaFormat.readObject(input.limit(tag.metaSize.toInt()))
+
+
+        return PartialEnvelope(meta, 30u + tag.metaSize, tag.dataSize)
+    }
+
+    private data class Tag(
+        val metaFormatKey: Short,
+        val metaSize: UInt,
+        val dataSize: ULong,
+    )
+
+    override fun toMeta(): Meta = Meta {
+        IOFormat.NAME_KEY put name.toString()
+    }
+
+    public companion object : EnvelopeFormatFactory {
+        private const val START_SEQUENCE = "#!"
+        private const val END_SEQUENCE = "!#\r\n"
+
+        override val name: Name = super.name + "numass"
+
+        override fun invoke(meta: Meta, context: Context): EnvelopeFormat {
+            val io = context.io
+
+            val metaFormatName = meta["name"].string?.toName() ?: JsonMetaFormat.name
+            //Check if appropriate factory exists
+            io.metaFormatFactories.find { it.name == metaFormatName } ?: error("Meta format could not be resolved")
+
+            return TaggedNumassEnvelopeFormat(io)
+        }
+
+        private fun Input.readTag(): Tag {
+            val start = readRawString(2)
+            if (start != START_SEQUENCE) error("The input is not an envelope")
+            val versionString = readRawString(4)
+            val junk1 = readInt()
+            val metaFormatKey = readShort()
+            val junk2 = readShort()
+            val metaLength = readUInt()
+            val dataLength: ULong = readULong()
+            val end = readRawString(4)
+            if (end != END_SEQUENCE) error("The input is not an envelope")
+            return Tag(metaFormatKey, metaLength, dataLength)
+        }
+
+        override fun peekFormat(io: IOPlugin, input: Input): EnvelopeFormat? {
+            return try {
+                input.preview {
+                    val header = readRawString(30)
+                    if (header.startsWith(START_SEQUENCE) && header.endsWith(END_SEQUENCE)) {
+                        TaggedNumassEnvelopeFormat(io)
+                    } else {
+                        null
+                    }
+                }
+            } catch (ex: Exception) {
+                null
+            }
+        }
+
+        private val default by lazy { invoke() }
+
+        override fun readPartial(input: Input): PartialEnvelope =
+            default.run { readPartial(input) }
+
+        override fun writeEnvelope(
+            output: Output,
+            envelope: Envelope,
+            metaFormatFactory: MetaFormatFactory,
+            formatMeta: Meta,
+        ): Unit = default.run {
+            writeEnvelope(
+                output,
+                envelope,
+                metaFormatFactory,
+                formatMeta
+            )
+        }
+
+        override fun readObject(input: Input): Envelope = default.readObject(input)
+    }
+
+}
+
+
+///**
+// * An envelope type for legacy numass tags. Reads legacy tag and writes DF02 tags
+// */
+//object NumassEnvelopeType : EnvelopeFormatFactory {
+//
+//    override val code: Int = DefaultEnvelopeType.DEFAULT_ENVELOPE_CODE
+//
+//    override val name: String = "numass"
+//
+//    override fun description(): String = "Numass legacy envelope"
+//
+//    /**
+//     * Read as legacy
+//     */
+//    override fun getReader(properties: Map<String, String>): EnvelopeReader {
+//        return NumassEnvelopeReader()
+//    }
+//
+//    /**
+//     * Write as default
+//     */
+//    override fun getWriter(properties: Map<String, String>): EnvelopeWriter {
+//        return DefaultEnvelopeWriter(this, MetaType.resolve(properties))
+//    }
+//
+//    class LegacyTag : EnvelopeTag() {
+//        override val startSequence: ByteArray
+//            get() = LEGACY_START_SEQUENCE
+//
+//        override val endSequence: ByteArray
+//            get() = LEGACY_END_SEQUENCE
+//
+//        /**
+//         * Get the length of tag in bytes. -1 means undefined size in case tag was modified
+//         *
+//         * @return
+//         */
+//        override val length: Int
+//            get() = 30
+//
+//        /**
+//         * Read leagscy version 1 tag without leading tag head
+//         *
+//         * @param buffer
+//         * @return
+//         * @throws IOException
+//         */
+//        override fun readHeader(buffer: ByteBuffer): Map<String, Value> {
+//            val res = HashMap<String, Value>()
+//
+//            val type = buffer.getInt(2)
+//            res[Envelope.TYPE_PROPERTY] = Value.of(type)
+//
+//            val metaTypeCode = buffer.getShort(10)
+//            val metaType = MetaType.resolve(metaTypeCode)
+//
+//            if (metaType != null) {
+//                res[Envelope.META_TYPE_PROPERTY] = metaType.name.parseValue()
+//            } else {
+//                LoggerFactory.getLogger(EnvelopeTag::class.java).warn("Could not resolve meta type. Using default")
+//            }
+//
+//            val metaLength = Integer.toUnsignedLong(buffer.getInt(14))
+//            res[Envelope.META_LENGTH_PROPERTY] = Value.of(metaLength)
+//            val dataLength = Integer.toUnsignedLong(buffer.getInt(22))
+//            res[Envelope.DATA_LENGTH_PROPERTY] = Value.of(dataLength)
+//            return res
+//        }
+//    }
+//
+//    private class NumassEnvelopeReader : DefaultEnvelopeReader() {
+//        override fun newTag(): EnvelopeTag {
+//            return LegacyTag()
+//        }
+//    }
+//
+//    companion object {
+//        val INSTANCE = NumassEnvelopeType()
+//
+//        val LEGACY_START_SEQUENCE = byteArrayOf('#'.toByte(), '!'.toByte())
+//        val LEGACY_END_SEQUENCE = byteArrayOf('!'.toByte(), '#'.toByte(), '\r'.toByte(), '\n'.toByte())
+//
+//        /**
+//         * Replacement for standard type infer to include legacy type
+//         *
+//         * @param path
+//         * @return
+//         */
+//        fun infer(path: Path): EnvelopeType? {
+//            return try {
+//                FileChannel.open(path, StandardOpenOption.READ).use {
+//                    val buffer = it.map(FileChannel.MapMode.READ_ONLY, 0, 6)
+//                    when {
+//                        //TODO use templates from appropriate types
+//                        buffer.get(0) == '#'.toByte() && buffer.get(1) == '!'.toByte() -> INSTANCE
+//                        buffer.get(0) == '#'.toByte() && buffer.get(1) == '!'.toByte() &&
+//                                buffer.get(4) == 'T'.toByte() && buffer.get(5) == 'L'.toByte() -> TaglessEnvelopeType.INSTANCE
+//                        buffer.get(0) == '#'.toByte() && buffer.get(1) == '~'.toByte() -> DefaultEnvelopeType.INSTANCE
+//                        else -> null
+//                    }
+//                }
+//            } catch (ex: Exception) {
+//                LoggerFactory.getLogger(EnvelopeType::class.java).warn("Could not infer envelope type of file {} due to exception: {}", path, ex)
+//                null
+//            }
+//
+//        }
+//
+//    }
+//
+//}

numass-data-proto/src/test/kotlin/ru/inr/mass/data/proto/TestNumassDirectory.kt

@@ -0,0 +1,26 @@
+package ru.inr.mass.data.proto
+
+import hep.dataforge.context.Context
+import hep.dataforge.meta.get
+import hep.dataforge.meta.string
+import hep.dataforge.meta.value
+import hep.dataforge.values.ListValue
+import org.junit.jupiter.api.Test
+import java.nio.file.Path
+import kotlin.test.assertEquals
+
+class TestNumassDirectory {
+    val context = Context("numass-test") {
+        plugin(NumassProtoPlugin)
+    }
+
+    @Test
+    fun testDirectoryRead() {
+        val dataPath = Path.of("src/test/resources", "testData/set_4")
+        val testSet = context.readNumassDirectory(dataPath)
+        assertEquals("2018-04-13T22:01:46", testSet.meta["end_time"].string)
+        assertEquals(ListValue.EMPTY, testSet.meta["comments"].value)
+        assertEquals(31, testSet.points.size)
+        assertEquals("2018-04-13T21:56:09", testSet.points.find { it.index == 22 }?.meta["end_time"].string)
+    }
+}

numass-data-proto/src/test/resources/testData/set_4/scenario

@@ -0,0 +1,31 @@
+POINT 10 18700 0
+POINT 10 19015 0
+POINT 10 19005 0
+POINT 10 19000 0
+POINT 10 18995 0
+POINT 10 18990 0
+POINT 10 18700 0
+POINT 10 18980 0
+POINT 10 18950 0
+POINT 10 18900 0
+POINT 10 18800 0
+POINT 10 18700 0
+POINT 10 18600 0
+POINT 10 18400 0
+POINT 10 18200 0
+POINT 10 18000 0
+POINT 10 17500 0
+POINT 10 17000 0
+POINT 10 18700 0
+POINT 10 16500 0
+POINT 10 16000 0
+POINT 10 15500 0
+POINT 10 15000 0
+POINT 10 14500 0
+POINT 10 18700 0
+POINT 10 14000 0
+POINT 10 13500 0
+POINT 10 13000 0
+POINT 10 12500 0
+POINT 10 12000 0
+POINT 10 18700 0