Text tables test working

dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Envelope.kt

@@ -28,7 +28,7 @@ interface Envelope {
         /**
          * Build a static envelope using provided builder
          */
-        operator fun invoke(block: EnvelopeBuilder.() -> Unit) = EnvelopeBuilder().apply(block).build()
+        inline operator fun invoke(block: EnvelopeBuilder.() -> Unit) = EnvelopeBuilder().apply(block).build()
     }
 }
 

dataforge-io/src/commonMain/kotlin/hep/dataforge/io/EnvelopeBuilder.kt

@@ -18,8 +18,15 @@ class EnvelopeBuilder {
         metaBuilder.update(meta)
     }
 
+    /**
+     * The general purpose of the envelope
+     */
     var type by metaBuilder.string(key = Envelope.ENVELOPE_TYPE_KEY)
     var dataType by metaBuilder.string(key = Envelope.ENVELOPE_DATA_TYPE_KEY)
+
+    /**
+     * Data unique identifier to bypass identity checks
+     */
     var dataID by metaBuilder.string(key = Envelope.ENVELOPE_DATA_ID_KEY)
     var description by metaBuilder.string(key = Envelope.ENVELOPE_DESCRIPTION_KEY)
     var name by metaBuilder.string(key = Envelope.ENVELOPE_NAME_KEY)
@@ -32,5 +39,14 @@ class EnvelopeBuilder {
         data = ArrayBinary.write(builder = block)
     }
 
-    internal fun build() = SimpleEnvelope(metaBuilder.seal(), data)
+    fun build() = SimpleEnvelope(metaBuilder.seal(), data)
+
 }
+
+//@ExperimentalContracts
+//suspend fun EnvelopeBuilder.buildData(block: suspend Output.() -> Unit): Binary{
+//    contract {
+//        callsInPlace(block, InvocationKind.EXACTLY_ONCE)
+//    }
+//    val scope = CoroutineScope(coroutineContext)
+//}

dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MutableMeta.kt

@@ -115,6 +115,8 @@ operator fun MutableMeta<*>.set(name: NameToken, value: Any?) = set(name.asName(
 
 operator fun MutableMeta<*>.set(key: String, value: Any?) = set(key.toName(), value)
 
+operator fun MutableMeta<*>.set(key: String, index: String, value: Any?) = set(key.toName().withIndex(index), value)
+
 /**
  * Update existing mutable node with another node. The rules are following:
  *  * value replaces anything

dataforge-tables/src/commonMain/kotlin/hep/dataforge/tables/ColumnHeader.kt

@@ -0,0 +1,36 @@
+package hep.dataforge.tables
+
+import hep.dataforge.meta.Meta
+import hep.dataforge.meta.get
+import hep.dataforge.meta.int
+import hep.dataforge.meta.string
+import hep.dataforge.values.Value
+import hep.dataforge.values.ValueType
+import kotlin.reflect.KClass
+
+typealias TableHeader<C> = List<ColumnHeader<C>>
+
+typealias ValueTableHeader = List<ColumnHeader<Value>>
+
+interface ColumnHeader<out T : Any> {
+    val name: String
+    val type: KClass<out T>
+    val meta: Meta
+}
+
+data class SimpleColumnHeader<T : Any>(
+    override val name: String,
+    override val type: KClass<out T>,
+    override val meta: Meta
+) : ColumnHeader<T>
+
+val ColumnHeader<Value>.valueType: ValueType? get() = meta["valueType"].string?.let { ValueType.valueOf(it) }
+
+val ColumnHeader<Value>.textWidth: Int
+    get() = meta["columnWidth"].int ?: when (valueType) {
+        ValueType.NUMBER -> 8
+        ValueType.STRING -> 16
+        ValueType.BOOLEAN -> 5
+        ValueType.NULL -> 5
+        null -> 16
+    }

dataforge-tables/src/commonMain/kotlin/hep/dataforge/tables/ColumnTable.kt

@@ -12,16 +12,20 @@ class ColumnTable<C : Any>(override val columns: Collection<Column<C>>) : Table<
         require(columns.all { it.size == rowsNum }) { "All columns must be of the same size" }
     }
 
-    override val rows: List<Row>
+    override val rows: List<Row<C>>
         get() = (0 until rowsNum).map { VirtualRow(this, it) }
 
-    override fun <T : Any> getValue(row: Int, column: String, type: KClass<out T>): T? {
+    override fun <T : C> getValue(row: Int, column: String, type: KClass<out T>): T? {
         val value = columns[column]?.get(row)
         return type.cast(value)
     }
 }
 
-internal class VirtualRow<C : Any>(val table: Table<C>, val index: Int) : Row {
+internal class VirtualRow<C : Any>(val table: Table<C>, val index: Int) : Row<C> {
-    override fun <T : Any> getValue(column: String, type: KClass<out T>): T? = table.getValue(index, column, type)
+    override fun <T : C> getValue(column: String, type: KClass<out T>): T? = table.getValue(index, column, type)
+
+//    override fun <T : C> get(columnHeader: ColumnHeader<T>): T? {
+//        return table.co[columnHeader][index]
+//    }
 }
 

dataforge-tables/src/commonMain/kotlin/hep/dataforge/tables/MapRow.kt

@@ -2,8 +2,8 @@ package hep.dataforge.tables
 
 import kotlin.reflect.KClass
 
-inline class MapRow(val values: Map<String, Any?>) : Row {
+inline class MapRow<C: Any>(val values: Map<String, C?>) : Row<C> {
-    override fun <T : Any> getValue(column: String, type: KClass<out T>): T? {
+    override fun <T : C> getValue(column: String, type: KClass<out T>): T? {
         val value = values[column]
         return type.cast(value)
     }

dataforge-tables/src/commonMain/kotlin/hep/dataforge/tables/MutableColumnTable.kt

@@ -10,11 +10,11 @@ class MutableColumnTable<C: Any>(val size: Int) : Table<C> {
     private val _columns = ArrayList<Column<C>>()
 
     override val columns: List<Column<C>> get() = _columns
-    override val rows: List<Row> get() = (0 until size).map {
+    override val rows: List<Row<C>> get() = (0 until size).map {
         VirtualRow(this, it)
     }
 
-    override fun <T : Any> getValue(row: Int, column: String, type: KClass<out T>): T? {
+    override fun <T : C> getValue(row: Int, column: String, type: KClass<out T>): T? {
         val value = columns[column]?.get(row)
         return type.cast(value)
     }

dataforge-tables/src/commonMain/kotlin/hep/dataforge/tables/MutableTable.kt

@@ -1,18 +1,14 @@
 package hep.dataforge.tables
 
 import hep.dataforge.meta.Meta
+import hep.dataforge.values.Value
 import kotlin.reflect.KClass
 
-class SimpleColumnHeader<T : Any>(
-    override val name: String,
-    override val type: KClass<out T>,
-    override val meta: Meta
-) : ColumnHeader<T>
-
 class MutableTable<C : Any>(
-    override val rows: MutableList<Row>,
+    override val rows: MutableList<Row<C>>,
     override val header: MutableList<ColumnHeader<C>>
-) : RowTable<C, Row>(rows, header) {
+) : RowTable<C>(rows, header) {
+
     fun <T : C> column(name: String, type: KClass<out T>, meta: Meta): ColumnHeader<T> {
         val column = SimpleColumnHeader(name, type, meta)
         header.add(column)
@@ -21,23 +17,24 @@ class MutableTable<C : Any>(
 
     inline fun <reified T : C> column(
         name: String,
-        noinline columnMetaBuilder: ColumnScheme.() -> Unit
+        noinline columnMetaBuilder: ColumnScheme.() -> Unit = {}
     ): ColumnHeader<T> {
         return column(name, T::class, ColumnScheme(columnMetaBuilder).toMeta())
     }
 
-    fun row(block: MutableMap<String, Any?>.() -> Unit): Row {
+    fun row(map: Map<String, C?>): Row<C> {
-        val map = HashMap<String, Any?>().apply(block)
         val row = MapRow(map)
         rows.add(row)
         return row
     }
 
-    operator fun <T : Any> MutableMap<String, Any?>.set(header: ColumnHeader<T>, value: T?) {
+    fun <T : C> row(vararg pairs: Pair<ColumnHeader<T>, T>): Row<C> =
-        set(header.name, value)
+        row(pairs.associate { it.first.name to it.second })
-    }
 }
 
+fun MutableTable<Value>.row(vararg pairs: Pair<ColumnHeader<Value>, Any?>): Row<Value> =
+    row(pairs.associate { it.first.name to Value.of(it.second) })
+
 fun <C : Any> Table<C>.edit(block: MutableTable<C>.() -> Unit): Table<C> {
     return MutableTable(rows.toMutableList(), header.toMutableList()).apply(block)
 }

dataforge-tables/src/commonMain/kotlin/hep/dataforge/tables/RowTable.kt

@@ -13,12 +13,11 @@ internal class RowTableColumn<C : Any, T : C>(val table: Table<C>, val header: C
     override fun get(index: Int): T? = table.rows[index].getValue(name, type)
 }
 
-open class RowTable<C : Any, R : Row>(override val rows: List<R>, override val header: List<ColumnHeader<C>>) :
+open class RowTable<C : Any>(override val rows: List<Row<C>>, override val header: List<ColumnHeader<C>>) : Table<C> {
-    Table<C> {
+    override fun <T : C> getValue(row: Int, column: String, type: KClass<out T>): T? =
-    override fun <T : Any> getValue(row: Int, column: String, type: KClass<out T>): T? =
         rows[row].getValue(column, type)
 
     override val columns: List<Column<C>> get() = header.map { RowTableColumn(this, it) }
 }
 
-suspend fun Rows.collect(): Table<*> = this as? Table<*> ?: RowTable(rowFlow().toList(), header)
+suspend fun <C : Any> Rows<C>.collect(): Table<C> = this as? Table<C> ?: RowTable(rowFlow().toList(), header)

dataforge-tables/src/commonMain/kotlin/hep/dataforge/tables/Table.kt

@@ -1,6 +1,5 @@
 package hep.dataforge.tables
 
-import hep.dataforge.meta.Meta
 import kotlinx.coroutines.flow.Flow
 import kotlinx.coroutines.flow.asFlow
 import kotlin.reflect.KClass
@@ -15,28 +14,30 @@ internal fun <T : Any> KClass<T>.cast(value: Any?): T? {
     }
 }
 
-typealias TableHeader<C> = List<ColumnHeader<C>>
-
 /**
  * Finite or infinite row set. Rows are produced in a lazy suspendable [Flow].
  * Each row must contain at least all the fields mentioned in [header].
  */
-interface Rows {
+interface Rows<C : Any> {
-    val header: TableHeader<*>
+    val header: TableHeader<C>
-    fun rowFlow(): Flow<Row>
+    fun rowFlow(): Flow<Row<C>>
 }
 
-interface Table<out C : Any> : Rows {
+interface Table<C : Any> : Rows<C> {
-    fun <T : Any> getValue(row: Int, column: String, type: KClass<out T>): T?
+    fun <T : C> getValue(row: Int, column: String, type: KClass<out T>): T?
     val columns: Collection<Column<C>>
     override val header: TableHeader<C> get() = columns.toList()
-    val rows: List<Row>
+    val rows: List<Row<C>>
-    override fun rowFlow(): Flow<Row> = rows.asFlow()
+    override fun rowFlow(): Flow<Row<C>> = rows.asFlow()
 
     /**
      * Apply typed query to this table and return lazy [Flow] of resulting rows. The flow could be empty.
      */
     //fun select(query: Any): Flow<Row> = error("Query of type ${query::class} is not supported by this table")
+    companion object {
+        inline operator fun <T : Any> invoke(block: MutableTable<T>.() -> Unit): Table<T> =
+            MutableTable<T>(arrayListOf(), arrayListOf()).apply(block)
+    }
 }
 
 operator fun Collection<Column<*>>.get(name: String): Column<*>? = find { it.name == name }
@@ -44,15 +45,9 @@ operator fun Collection<Column<*>>.get(name: String): Column<*>? = find { it.nam
 inline operator fun <C : Any, reified T : C> Table<C>.get(row: Int, column: String): T? =
     getValue(row, column, T::class)
 
-interface ColumnHeader<out T : Any> {
-    val name: String
-    val type: KClass<out T>
-    val meta: Meta
-}
-
 operator fun <C : Any, T : C> Table<C>.get(row: Int, column: Column<T>): T? = getValue(row, column.name, column.type)
 
-interface Column<out T : Any> : ColumnHeader<T> {
+interface Column<T : Any> : ColumnHeader<T> {
     val size: Int
     operator fun get(index: Int): T?
 }
@@ -65,9 +60,9 @@ operator fun <T : Any> Column<T>.iterator() = iterator {
     }
 }
 
-interface Row {
+interface Row<C: Any> {
-    fun <T : Any> getValue(column: String, type: KClass<out T>): T?
+    fun <T : C> getValue(column: String, type: KClass<out T>): T?
 }
 
-inline operator fun <reified T : Any> Row.get(column: String): T? = getValue(column, T::class)
+inline operator fun <C : Any, reified T : C> Row<C>.get(column: String): T? = getValue(column, T::class)
-operator fun <T : Any> Row.get(column: ColumnHeader<T>): T? = getValue(column.name, column.type)
+operator fun <C : Any, T : C> Row<C>.get(column: ColumnHeader<T>): T? = getValue(column.name, column.type)

dataforge-tables/src/commonMain/kotlin/hep/dataforge/tables/io/TextRows.kt

@@ -1,38 +1,56 @@
 package hep.dataforge.tables.io
 
-import hep.dataforge.meta.get
-import hep.dataforge.meta.int
-import hep.dataforge.meta.string
 import hep.dataforge.tables.*
 import hep.dataforge.values.*
 import kotlinx.coroutines.flow.Flow
 import kotlinx.coroutines.flow.collect
 import kotlinx.coroutines.flow.flow
+import kotlinx.coroutines.flow.toList
 import kotlinx.io.Binary
 import kotlinx.io.ExperimentalIoApi
 import kotlinx.io.Output
 import kotlinx.io.RandomAccessBinary
 import kotlinx.io.text.forEachUtf8Line
 import kotlinx.io.text.readUtf8Line
+import kotlinx.io.text.readUtf8StringUntilDelimiter
 import kotlinx.io.text.writeUtf8String
 import kotlin.reflect.KClass
 
-private fun readLine(header: List<ColumnHeader<Value>>, line: String): Row {
+/**
-    val values = line.split("\\s+".toRegex()).map { it.parseValue() }
+ * Read a lin as a fixed width [Row]
+ */
+private fun readLine(header: ValueTableHeader, line: String): Row<Value> {
+    val values = line.trim().split("\\s+".toRegex()).map { it.lazyParseValue() }
 
     if (values.size == header.size) {
         val map = header.map { it.name }.zip(values).toMap()
         return MapRow(map)
     } else {
-        error("Can't read line $line. Expected ${header.size} values in a line, but found ${values.size}")
+        error("Can't read line \"$line\". Expected ${header.size} values in a line, but found ${values.size}")
     }
 }
 
-
+/**
+ * Finite or infinite [Rows] created from a fixed width text binary
+ */
 @ExperimentalIoApi
-class TextRows(override val header: List<ColumnHeader<Value>>, val binary: Binary) : Rows {
+class TextRows(override val header: ValueTableHeader, val binary: Binary) : Rows<Value> {
 
-    override fun rowFlow(): Flow<Row> = binary.read {
+    /**
+     * A flow of indexes of string start offsets ignoring empty strings
+     */
+    fun indexFlow(): Flow<Int> = binary.read {
+        var counter: Int = 0
+        flow {
+            val string = readUtf8StringUntilDelimiter('\n')
+            counter += string.length
+            if (!string.isBlank()) {
+                emit(counter)
+            }
+        }
+    }
+
+    override fun rowFlow(): Flow<Row<Value>> = binary.read {
         flow {
             forEachUtf8Line { line ->
                 if (line.isNotBlank()) {
@@ -42,35 +60,57 @@ class TextRows(override val header: List<ColumnHeader<Value>>, val binary: Binar
             }
         }
     }
+
+    companion object
 }
 
+/**
+ * Create a row offset index for [TextRows]
+ */
+@ExperimentalIoApi
+suspend fun TextRows.buildRowIndex(): List<Int> = indexFlow().toList()
+
+/**
+ * Finite table created from [RandomAccessBinary] with fixed width text table
+ */
 @ExperimentalIoApi
 class TextTable(
-    override val header: List<ColumnHeader<Value>>,
+    override val header: ValueTableHeader,
     val binary: RandomAccessBinary,
     val index: List<Int>
 ) : Table<Value> {
 
     override val columns: Collection<Column<Value>> get() = header.map { RowTableColumn(this, it) }
 
-    override val rows: List<Row> get() = index.map { readAt(it) }
+    override val rows: List<Row<Value>> get() = index.map { readAt(it) }
 
-    override fun rowFlow(): Flow<Row> = TextRows(header, binary).rowFlow()
+    override fun rowFlow(): Flow<Row<Value>> = TextRows(header, binary).rowFlow()
 
-    private fun readAt(offset: Int): Row {
+    private fun readAt(offset: Int): Row<Value> {
         return binary.read(offset) {
             val line = readUtf8Line()
             return@read readLine(header, line)
         }
     }
 
-    override fun <T : Any> getValue(row: Int, column: String, type: KClass<out T>): T? {
+    override fun <T : Value> getValue(row: Int, column: String, type: KClass<out T>): T? {
         val offset = index[row]
         return type.cast(readAt(offset)[column])
     }
+
+    companion object {
+        suspend operator fun invoke(header: ValueTableHeader, binary: RandomAccessBinary): TextTable {
+            val index = TextRows(header, binary).buildRowIndex()
+            return TextTable(header, binary, index)
+        }
+    }
 }
 
-fun Output.writeValue(value: Value, width: Int, left: Boolean = true) {
+
+/**
+ * Write a fixed width value to the output
+ */
+private fun Output.writeValue(value: Value, width: Int, left: Boolean = true) {
     require(width > 5) { "Width could not be less than 5" }
     val str: String = when (value.type) {
         ValueType.NUMBER -> value.number.toString() //TODO apply decimal format
@@ -90,31 +130,20 @@ fun Output.writeValue(value: Value, width: Int, left: Boolean = true) {
     writeUtf8String(padded)
 }
 
-val ColumnHeader<Value>.valueType: ValueType? get() = meta["valueType"].string?.let { ValueType.valueOf(it) }
-
-private val ColumnHeader<Value>.width: Int
-    get() = meta["columnWidth"].int ?: when (valueType) {
-        ValueType.NUMBER -> 8
-        ValueType.STRING -> 16
-        ValueType.BOOLEAN -> 5
-        ValueType.NULL -> 5
-        null -> 16
-    }
-
-
 /**
  * Write rows without header to the output
  */
-suspend fun Output.writeRows(rows: Rows) {
+suspend fun Output.writeRows(rows: Rows<Value>) {
     @Suppress("UNCHECKED_CAST") val header = rows.header.map {
         if (it.type != Value::class) error("Expected Value column, but found ${it.type}") else (it as ColumnHeader<Value>)
     }
     val widths: List<Int> = header.map {
-        it.width
+        it.textWidth
     }
     rows.rowFlow().collect { row ->
         header.forEachIndexed { index, columnHeader ->
             writeValue(row[columnHeader] ?: Null, widths[index])
         }
+        writeUtf8String("\r\n")
     }
 }

dataforge-tables/src/commonMain/kotlin/hep/dataforge/tables/io/textTableEnvelope.kt

@@ -0,0 +1,42 @@
+package hep.dataforge.tables.io
+
+import hep.dataforge.io.Envelope
+import hep.dataforge.meta.*
+import hep.dataforge.tables.SimpleColumnHeader
+import hep.dataforge.tables.Table
+import hep.dataforge.values.Value
+import kotlinx.io.ByteArrayOutput
+import kotlinx.io.EmptyBinary
+import kotlinx.io.ExperimentalIoApi
+import kotlinx.io.asBinary
+
+
+@ExperimentalIoApi
+suspend fun Table<Value>.wrap(): Envelope = Envelope {
+    meta {
+        header.forEachIndexed { index, columnHeader ->
+            set("column", index.toString(), buildMeta {
+                "name" put columnHeader.name
+                if (!columnHeader.meta.isEmpty()) {
+                    "meta" put columnHeader.meta
+                }
+            })
+        }
+    }
+
+    type = "table.value"
+    dataID = "valueTable[${this@wrap.hashCode()}]"
+
+    data = ByteArrayOutput().apply { writeRows(this@wrap) }.toByteArray().asBinary()
+}
+
+@DFExperimental
+@ExperimentalIoApi
+fun TextRows.Companion.readEnvelope(envelope: Envelope): TextRows {
+    val header = envelope.meta.getIndexed("column")
+        .entries.sortedBy { it.key.toInt() }
+        .map { (_, item) ->
+            SimpleColumnHeader(item.node["name"].string!!, Value::class, item.node["meta"].node ?: Meta.EMPTY)
+        }
+    return TextRows(header, envelope.data ?: EmptyBinary)
+}

dataforge-tables/src/commonTest/kotlin/hep/dataforge/tables/io/TextRowsTest.kt

@@ -1,6 +0,0 @@
-package hep.dataforge.tables.io
-
-
-class TextRowsTest{
-
-}

dataforge-tables/src/jvmMain/kotlin/hep/dataforge/tables/CastColumn.kt

@@ -8,7 +8,7 @@ import kotlin.reflect.full.cast
 import kotlin.reflect.full.isSubclassOf
 
 @Suppress("UNCHECKED_CAST")
-fun <T : Any> Column<*>.cast(type: KClass<T>): Column<T> {
+fun <T : Any> Column<*>.cast(type: KClass<out T>): Column<T> {
     return if (type.isSubclassOf(this.type)) {
         this as Column<T>
     } else {
@@ -16,7 +16,7 @@ fun <T : Any> Column<*>.cast(type: KClass<T>): Column<T> {
     }
 }
 
-class CastColumn<T : Any>(val origin: Column<*>, override val type: KClass<T>) : Column<T> {
+class CastColumn<T : Any>(val origin: Column<*>, override val type: KClass<out T>) : Column<T> {
     override val name: String get() = origin.name
     override val meta: Meta get() = origin.meta
     override val size: Int get() = origin.size
@@ -32,5 +32,5 @@ class ColumnProperty<C: Any, T : C>(val table: Table<C>, val type: KClass<T>) :
     }
 }
 
-operator fun <T : Any> Collection<Column<*>>.get(header: ColumnHeader<T>): Column<T>? =
+operator fun <C: Any, T : C> Collection<Column<C>>.get(header: ColumnHeader<T>): Column<T>? =
     find { it.name == header.name }?.cast(header.type)

dataforge-tables/src/jvmTest/kotlin/hep/dataforge/tables/io/TextRowsTest.kt

@@ -0,0 +1,39 @@
+package hep.dataforge.tables.io
+
+import hep.dataforge.meta.DFExperimental
+import hep.dataforge.tables.Table
+import hep.dataforge.tables.get
+import hep.dataforge.tables.row
+import hep.dataforge.values.Value
+import hep.dataforge.values.int
+import kotlinx.coroutines.flow.toList
+import kotlinx.coroutines.runBlocking
+import kotlinx.io.ExperimentalIoApi
+import kotlinx.io.toByteArray
+import kotlin.test.Test
+import kotlin.test.assertEquals
+
+
+@DFExperimental
+@ExperimentalIoApi
+class TextRowsTest {
+    val table = Table<Value> {
+        val a = column<Value>("a")
+        val b = column<Value>("b")
+        row(a to 1, b to "b1")
+        row(a to 2, b to "b2")
+    }
+
+    @Test
+    fun testTableWriteRead() {
+        runBlocking {
+            val envelope = table.wrap()
+            val string = envelope.data!!.toByteArray().decodeToString()
+            println(string)
+            val table = TextRows.readEnvelope(envelope)
+            val rows = table.rowFlow().toList()
+            assertEquals(1, rows[0]["a"]?.int)
+            assertEquals("b2", rows[1]["b"]?.string)
+        }
+    }
+}