Mutable tables

dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileEnvelope.kt

@@ -2,10 +2,12 @@ package hep.dataforge.io
 
 import hep.dataforge.meta.Meta
 import kotlinx.io.Binary
+import kotlinx.io.ExperimentalIoApi
 import kotlinx.io.FileBinary
 import kotlinx.io.read
 import java.nio.file.Path
 
+@ExperimentalIoApi
 class FileEnvelope internal constructor(val path: Path, val format: EnvelopeFormat) : Envelope {
     //TODO do not like this constructor. Hope to replace it later
 

dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Meta.kt

@@ -208,7 +208,7 @@ val MetaItem<*>?.int get() = number?.toInt()
 val MetaItem<*>?.long get() = number?.toLong()
 val MetaItem<*>?.short get() = number?.toShort()
 
-inline fun <reified E : Enum<E>> MetaItem<*>?.enum() = if (this is ValueItem && this.value is EnumValue<*>) {
+inline fun <reified E : Enum<E>> MetaItem<*>?.enum(): E? = if (this is ValueItem && this.value is EnumValue<*>) {
     this.value.value as E
 } else {
     string?.let { enumValueOf<E>(it) }

dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/scheme/ConfigurableDelegate.kt

@@ -168,8 +168,8 @@ fun Configurable.float(default: Float, key: Name? = null): ReadWriteProperty<Any
 /**
  * Enum delegate
  */
-inline fun <reified E : Enum<E>> Configurable.enum(default: E, key: Name? = null): ReadWriteProperty<Any?, E?> =
+inline fun <reified E : Enum<E>> Configurable.enum(default: E, key: Name? = null): ReadWriteProperty<Any?, E> =
-    item(default, key).transform { it.enum<E>() }
+    item(default, key).transform { it.enum<E>() ?: default }
 
 /*
  * Extra delegates for special cases

dataforge-tables/src/commonMain/kotlin/hep/dataforge/tables/ColumnScheme.kt

@@ -2,7 +2,16 @@ package hep.dataforge.tables
 
 import hep.dataforge.meta.scheme.Scheme
 import hep.dataforge.meta.scheme.SchemeSpec
+import hep.dataforge.meta.scheme.enum
+import hep.dataforge.meta.scheme.string
+import hep.dataforge.values.ValueType
+
+open class ColumnScheme : Scheme() {
+    var title by string()
 
-class ColumnScheme : Scheme() {
     companion object : SchemeSpec<ColumnScheme>(::ColumnScheme)
+}
+
+class ValueColumnScheme : ColumnScheme() {
+    var valueType by enum(ValueType.STRING)
 }

dataforge-tables/src/commonMain/kotlin/hep/dataforge/tables/MapRow.kt

@@ -2,7 +2,7 @@ package hep.dataforge.tables
 
 import kotlin.reflect.KClass
 
-class MapRow(val values: Map<String, Any>) : Row {
+inline class MapRow(val values: Map<String, Any?>) : Row {
     override fun <T : Any> getValue(column: String, type: KClass<out T>): T? {
         val value = values[column]
         return type.cast(value)

dataforge-tables/src/commonMain/kotlin/hep/dataforge/tables/MetaQuery.kt

@@ -1,11 +0,0 @@
-package hep.dataforge.tables
-
-import hep.dataforge.meta.scheme.Scheme
-import hep.dataforge.meta.scheme.SchemeSpec
-import hep.dataforge.meta.scheme.string
-
-class MetaQuery : Scheme() {
-    var field by string()
-
-    companion object : SchemeSpec<MetaQuery>(::MetaQuery)
-}

dataforge-tables/src/commonMain/kotlin/hep/dataforge/tables/MutableColumnTable.kt

@@ -3,7 +3,10 @@ package hep.dataforge.tables
 import hep.dataforge.meta.Meta
 import kotlin.reflect.KClass
 
-class ColumnTableBuilder<C: Any>(val size: Int) : Table<C> {
+/**
+ * Mutable table with a fixed size, but dynamic columns
+ */
+class MutableColumnTable<C: Any>(val size: Int) : Table<C> {
     private val _columns = ArrayList<Column<C>>()
 
     override val columns: List<Column<C>> get() = _columns

dataforge-tables/src/commonMain/kotlin/hep/dataforge/tables/MutableTable.kt

@@ -0,0 +1,43 @@
+package hep.dataforge.tables
+
+import hep.dataforge.meta.Meta
+import kotlin.reflect.KClass
+
+class SimpleColumnHeader<T : Any>(
+    override val name: String,
+    override val type: KClass<out T>,
+    override val meta: Meta
+) : ColumnHeader<T>
+
+class MutableTable<C : Any>(
+    override val rows: MutableList<Row>,
+    override val header: MutableList<ColumnHeader<C>>
+) : RowTable<C, Row>(rows, header) {
+    fun <T : C> column(name: String, type: KClass<out T>, meta: Meta): ColumnHeader<T> {
+        val column = SimpleColumnHeader(name, type, meta)
+        header.add(column)
+        return column
+    }
+
+    inline fun <reified T : C> column(
+        name: String,
+        noinline columnMetaBuilder: ColumnScheme.() -> Unit
+    ): ColumnHeader<T> {
+        return column(name, T::class, ColumnScheme(columnMetaBuilder).toMeta())
+    }
+
+    fun row(block: MutableMap<String, Any?>.() -> Unit): Row {
+        val map = HashMap<String, Any?>().apply(block)
+        val row = MapRow(map)
+        rows.add(row)
+        return row
+    }
+
+    operator fun <T : Any> MutableMap<String, Any?>.set(header: ColumnHeader<T>, value: T?) {
+        set(header.name, value)
+    }
+}
+
+fun <C : Any> Table<C>.edit(block: MutableTable<C>.() -> Unit): Table<C> {
+    return MutableTable(rows.toMutableList(), header.toMutableList()).apply(block)
+}

dataforge-tables/src/commonMain/kotlin/hep/dataforge/tables/RowTable.kt

@@ -1,23 +1,24 @@
 package hep.dataforge.tables
 
 import hep.dataforge.meta.Meta
+import kotlinx.coroutines.flow.toList
 import kotlin.reflect.KClass
 
+internal class RowTableColumn<C : Any, T : C>(val table: Table<C>, val header: ColumnHeader<T>) : Column<T> {
+    override val name: String get() = header.name
+    override val type: KClass<out T> get() = header.type
+    override val meta: Meta get() = header.meta
+    override val size: Int get() = table.rows.size
 
-class RowTable<C: Any, R : Row>(override val rows: List<R>, override val header:  List<ColumnHeader<C>>) : Table<C> {
+    override fun get(index: Int): T? = table.rows[index].getValue(name, type)
+}
+
+open class RowTable<C : Any, R : Row>(override val rows: List<R>, override val header: List<ColumnHeader<C>>) :
+    Table<C> {
     override fun <T : Any> getValue(row: Int, column: String, type: KClass<out T>): T? =
         rows[row].getValue(column, type)
 
-    override val columns: List<Column<C>> get() = header.map { RotTableColumn(it) }
+    override val columns: List<Column<C>> get() = header.map { RowTableColumn(this, it) }
-
-    private inner class RotTableColumn<T : C>(val header: ColumnHeader<T>) : Column<T> {
-        override val name: String get() = header.name
-        override val type: KClass<out T> get() = header.type
-        override val meta: Meta get() = header.meta
-        override val size: Int get() = rows.size
-
-        override fun get(index: Int): T? = rows[index].getValue(name, type)
-    }
-
 }
 
+suspend fun Rows.collect(): Table<*> = this as? Table<*> ?: RowTable(rowFlow().toList(), header)

dataforge-tables/src/commonMain/kotlin/hep/dataforge/tables/Table.kt

@@ -1,6 +1,8 @@
 package hep.dataforge.tables
 
 import hep.dataforge.meta.Meta
+import kotlinx.coroutines.flow.Flow
+import kotlinx.coroutines.flow.asFlow
 import kotlin.reflect.KClass
 
 //TODO to be removed in 1.3.70
@@ -13,24 +15,34 @@ internal fun <T : Any> KClass<T>.cast(value: Any?): T? {
     }
 }
 
-typealias TableHeader = List<ColumnHeader<*>>
+typealias TableHeader<C> = List<ColumnHeader<C>>
 
+/**
+ * Finite or infinite row set. Rows are produced in a lazy suspendable [Flow].
+ * Each row must contain at least all the fields mentioned in [header].
+ */
+interface Rows {
+    val header: TableHeader<*>
+    fun rowFlow(): Flow<Row>
+}
 
-interface Table<out C: Any> {
+interface Table<out C : Any> : Rows {
     fun <T : Any> getValue(row: Int, column: String, type: KClass<out T>): T?
     val columns: Collection<Column<C>>
-    val header: TableHeader get() = columns.toList()
+    override val header: TableHeader<C> get() = columns.toList()
     val rows: List<Row>
+    override fun rowFlow(): Flow<Row> = rows.asFlow()
 
     /**
-     * Apply query to a table and return lazy Flow
+     * Apply typed query to this table and return lazy [Flow] of resulting rows. The flow could be empty.
      */
-    //fun find(query: Any): Flow<Row>
+    //fun select(query: Any): Flow<Row> = error("Query of type ${query::class} is not supported by this table")
 }
 
 operator fun Collection<Column<*>>.get(name: String): Column<*>? = find { it.name == name }
 
-inline operator fun <C: Any, reified T : C> Table<C>.get(row: Int, column: String): T? = getValue(row, column, T::class)
+inline operator fun <C : Any, reified T : C> Table<C>.get(row: Int, column: String): T? =
+    getValue(row, column, T::class)
 
 interface ColumnHeader<out T : Any> {
     val name: String
@@ -38,7 +50,7 @@ interface ColumnHeader<out T : Any> {
     val meta: Meta
 }
 
-operator fun <C: Any, T : C> Table<C>.get(row: Int, column: Column<T>): T? = getValue(row, column.name, column.type)
+operator fun <C : Any, T : C> Table<C>.get(row: Int, column: Column<T>): T? = getValue(row, column.name, column.type)
 
 interface Column<out T : Any> : ColumnHeader<T> {
     val size: Int
@@ -58,4 +70,4 @@ interface Row {
 }
 
 inline operator fun <reified T : Any> Row.get(column: String): T? = getValue(column, T::class)
-operator fun <T : Any> Row.get(column: Column<T>): T? = getValue(column.name, column.type)
+operator fun <T : Any> Row.get(column: ColumnHeader<T>): T? = getValue(column.name, column.type)

dataforge-tables/src/commonMain/kotlin/hep/dataforge/tables/io/TextRows.kt

@@ -0,0 +1,120 @@
+package hep.dataforge.tables.io
+
+import hep.dataforge.meta.get
+import hep.dataforge.meta.int
+import hep.dataforge.meta.string
+import hep.dataforge.tables.*
+import hep.dataforge.values.*
+import kotlinx.coroutines.flow.Flow
+import kotlinx.coroutines.flow.collect
+import kotlinx.coroutines.flow.flow
+import kotlinx.io.Binary
+import kotlinx.io.ExperimentalIoApi
+import kotlinx.io.Output
+import kotlinx.io.RandomAccessBinary
+import kotlinx.io.text.forEachUtf8Line
+import kotlinx.io.text.readUtf8Line
+import kotlinx.io.text.writeUtf8String
+import kotlin.reflect.KClass
+
+private fun readLine(header: List<ColumnHeader<Value>>, line: String): Row {
+    val values = line.split("\\s+".toRegex()).map { it.parseValue() }
+
+    if (values.size == header.size) {
+        val map = header.map { it.name }.zip(values).toMap()
+        return MapRow(map)
+    } else {
+        error("Can't read line $line. Expected ${header.size} values in a line, but found ${values.size}")
+    }
+}
+
+
+@ExperimentalIoApi
+class TextRows(override val header: List<ColumnHeader<Value>>, val binary: Binary) : Rows {
+
+    override fun rowFlow(): Flow<Row> = binary.read {
+        flow {
+            forEachUtf8Line { line ->
+                if (line.isNotBlank()) {
+                    val row = readLine(header, line)
+                    emit(row)
+                }
+            }
+        }
+    }
+}
+
+@ExperimentalIoApi
+class TextTable(
+    override val header: List<ColumnHeader<Value>>,
+    val binary: RandomAccessBinary,
+    val index: List<Int>
+) : Table<Value> {
+
+    override val columns: Collection<Column<Value>> get() = header.map { RowTableColumn(this, it) }
+
+    override val rows: List<Row> get() = index.map { readAt(it) }
+
+    override fun rowFlow(): Flow<Row> = TextRows(header, binary).rowFlow()
+
+    private fun readAt(offset: Int): Row {
+        return binary.read(offset) {
+            val line = readUtf8Line()
+            return@read readLine(header, line)
+        }
+    }
+
+    override fun <T : Any> getValue(row: Int, column: String, type: KClass<out T>): T? {
+        val offset = index[row]
+        return type.cast(readAt(offset)[column])
+    }
+}
+
+fun Output.writeValue(value: Value, width: Int, left: Boolean = true) {
+    require(width > 5) { "Width could not be less than 5" }
+    val str: String = when (value.type) {
+        ValueType.NUMBER -> value.number.toString() //TODO apply decimal format
+        ValueType.STRING -> value.string.take(width)
+        ValueType.BOOLEAN -> if (value.boolean) {
+            "true"
+        } else {
+            "false"
+        }
+        ValueType.NULL -> "@null"
+    }
+    val padded = if (left) {
+        str.padEnd(width)
+    } else {
+        str.padStart(width)
+    }
+    writeUtf8String(padded)
+}
+
+val ColumnHeader<Value>.valueType: ValueType? get() = meta["valueType"].string?.let { ValueType.valueOf(it) }
+
+private val ColumnHeader<Value>.width: Int
+    get() = meta["columnWidth"].int ?: when (valueType) {
+        ValueType.NUMBER -> 8
+        ValueType.STRING -> 16
+        ValueType.BOOLEAN -> 5
+        ValueType.NULL -> 5
+        null -> 16
+    }
+
+
+/**
+ * Write rows without header to the output
+ */
+suspend fun Output.writeRows(rows: Rows) {
+    @Suppress("UNCHECKED_CAST") val header = rows.header.map {
+        if (it.type != Value::class) error("Expected Value column, but found ${it.type}") else (it as ColumnHeader<Value>)
+    }
+    val widths: List<Int> = header.map {
+        it.width
+    }
+    rows.rowFlow().collect { row ->
+        header.forEachIndexed { index, columnHeader ->
+            writeValue(row[columnHeader] ?: Null, widths[index])
+        }
+    }
+}

dataforge-tables/src/commonTest/kotlin/hep/dataforge/tables/io/TextRowsTest.kt

@@ -0,0 +1,6 @@
+package hep.dataforge.tables.io
+
+
+class TextRowsTest{
+
+}