From cd4f07267b20868f0ff6b0bcb2eb7895a271eb3a Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Mon, 10 Feb 2020 21:49:53 +0300 Subject: [PATCH] Mutable tables --- .../kotlin/hep/dataforge/io/FileEnvelope.kt | 2 + .../kotlin/hep/dataforge/meta/Meta.kt | 2 +- .../meta/scheme/ConfigurableDelegate.kt | 4 +- .../hep/dataforge/tables/ColumnScheme.kt | 11 +- .../kotlin/hep/dataforge/tables/MapRow.kt | 2 +- .../kotlin/hep/dataforge/tables/MetaQuery.kt | 11 -- ...nTableBuilder.kt => MutableColumnTable.kt} | 5 +- .../hep/dataforge/tables/MutableTable.kt | 43 +++++++ .../kotlin/hep/dataforge/tables/RowTable.kt | 25 ++-- .../kotlin/hep/dataforge/tables/Table.kt | 28 ++-- .../hep/dataforge/tables/io/TextRows.kt | 120 ++++++++++++++++++ .../hep/dataforge/tables/io/TextRowsTest.kt | 6 + 12 files changed, 222 insertions(+), 37 deletions(-) delete mode 100644 dataforge-tables/src/commonMain/kotlin/hep/dataforge/tables/MetaQuery.kt rename dataforge-tables/src/commonMain/kotlin/hep/dataforge/tables/{ColumnTableBuilder.kt => MutableColumnTable.kt} (93%) create mode 100644 dataforge-tables/src/commonMain/kotlin/hep/dataforge/tables/MutableTable.kt create mode 100644 dataforge-tables/src/commonMain/kotlin/hep/dataforge/tables/io/TextRows.kt create mode 100644 dataforge-tables/src/commonTest/kotlin/hep/dataforge/tables/io/TextRowsTest.kt diff --git a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileEnvelope.kt b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileEnvelope.kt index 3b34c26c..21cca102 100644 --- a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileEnvelope.kt +++ b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileEnvelope.kt @@ -2,10 +2,12 @@ package hep.dataforge.io import hep.dataforge.meta.Meta import kotlinx.io.Binary +import kotlinx.io.ExperimentalIoApi import kotlinx.io.FileBinary import kotlinx.io.read import java.nio.file.Path +@ExperimentalIoApi class FileEnvelope internal constructor(val path: Path, val format: EnvelopeFormat) : Envelope { //TODO do not like this constructor. Hope to replace it later diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Meta.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Meta.kt index d917559d..93e416de 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Meta.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Meta.kt @@ -208,7 +208,7 @@ val MetaItem<*>?.int get() = number?.toInt() val MetaItem<*>?.long get() = number?.toLong() val MetaItem<*>?.short get() = number?.toShort() -inline fun > MetaItem<*>?.enum() = if (this is ValueItem && this.value is EnumValue<*>) { +inline fun > MetaItem<*>?.enum(): E? = if (this is ValueItem && this.value is EnumValue<*>) { this.value.value as E } else { string?.let { enumValueOf(it) } diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/scheme/ConfigurableDelegate.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/scheme/ConfigurableDelegate.kt index 9b86abba..c6e9a44a 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/scheme/ConfigurableDelegate.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/scheme/ConfigurableDelegate.kt @@ -168,8 +168,8 @@ fun Configurable.float(default: Float, key: Name? = null): ReadWriteProperty> Configurable.enum(default: E, key: Name? = null): ReadWriteProperty = - item(default, key).transform { it.enum() } +inline fun > Configurable.enum(default: E, key: Name? = null): ReadWriteProperty = + item(default, key).transform { it.enum() ?: default } /* * Extra delegates for special cases diff --git a/dataforge-tables/src/commonMain/kotlin/hep/dataforge/tables/ColumnScheme.kt b/dataforge-tables/src/commonMain/kotlin/hep/dataforge/tables/ColumnScheme.kt index aa0d2aec..2b65b234 100644 --- a/dataforge-tables/src/commonMain/kotlin/hep/dataforge/tables/ColumnScheme.kt +++ b/dataforge-tables/src/commonMain/kotlin/hep/dataforge/tables/ColumnScheme.kt @@ -2,7 +2,16 @@ package hep.dataforge.tables import hep.dataforge.meta.scheme.Scheme import hep.dataforge.meta.scheme.SchemeSpec +import hep.dataforge.meta.scheme.enum +import hep.dataforge.meta.scheme.string +import hep.dataforge.values.ValueType + +open class ColumnScheme : Scheme() { + var title by string() -class ColumnScheme : Scheme() { companion object : SchemeSpec(::ColumnScheme) +} + +class ValueColumnScheme : ColumnScheme() { + var valueType by enum(ValueType.STRING) } \ No newline at end of file diff --git a/dataforge-tables/src/commonMain/kotlin/hep/dataforge/tables/MapRow.kt b/dataforge-tables/src/commonMain/kotlin/hep/dataforge/tables/MapRow.kt index 76002e9d..421241d4 100644 --- a/dataforge-tables/src/commonMain/kotlin/hep/dataforge/tables/MapRow.kt +++ b/dataforge-tables/src/commonMain/kotlin/hep/dataforge/tables/MapRow.kt @@ -2,7 +2,7 @@ package hep.dataforge.tables import kotlin.reflect.KClass -class MapRow(val values: Map) : Row { +inline class MapRow(val values: Map) : Row { override fun getValue(column: String, type: KClass): T? { val value = values[column] return type.cast(value) diff --git a/dataforge-tables/src/commonMain/kotlin/hep/dataforge/tables/MetaQuery.kt b/dataforge-tables/src/commonMain/kotlin/hep/dataforge/tables/MetaQuery.kt deleted file mode 100644 index cf07a5c8..00000000 --- a/dataforge-tables/src/commonMain/kotlin/hep/dataforge/tables/MetaQuery.kt +++ /dev/null @@ -1,11 +0,0 @@ -package hep.dataforge.tables - -import hep.dataforge.meta.scheme.Scheme -import hep.dataforge.meta.scheme.SchemeSpec -import hep.dataforge.meta.scheme.string - -class MetaQuery : Scheme() { - var field by string() - - companion object : SchemeSpec(::MetaQuery) -} \ No newline at end of file diff --git a/dataforge-tables/src/commonMain/kotlin/hep/dataforge/tables/ColumnTableBuilder.kt b/dataforge-tables/src/commonMain/kotlin/hep/dataforge/tables/MutableColumnTable.kt similarity index 93% rename from dataforge-tables/src/commonMain/kotlin/hep/dataforge/tables/ColumnTableBuilder.kt rename to dataforge-tables/src/commonMain/kotlin/hep/dataforge/tables/MutableColumnTable.kt index 78a32366..d7a360c9 100644 --- a/dataforge-tables/src/commonMain/kotlin/hep/dataforge/tables/ColumnTableBuilder.kt +++ b/dataforge-tables/src/commonMain/kotlin/hep/dataforge/tables/MutableColumnTable.kt @@ -3,7 +3,10 @@ package hep.dataforge.tables import hep.dataforge.meta.Meta import kotlin.reflect.KClass -class ColumnTableBuilder(val size: Int) : Table { +/** + * Mutable table with a fixed size, but dynamic columns + */ +class MutableColumnTable(val size: Int) : Table { private val _columns = ArrayList>() override val columns: List> get() = _columns diff --git a/dataforge-tables/src/commonMain/kotlin/hep/dataforge/tables/MutableTable.kt b/dataforge-tables/src/commonMain/kotlin/hep/dataforge/tables/MutableTable.kt new file mode 100644 index 00000000..686e3c34 --- /dev/null +++ b/dataforge-tables/src/commonMain/kotlin/hep/dataforge/tables/MutableTable.kt @@ -0,0 +1,43 @@ +package hep.dataforge.tables + +import hep.dataforge.meta.Meta +import kotlin.reflect.KClass + +class SimpleColumnHeader( + override val name: String, + override val type: KClass, + override val meta: Meta +) : ColumnHeader + +class MutableTable( + override val rows: MutableList, + override val header: MutableList> +) : RowTable(rows, header) { + fun column(name: String, type: KClass, meta: Meta): ColumnHeader { + val column = SimpleColumnHeader(name, type, meta) + header.add(column) + return column + } + + inline fun column( + name: String, + noinline columnMetaBuilder: ColumnScheme.() -> Unit + ): ColumnHeader { + return column(name, T::class, ColumnScheme(columnMetaBuilder).toMeta()) + } + + fun row(block: MutableMap.() -> Unit): Row { + val map = HashMap().apply(block) + val row = MapRow(map) + rows.add(row) + return row + } + + operator fun MutableMap.set(header: ColumnHeader, value: T?) { + set(header.name, value) + } +} + +fun Table.edit(block: MutableTable.() -> Unit): Table { + return MutableTable(rows.toMutableList(), header.toMutableList()).apply(block) +} \ No newline at end of file diff --git a/dataforge-tables/src/commonMain/kotlin/hep/dataforge/tables/RowTable.kt b/dataforge-tables/src/commonMain/kotlin/hep/dataforge/tables/RowTable.kt index bc2ca808..98d7d4e1 100644 --- a/dataforge-tables/src/commonMain/kotlin/hep/dataforge/tables/RowTable.kt +++ b/dataforge-tables/src/commonMain/kotlin/hep/dataforge/tables/RowTable.kt @@ -1,23 +1,24 @@ package hep.dataforge.tables import hep.dataforge.meta.Meta +import kotlinx.coroutines.flow.toList import kotlin.reflect.KClass +internal class RowTableColumn(val table: Table, val header: ColumnHeader) : Column { + override val name: String get() = header.name + override val type: KClass get() = header.type + override val meta: Meta get() = header.meta + override val size: Int get() = table.rows.size -class RowTable(override val rows: List, override val header: List>) : Table { + override fun get(index: Int): T? = table.rows[index].getValue(name, type) +} + +open class RowTable(override val rows: List, override val header: List>) : + Table { override fun getValue(row: Int, column: String, type: KClass): T? = rows[row].getValue(column, type) - override val columns: List> get() = header.map { RotTableColumn(it) } - - private inner class RotTableColumn(val header: ColumnHeader) : Column { - override val name: String get() = header.name - override val type: KClass get() = header.type - override val meta: Meta get() = header.meta - override val size: Int get() = rows.size - - override fun get(index: Int): T? = rows[index].getValue(name, type) - } - + override val columns: List> get() = header.map { RowTableColumn(this, it) } } +suspend fun Rows.collect(): Table<*> = this as? Table<*> ?: RowTable(rowFlow().toList(), header) \ No newline at end of file diff --git a/dataforge-tables/src/commonMain/kotlin/hep/dataforge/tables/Table.kt b/dataforge-tables/src/commonMain/kotlin/hep/dataforge/tables/Table.kt index 96d7964d..4e8215e0 100644 --- a/dataforge-tables/src/commonMain/kotlin/hep/dataforge/tables/Table.kt +++ b/dataforge-tables/src/commonMain/kotlin/hep/dataforge/tables/Table.kt @@ -1,6 +1,8 @@ package hep.dataforge.tables import hep.dataforge.meta.Meta +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.asFlow import kotlin.reflect.KClass //TODO to be removed in 1.3.70 @@ -13,24 +15,34 @@ internal fun KClass.cast(value: Any?): T? { } } -typealias TableHeader = List> +typealias TableHeader = List> +/** + * Finite or infinite row set. Rows are produced in a lazy suspendable [Flow]. + * Each row must contain at least all the fields mentioned in [header]. + */ +interface Rows { + val header: TableHeader<*> + fun rowFlow(): Flow +} -interface Table { +interface Table : Rows { fun getValue(row: Int, column: String, type: KClass): T? val columns: Collection> - val header: TableHeader get() = columns.toList() + override val header: TableHeader get() = columns.toList() val rows: List + override fun rowFlow(): Flow = rows.asFlow() /** - * Apply query to a table and return lazy Flow + * Apply typed query to this table and return lazy [Flow] of resulting rows. The flow could be empty. */ - //fun find(query: Any): Flow + //fun select(query: Any): Flow = error("Query of type ${query::class} is not supported by this table") } operator fun Collection>.get(name: String): Column<*>? = find { it.name == name } -inline operator fun Table.get(row: Int, column: String): T? = getValue(row, column, T::class) +inline operator fun Table.get(row: Int, column: String): T? = + getValue(row, column, T::class) interface ColumnHeader { val name: String @@ -38,7 +50,7 @@ interface ColumnHeader { val meta: Meta } -operator fun Table.get(row: Int, column: Column): T? = getValue(row, column.name, column.type) +operator fun Table.get(row: Int, column: Column): T? = getValue(row, column.name, column.type) interface Column : ColumnHeader { val size: Int @@ -58,4 +70,4 @@ interface Row { } inline operator fun Row.get(column: String): T? = getValue(column, T::class) -operator fun Row.get(column: Column): T? = getValue(column.name, column.type) \ No newline at end of file +operator fun Row.get(column: ColumnHeader): T? = getValue(column.name, column.type) \ No newline at end of file diff --git a/dataforge-tables/src/commonMain/kotlin/hep/dataforge/tables/io/TextRows.kt b/dataforge-tables/src/commonMain/kotlin/hep/dataforge/tables/io/TextRows.kt new file mode 100644 index 00000000..f4e73702 --- /dev/null +++ b/dataforge-tables/src/commonMain/kotlin/hep/dataforge/tables/io/TextRows.kt @@ -0,0 +1,120 @@ +package hep.dataforge.tables.io + +import hep.dataforge.meta.get +import hep.dataforge.meta.int +import hep.dataforge.meta.string +import hep.dataforge.tables.* +import hep.dataforge.values.* +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.collect +import kotlinx.coroutines.flow.flow +import kotlinx.io.Binary +import kotlinx.io.ExperimentalIoApi +import kotlinx.io.Output +import kotlinx.io.RandomAccessBinary +import kotlinx.io.text.forEachUtf8Line +import kotlinx.io.text.readUtf8Line +import kotlinx.io.text.writeUtf8String +import kotlin.reflect.KClass + +private fun readLine(header: List>, line: String): Row { + val values = line.split("\\s+".toRegex()).map { it.parseValue() } + + if (values.size == header.size) { + val map = header.map { it.name }.zip(values).toMap() + return MapRow(map) + } else { + error("Can't read line $line. Expected ${header.size} values in a line, but found ${values.size}") + } +} + + +@ExperimentalIoApi +class TextRows(override val header: List>, val binary: Binary) : Rows { + + override fun rowFlow(): Flow = binary.read { + flow { + forEachUtf8Line { line -> + if (line.isNotBlank()) { + val row = readLine(header, line) + emit(row) + } + } + } + } +} + +@ExperimentalIoApi +class TextTable( + override val header: List>, + val binary: RandomAccessBinary, + val index: List +) : Table { + + override val columns: Collection> get() = header.map { RowTableColumn(this, it) } + + override val rows: List get() = index.map { readAt(it) } + + override fun rowFlow(): Flow = TextRows(header, binary).rowFlow() + + private fun readAt(offset: Int): Row { + return binary.read(offset) { + val line = readUtf8Line() + return@read readLine(header, line) + } + } + + override fun getValue(row: Int, column: String, type: KClass): T? { + val offset = index[row] + return type.cast(readAt(offset)[column]) + } +} + +fun Output.writeValue(value: Value, width: Int, left: Boolean = true) { + require(width > 5) { "Width could not be less than 5" } + val str: String = when (value.type) { + ValueType.NUMBER -> value.number.toString() //TODO apply decimal format + ValueType.STRING -> value.string.take(width) + ValueType.BOOLEAN -> if (value.boolean) { + "true" + } else { + "false" + } + ValueType.NULL -> "@null" + } + val padded = if (left) { + str.padEnd(width) + } else { + str.padStart(width) + } + writeUtf8String(padded) +} + +val ColumnHeader.valueType: ValueType? get() = meta["valueType"].string?.let { ValueType.valueOf(it) } + +private val ColumnHeader.width: Int + get() = meta["columnWidth"].int ?: when (valueType) { + ValueType.NUMBER -> 8 + ValueType.STRING -> 16 + ValueType.BOOLEAN -> 5 + ValueType.NULL -> 5 + null -> 16 + } + + +/** + * Write rows without header to the output + */ +suspend fun Output.writeRows(rows: Rows) { + @Suppress("UNCHECKED_CAST") val header = rows.header.map { + if (it.type != Value::class) error("Expected Value column, but found ${it.type}") else (it as ColumnHeader) + } + val widths: List = header.map { + it.width + } + rows.rowFlow().collect { row -> + header.forEachIndexed { index, columnHeader -> + writeValue(row[columnHeader] ?: Null, widths[index]) + } + } +} \ No newline at end of file diff --git a/dataforge-tables/src/commonTest/kotlin/hep/dataforge/tables/io/TextRowsTest.kt b/dataforge-tables/src/commonTest/kotlin/hep/dataforge/tables/io/TextRowsTest.kt new file mode 100644 index 00000000..010f903f --- /dev/null +++ b/dataforge-tables/src/commonTest/kotlin/hep/dataforge/tables/io/TextRowsTest.kt @@ -0,0 +1,6 @@ +package hep.dataforge.tables.io + + +class TextRowsTest{ + +} \ No newline at end of file