kscience/dataforge-core
8
0
Fork 0
Code Issues Pull Requests 1 Packages Projects Releases 12 Wiki Activity

Explicit mode

Browse Source
This commit is contained in:
Alexander Nozik 2020-09-08 21:45:50 +03:00
parent 8a8484172c
commit 606faa5e1b
30 changed files with 251 additions and 258 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
dataforge-io/dataforge-io-yaml/src/main/kotlin/hep/dataforge/io/yaml/FrontMatterEnvelopeFormat.kt
View File

@ -11,9 +11,9 @@ import kotlinx.io.text.readUtf8Line
import kotlinx.io.text.writeUtf8String import kotlinx.io.text.writeUtf8String
@DFExperimental @DFExperimental
class FrontMatterEnvelopeFormat( public class FrontMatterEnvelopeFormat(
val io: IOPlugin, private val io: IOPlugin,
val meta: Meta = Meta.EMPTY, private val meta: Meta = Meta.EMPTY,
) : EnvelopeFormat { ) : EnvelopeFormat {
override fun readPartial(input: Input): PartialEnvelope { override fun readPartial(input: Input): PartialEnvelope {
@ -80,8 +80,8 @@ class FrontMatterEnvelopeFormat(
META_KEY put meta META_KEY put meta
} }
companion object : EnvelopeFormatFactory { public companion object : EnvelopeFormatFactory {
const val SEPARATOR = "---" public const val SEPARATOR = "---"
private val metaTypeRegex = "---(\\w*)\\s*".toRegex() private val metaTypeRegex = "---(\\w*)\\s*".toRegex()

14
dataforge-io/src/commonMain/kotlin/hep/dataforge/io/ioMisc.kt
View File

@ -3,26 +3,26 @@ package hep.dataforge.io
import kotlinx.io.* import kotlinx.io.*
import kotlin.math.min import kotlin.math.min
fun Output.writeRawString(str: String) { public fun Output.writeRawString(str: String) {
str.forEach { writeByte(it.toByte()) } str.forEach { writeByte(it.toByte()) }
} }
fun Input.readRawString(size: Int): String { public fun Input.readRawString(size: Int): String {
val array = CharArray(size) { readByte().toChar() } val array = CharArray(size) { readByte().toChar() }
return array.concatToString() return array.concatToString()
} }
inline fun buildByteArray(expectedSize: Int = 16, block: Output.() -> Unit): ByteArray = public inline fun buildByteArray(expectedSize: Int = 16, block: Output.() -> Unit): ByteArray =
ByteArrayOutput(expectedSize).apply(block).toByteArray() ByteArrayOutput(expectedSize).apply(block).toByteArray()
@Suppress("FunctionName") @Suppress("FunctionName")
inline fun Binary(expectedSize: Int = 16, block: Output.() -> Unit): Binary = public inline fun Binary(expectedSize: Int = 16, block: Output.() -> Unit): Binary =
buildByteArray(expectedSize, block).asBinary() buildByteArray(expectedSize, block).asBinary()
/** /**
* View section of a [Binary] as an independent binary * View section of a [Binary] as an independent binary
*/ */
class BinaryView(private val source: Binary, private val start: Int, override val size: Int) : Binary { public class BinaryView(private val source: Binary, private val start: Int, override val size: Int) : Binary {
init { init {
require(start > 0) require(start > 0)
@ -34,6 +34,6 @@ class BinaryView(private val source: Binary, private val start: Int, override va
} }
} }
fun Binary.view(start: Int, size: Int) = BinaryView(this, start, size) public fun Binary.view(start: Int, size: Int): BinaryView = BinaryView(this, start, size)
operator fun Binary.get(range: IntRange) = view(range.first, range.last - range.first) public operator fun Binary.get(range: IntRange): BinaryView = view(range.first, range.last - range.first)

24
dataforge-output/src/commonMain/kotlin/hep/dataforge/output/OutputManager.kt
View File

@ -11,7 +11,7 @@ import kotlin.reflect.KClass
/** /**
* A manager for outputs * A manager for outputs
*/ */
interface OutputManager { public interface OutputManager {
/** /**
* Get an output specialized for given type, name and stage. * Get an output specialized for given type, name and stage.
@ -19,7 +19,7 @@ interface OutputManager {
* @param name represents the name inside the node. * @param name represents the name inside the node.
* @param meta configuration for [Renderer] (not for rendered object) * @param meta configuration for [Renderer] (not for rendered object)
*/ */
operator fun <T : Any> get( public operator fun <T : Any> get(
type: KClass<out T>, type: KClass<out T>,
name: Name, name: Name,
stage: Name = Name.EMPTY, stage: Name = Name.EMPTY,
@ -30,12 +30,12 @@ interface OutputManager {
/** /**
* Get an output manager for a context * Get an output manager for a context
*/ */
val Context.output: OutputManager get() = plugins.get() ?: ConsoleOutputManager() public val Context.output: OutputManager get() = plugins.get() ?: ConsoleOutputManager()
/** /**
* Get an output with given [name], [stage] and reified content type * Get an output with given [name], [stage] and reified content type
*/ */
inline operator fun <reified T : Any> OutputManager.get( public inline operator fun <reified T : Any> OutputManager.get(
name: Name, name: Name,
stage: Name = Name.EMPTY, stage: Name = Name.EMPTY,
meta: Meta = Meta.EMPTY meta: Meta = Meta.EMPTY
@ -46,14 +46,14 @@ inline operator fun <reified T : Any> OutputManager.get(
/** /**
* Directly render an object using the most suitable renderer * Directly render an object using the most suitable renderer
*/ */
fun OutputManager.render(obj: Any, name: Name, stage: Name = Name.EMPTY, meta: Meta = Meta.EMPTY) = public fun OutputManager.render(obj: Any, name: Name, stage: Name = Name.EMPTY, meta: Meta = Meta.EMPTY): Unit =
get(obj::class, name, stage).render(obj, meta) get(obj::class, name, stage).render(obj, meta)
/** /**
* System console output. * System console output.
* The [CONSOLE_RENDERER] is used when no other [OutputManager] is provided. * The [CONSOLE_RENDERER] is used when no other [OutputManager] is provided.
*/ */
val CONSOLE_RENDERER: Renderer<Any> = object : Renderer<Any> { public val CONSOLE_RENDERER: Renderer<Any> = object : Renderer<Any> {
override fun render(obj: Any, meta: Meta) { override fun render(obj: Any, meta: Meta) {
println(obj) println(obj)
} }
@ -62,21 +62,21 @@ val CONSOLE_RENDERER: Renderer<Any> = object : Renderer<Any> {
} }
class ConsoleOutputManager : AbstractPlugin(), OutputManager { public class ConsoleOutputManager : AbstractPlugin(), OutputManager {
override val tag: PluginTag get() = ConsoleOutputManager.tag override val tag: PluginTag get() = ConsoleOutputManager.tag
override fun <T : Any> get(type: KClass<out T>, name: Name, stage: Name, meta: Meta): Renderer<T> = CONSOLE_RENDERER override fun <T : Any> get(type: KClass<out T>, name: Name, stage: Name, meta: Meta): Renderer<T> = CONSOLE_RENDERER
companion object : PluginFactory<ConsoleOutputManager> { public companion object : PluginFactory<ConsoleOutputManager> {
override val tag = PluginTag("output.console", group = DATAFORGE_GROUP) override val tag: PluginTag = PluginTag("output.console", group = DATAFORGE_GROUP)
override val type = ConsoleOutputManager::class override val type: KClass<ConsoleOutputManager> = ConsoleOutputManager::class
override fun invoke(meta: Meta, context: Context) = ConsoleOutputManager() override fun invoke(meta: Meta, context: Context): ConsoleOutputManager = ConsoleOutputManager()
} }
} }
/** /**
* A dispatcher for output tasks. * A dispatcher for output tasks.
*/ */
expect val Dispatchers.Output: CoroutineDispatcher public expect val Dispatchers.Output: CoroutineDispatcher

4
dataforge-output/src/commonMain/kotlin/hep/dataforge/output/Renderer.kt
View File

@ -10,12 +10,12 @@ import hep.dataforge.meta.Meta
* based on its configuration and provided meta * based on its configuration and provided meta
* *
*/ */
interface Renderer<in T : Any> : ContextAware { public interface Renderer<in T : Any> : ContextAware {
/** /**
* Render specific object with configuration. * Render specific object with configuration.
* *
* By convention actual render is called in asynchronous mode, so this method should never * By convention actual render is called in asynchronous mode, so this method should never
* block execution * block execution
*/ */
fun render(obj: T, meta: Meta = Meta.EMPTY) public fun render(obj: T, meta: Meta = Meta.EMPTY)
} }

66
dataforge-output/src/commonMain/kotlin/hep/dataforge/output/TextRenderer.kt
View File

@ -11,7 +11,41 @@ import kotlinx.io.Output
import kotlinx.io.text.writeUtf8String import kotlinx.io.text.writeUtf8String
import kotlin.reflect.KClass import kotlin.reflect.KClass
class TextRenderer(override val context: Context, private val output: Output) : Renderer<Any> {
/**
* A text or binary renderer based on [Output]
*/
@Type(TEXT_RENDERER_TYPE)
public interface TextFormat {
/**
* The priority of this renderer compared to other renderers
*/
public val priority: Int
/**
* The type of the content served by this renderer
*/
public val type: KClass<*>
public suspend fun Output.render(obj: Any)
public companion object {
public const val TEXT_RENDERER_TYPE: String = "dataforge.textRenderer"
}
}
public object DefaultTextFormat : TextFormat {
override val priority: Int = Int.MAX_VALUE
override val type: KClass<*> = Any::class
override suspend fun Output.render(obj: Any) {
writeUtf8String(obj.toString() + "\n")
}
}
/**
* A text-based renderer
*/
public class TextRenderer(override val context: Context, private val output: Output) : Renderer<Any> {
private val cache = HashMap<KClass<*>, TextFormat>() private val cache = HashMap<KClass<*>, TextFormat>()
/** /**
@ -40,33 +74,3 @@ class TextRenderer(override val context: Context, private val output: Output) :
} }
} }
} }
/**
* A text or binary renderer based on [Output]
*/
@Type(TEXT_RENDERER_TYPE)
interface TextFormat {
/**
* The priority of this renderer compared to other renderers
*/
val priority: Int
/**
* The type of the content served by this renderer
*/
val type: KClass<*>
suspend fun Output.render(obj: Any)
companion object {
const val TEXT_RENDERER_TYPE = "dataforge.textRenderer"
}
}
object DefaultTextFormat : TextFormat {
override val priority: Int = Int.MAX_VALUE
override val type: KClass<*> = Any::class
override suspend fun Output.render(obj: Any) {
writeUtf8String(obj.toString() + "\n")
}
}

2
dataforge-output/src/jsMain/kotlin/hep/dataforge/output/outputJS.kt
View File

@ -4,4 +4,4 @@ import kotlinx.coroutines.CoroutineDispatcher
import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.Dispatchers
actual val Dispatchers.Output: CoroutineDispatcher get() = Dispatchers.Default public actual val Dispatchers.Output: CoroutineDispatcher get() = Default

3
dataforge-output/src/jvmMain/kotlin/hep/dataforge/output/outputJVM.kt
View File

@ -1,5 +1,6 @@
package hep.dataforge.output package hep.dataforge.output
import kotlinx.coroutines.CoroutineDispatcher
import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.Dispatchers
actual val Dispatchers.Output get() = Dispatchers.IO public actual val Dispatchers.Output: CoroutineDispatcher get() = IO

2
dataforge-tables/src/commonMain/kotlin/hep/dataforge/tables/ColumnDef.kt
View File

@ -3,7 +3,7 @@ package hep.dataforge.tables
import hep.dataforge.meta.Meta import hep.dataforge.meta.Meta
import kotlin.reflect.KClass import kotlin.reflect.KClass
data class ColumnDef<out T : Any>( public data class ColumnDef<out T : Any>(
override val name: String, override val name: String,
override val type: KClass<out T>, override val type: KClass<out T>,
override val meta: Meta override val meta: Meta

18
dataforge-tables/src/commonMain/kotlin/hep/dataforge/tables/ColumnHeader.kt
View File

@ -8,25 +8,25 @@ import hep.dataforge.values.Value
import hep.dataforge.values.ValueType import hep.dataforge.values.ValueType
import kotlin.reflect.KClass import kotlin.reflect.KClass
typealias TableHeader<C> = List<ColumnHeader<C>> public typealias TableHeader<C> = List<ColumnHeader<C>>
typealias ValueTableHeader = List<ColumnHeader<Value>> public typealias ValueTableHeader = List<ColumnHeader<Value>>
interface ColumnHeader<out T : Any> { public interface ColumnHeader<out T : Any> {
val name: String public val name: String
val type: KClass<out T> public val type: KClass<out T>
val meta: Meta public val meta: Meta
} }
data class SimpleColumnHeader<T : Any>( public data class SimpleColumnHeader<T : Any>(
override val name: String, override val name: String,
override val type: KClass<out T>, override val type: KClass<out T>,
override val meta: Meta override val meta: Meta
) : ColumnHeader<T> ) : ColumnHeader<T>
val ColumnHeader<Value>.valueType: ValueType? get() = meta["valueType"].string?.let { ValueType.valueOf(it) } public val ColumnHeader<Value>.valueType: ValueType? get() = meta["valueType"].string?.let { ValueType.valueOf(it) }
val ColumnHeader<Value>.textWidth: Int public val ColumnHeader<Value>.textWidth: Int
get() = meta["columnWidth"].int ?: when (valueType) { get() = meta["columnWidth"].int ?: when (valueType) {
ValueType.NUMBER -> 8 ValueType.NUMBER -> 8
ValueType.STRING -> 16 ValueType.STRING -> 16

10
dataforge-tables/src/commonMain/kotlin/hep/dataforge/tables/ColumnScheme.kt
View File

@ -6,12 +6,12 @@ import hep.dataforge.meta.enum
import hep.dataforge.meta.string import hep.dataforge.meta.string
import hep.dataforge.values.ValueType import hep.dataforge.values.ValueType
open class ColumnScheme : Scheme() { public open class ColumnScheme : Scheme() {
var title by string() public var title: String? by string()
companion object : SchemeSpec<ColumnScheme>(::ColumnScheme) public companion object : SchemeSpec<ColumnScheme>(::ColumnScheme)
} }
class ValueColumnScheme : ColumnScheme() { public class ValueColumnScheme : ColumnScheme() {
var valueType by enum(ValueType.STRING) public var valueType: ValueType by enum(ValueType.STRING)
} }

2
dataforge-tables/src/commonMain/kotlin/hep/dataforge/tables/ColumnTable.kt
View File

@ -3,7 +3,7 @@ package hep.dataforge.tables
/** /**
* @param T bottom type for all columns in the table * @param T bottom type for all columns in the table
*/ */
class ColumnTable<T : Any>(override val columns: Collection<Column<T>>) : Table<T> { public class ColumnTable<T : Any>(override val columns: Collection<Column<T>>) : Table<T> {
private val rowsNum = columns.first().size private val rowsNum = columns.first().size
init { init {

10
dataforge-tables/src/commonMain/kotlin/hep/dataforge/tables/ListColumn.kt
View File

@ -3,7 +3,7 @@ package hep.dataforge.tables
import hep.dataforge.meta.Meta import hep.dataforge.meta.Meta
import kotlin.reflect.KClass import kotlin.reflect.KClass
class ListColumn<T : Any>( public class ListColumn<T : Any>(
override val name: String, override val name: String,
private val data: List<T?>, private val data: List<T?>,
override val type: KClass<out T>, override val type: KClass<out T>,
@ -13,14 +13,14 @@ class ListColumn<T : Any>(
override fun get(index: Int): T? = data[index] override fun get(index: Int): T? = data[index]
companion object { public companion object {
inline operator fun <reified T : Any> invoke( public inline operator fun <reified T : Any> invoke(
name: String, name: String,
def: ColumnScheme, def: ColumnScheme,
data: List<T?> data: List<T?>
): ListColumn<T> = ListColumn(name, data, T::class, def.toMeta()) ): ListColumn<T> = ListColumn(name, data, T::class, def.toMeta())
inline operator fun <reified T : Any> invoke( public inline operator fun <reified T : Any> invoke(
name: String, name: String,
def: ColumnScheme, def: ColumnScheme,
size: Int, size: Int,
@ -29,7 +29,7 @@ class ListColumn<T : Any>(
} }
} }
inline fun <T : Any, reified R : Any> Column<T>.map(meta: Meta = this.meta, noinline block: (T?) -> R): Column<R> { public inline fun <T : Any, reified R : Any> Column<T>.map(meta: Meta = this.meta, noinline block: (T?) -> R): Column<R> {
val data = List(size) { block(get(it)) } val data = List(size) { block(get(it)) }
return ListColumn(name, data, R::class, meta) return ListColumn(name, data, R::class, meta)
} }

6
dataforge-tables/src/commonMain/kotlin/hep/dataforge/tables/MutableColumnTable.kt
View File

@ -3,7 +3,7 @@ package hep.dataforge.tables
/** /**
* Mutable table with a fixed size, but dynamic columns * Mutable table with a fixed size, but dynamic columns
*/ */
class MutableColumnTable<C: Any>(val size: Int) : Table<C> { public class MutableColumnTable<C: Any>(public val size: Int) : Table<C> {
private val _columns = ArrayList<Column<C>>() private val _columns = ArrayList<Column<C>>()
override val columns: List<Column<C>> get() = _columns override val columns: List<Column<C>> get() = _columns
@ -16,7 +16,7 @@ class MutableColumnTable<C: Any>(val size: Int) : Table<C> {
/** /**
* Add a fixed column to the end of the table * Add a fixed column to the end of the table
*/ */
fun add(column: Column<C>) { public fun add(column: Column<C>) {
require(column.size == this.size) { "Required column size $size, but found ${column.size}" } require(column.size == this.size) { "Required column size $size, but found ${column.size}" }
_columns.add(column) _columns.add(column)
} }
@ -24,7 +24,7 @@ class MutableColumnTable<C: Any>(val size: Int) : Table<C> {
/** /**
* Insert a column at [index] * Insert a column at [index]
*/ */
fun insert(index: Int, column: Column<C>) { public fun insert(index: Int, column: Column<C>) {
require(column.size == this.size) { "Required column size $size, but found ${column.size}" } require(column.size == this.size) { "Required column size $size, but found ${column.size}" }
_columns.add(index, column) _columns.add(index, column)
} }

14
dataforge-tables/src/commonMain/kotlin/hep/dataforge/tables/MutableTable.kt
View File

@ -4,37 +4,37 @@ import hep.dataforge.meta.Meta
import hep.dataforge.values.Value import hep.dataforge.values.Value
import kotlin.reflect.KClass import kotlin.reflect.KClass
class MutableTable<C : Any>( public class MutableTable<C : Any>(
override val rows: MutableList<Row<C>>, override val rows: MutableList<Row<C>>,
override val header: MutableList<ColumnHeader<C>> override val header: MutableList<ColumnHeader<C>>
) : RowTable<C>(rows, header) { ) : RowTable<C>(rows, header) {
fun <R : C> column(name: String, type: KClass<out R>, meta: Meta): ColumnHeader<R> { public fun <R : C> column(name: String, type: KClass<out R>, meta: Meta): ColumnHeader<R> {
val column = SimpleColumnHeader(name, type, meta) val column = SimpleColumnHeader(name, type, meta)
header.add(column) header.add(column)
return column return column
} }
inline fun <reified T : C> column( public inline fun <reified T : C> column(
name: String, name: String,
noinline columnMetaBuilder: ColumnScheme.() -> Unit = {} noinline columnMetaBuilder: ColumnScheme.() -> Unit = {}
): ColumnHeader<T> { ): ColumnHeader<T> {
return column(name, T::class, ColumnScheme(columnMetaBuilder).toMeta()) return column(name, T::class, ColumnScheme(columnMetaBuilder).toMeta())
} }
fun row(map: Map<String, C?>): Row<C> { public fun row(map: Map<String, C?>): Row<C> {
val row = MapRow(map) val row = MapRow(map)
rows.add(row) rows.add(row)
return row return row
} }
fun <T : C> row(vararg pairs: Pair<ColumnHeader<T>, T>): Row<C> = public fun <T : C> row(vararg pairs: Pair<ColumnHeader<T>, T>): Row<C> =
row(pairs.associate { it.first.name to it.second }) row(pairs.associate { it.first.name to it.second })
} }
fun MutableTable<Value>.row(vararg pairs: Pair<ColumnHeader<Value>, Any?>): Row<Value> = public fun MutableTable<Value>.row(vararg pairs: Pair<ColumnHeader<Value>, Any?>): Row<Value> =
row(pairs.associate { it.first.name to Value.of(it.second) }) row(pairs.associate { it.first.name to Value.of(it.second) })
fun <C : Any> Table<C>.edit(block: MutableTable<C>.() -> Unit): Table<C> { public fun <C : Any> Table<C>.edit(block: MutableTable<C>.() -> Unit): Table<C> {
return MutableTable(rows.toMutableList(), header.toMutableList()).apply(block) return MutableTable(rows.toMutableList(), header.toMutableList()).apply(block)
} }

6
dataforge-tables/src/commonMain/kotlin/hep/dataforge/tables/RowTable.kt
View File

@ -4,7 +4,7 @@ import hep.dataforge.meta.Meta
import kotlinx.coroutines.flow.toList import kotlinx.coroutines.flow.toList
import kotlin.reflect.KClass import kotlin.reflect.KClass
inline class MapRow<C : Any>(val values: Map<String, C?>) : Row<C> { public inline class MapRow<C : Any>(private val values: Map<String, C?>) : Row<C> {
override fun getValue(column: String): C? = values[column] override fun getValue(column: String): C? = values[column]
} }
@ -17,10 +17,10 @@ internal class RowTableColumn<C : Any, T : C>(val table: Table<C>, val header: C
override fun get(index: Int): T? = table.rows[index].getValue(name, type) override fun get(index: Int): T? = table.rows[index].getValue(name, type)
} }
open class RowTable<C : Any>(override val rows: List<Row<C>>, override val header: List<ColumnHeader<C>>) : Table<C> { public open class RowTable<C : Any>(override val rows: List<Row<C>>, override val header: List<ColumnHeader<C>>) : Table<C> {
override fun getValue(row: Int, column: String): C? = rows[row].getValue(column) override fun getValue(row: Int, column: String): C? = rows[row].getValue(column)
override val columns: List<Column<C>> get() = header.map { RowTableColumn(this, it) } override val columns: List<Column<C>> get() = header.map { RowTableColumn(this, it) }
} }
suspend fun <C : Any> Rows<C>.collect(): Table<C> = this as? Table<C> ?: RowTable(rowFlow().toList(), header) public suspend fun <C : Any> Rows<C>.collect(): Table<C> = this as? Table<C> ?: RowTable(rowFlow().toList(), header)

46
dataforge-tables/src/commonMain/kotlin/hep/dataforge/tables/Table.kt
View File

@ -9,57 +9,57 @@ import kotlin.reflect.cast
* Finite or infinite row set. Rows are produced in a lazy suspendable [Flow]. * Finite or infinite row set. Rows are produced in a lazy suspendable [Flow].
* Each row must contain at least all the fields mentioned in [header]. * Each row must contain at least all the fields mentioned in [header].
*/ */
interface Rows<out T : Any> { public interface Rows<out T : Any> {
val header: TableHeader<T> public val header: TableHeader<T>
fun rowFlow(): Flow<Row<T>> public fun rowFlow(): Flow<Row<T>>
} }
interface Table<out T : Any> : Rows<T> { public interface Table<out T : Any> : Rows<T> {
fun getValue(row: Int, column: String): T? public fun getValue(row: Int, column: String): T?
val columns: Collection<Column<T>> public val columns: Collection<Column<T>>
override val header: TableHeader<T> get() = columns.toList() override val header: TableHeader<T> get() = columns.toList()
val rows: List<Row<T>> public val rows: List<Row<T>>
override fun rowFlow(): Flow<Row<T>> = rows.asFlow() override fun rowFlow(): Flow<Row<T>> = rows.asFlow()
/** /**
* Apply typed query to this table and return lazy [Flow] of resulting rows. The flow could be empty. * Apply typed query to this table and return lazy [Flow] of resulting rows. The flow could be empty.
*/ */
//fun select(query: Any): Flow<Row> = error("Query of type ${query::class} is not supported by this table") //fun select(query: Any): Flow<Row> = error("Query of type ${query::class} is not supported by this table")
companion object { public companion object {
inline operator fun <T : Any> invoke(block: MutableTable<T>.() -> Unit): Table<T> = public inline operator fun <T : Any> invoke(block: MutableTable<T>.() -> Unit): Table<T> =
MutableTable<T>(arrayListOf(), arrayListOf()).apply(block) MutableTable<T>(arrayListOf(), arrayListOf()).apply(block)
} }
} }
fun <C : Any, T : C> Table<C>.getValue(row: Int, column: String, type: KClass<out T>): T? = public fun <C : Any, T : C> Table<C>.getValue(row: Int, column: String, type: KClass<out T>): T? =
type.cast(getValue(row, column)) type.cast(getValue(row, column))
operator fun <T : Any> Collection<Column<T>>.get(name: String): Column<T>? = find { it.name == name } public operator fun <T : Any> Collection<Column<T>>.get(name: String): Column<T>? = find { it.name == name }
inline operator fun <C : Any, reified T : C> Table<C>.get(row: Int, column: String): T? = public inline operator fun <C : Any, reified T : C> Table<C>.get(row: Int, column: String): T? =
getValue(row, column, T::class) getValue(row, column, T::class)
operator fun <C : Any, T : C> Table<C>.get(row: Int, column: ColumnHeader<T>): T? = public operator fun <C : Any, T : C> Table<C>.get(row: Int, column: ColumnHeader<T>): T? =
getValue(row, column.name, column.type) getValue(row, column.name, column.type)
interface Column<out T : Any> : ColumnHeader<T> { public interface Column<out T : Any> : ColumnHeader<T> {
val size: Int public val size: Int
operator fun get(index: Int): T? public operator fun get(index: Int): T?
} }
val Column<*>.indices get() = (0 until size) public val Column<*>.indices: IntRange get() = (0 until size)
operator fun <T : Any> Column<T>.iterator() = iterator { public operator fun <T : Any> Column<T>.iterator(): Iterator<T?> = iterator {
for (i in indices) { for (i in indices) {
yield(get(i)) yield(get(i))
} }
} }
interface Row<out T : Any> { public interface Row<out T : Any> {
fun getValue(column: String): T? public fun getValue(column: String): T?
} }
fun <C : Any, T : C> Row<C>.getValue(column: String, type: KClass<out T>): T? = type.cast(getValue(column)) public fun <C : Any, T : C> Row<C>.getValue(column: String, type: KClass<out T>): T? = type.cast(getValue(column))
inline operator fun <reified T : Any> Row<T>.get(column: String): T? = T::class.cast(getValue(column)) public inline operator fun <reified T : Any> Row<T>.get(column: String): T? = T::class.cast(getValue(column))
operator fun <C : Any, T : C> Row<C>.get(column: ColumnHeader<T>): T? = getValue(column.name, column.type) public operator fun <C : Any, T : C> Row<C>.get(column: ColumnHeader<T>): T? = getValue(column.name, column.type)

18
dataforge-tables/src/commonMain/kotlin/hep/dataforge/tables/TransformationColumn.kt
View File

@ -6,12 +6,12 @@ import kotlin.reflect.KClass
/** /**
* A virtual column obtained by transforming Given row to a single value * A virtual column obtained by transforming Given row to a single value
*/ */
class TransformationColumn<T : Any, R : Any>( public class TransformationColumn<T : Any, R : Any>(
val table: Table<T>, public val table: Table<T>,
override val type: KClass<out R>, override val type: KClass<out R>,
override val name: String, override val name: String,
override val meta: Meta, override val meta: Meta,
val mapper: (Row<T>) -> R? public val mapper: (Row<T>) -> R?
) : Column<R> { ) : Column<R> {
override val size: Int get() = table.rows.size override val size: Int get() = table.rows.size
@ -23,12 +23,12 @@ class TransformationColumn<T : Any, R : Any>(
* *
* Calls are not thread safe * Calls are not thread safe
*/ */
class CachedTransformationColumn<T : Any, R : Any>( public class CachedTransformationColumn<T : Any, R : Any>(
val table: Table<T>, public val table: Table<T>,
override val type: KClass<out R>, override val type: KClass<out R>,
override val name: String, override val name: String,
override val meta: Meta, override val meta: Meta,
val mapper: (Row<T>) -> R? public val mapper: (Row<T>) -> R?
) : Column<R> { ) : Column<R> {
override val size: Int get() = table.rows.size override val size: Int get() = table.rows.size
private val values: HashMap<Int, R?> = HashMap() private val values: HashMap<Int, R?> = HashMap()
@ -38,7 +38,7 @@ class CachedTransformationColumn<T : Any, R : Any>(
/** /**
* Create a virtual column from a given column * Create a virtual column from a given column
*/ */
inline fun <T : Any, reified R : Any> Table<T>.mapRows( public inline fun <T : Any, reified R : Any> Table<T>.mapRows(
name: String, name: String,
meta: Meta = Meta.EMPTY, meta: Meta = Meta.EMPTY,
cache: Boolean = false, cache: Boolean = false,
@ -49,12 +49,12 @@ inline fun <T : Any, reified R : Any> Table<T>.mapRows(
TransformationColumn(this, R::class, name, meta, mapper) TransformationColumn(this, R::class, name, meta, mapper)
} }
fun <T : Any> Table<T>.mapRowsToDouble(name: String, meta: Meta = Meta.EMPTY, block: (Row<T>) -> Double): RealColumn { public fun <T : Any> Table<T>.mapRowsToDouble(name: String, meta: Meta = Meta.EMPTY, block: (Row<T>) -> Double): RealColumn {
val data = DoubleArray(rows.size) { block(rows[it]) } val data = DoubleArray(rows.size) { block(rows[it]) }
return RealColumn(name, data, meta) return RealColumn(name, data, meta)
} }
fun <T : Any> Table<T>.mapRowsToInt(name: String, meta: Meta = Meta.EMPTY, block: (Row<T>) -> Int): IntColumn { public fun <T : Any> Table<T>.mapRowsToInt(name: String, meta: Meta = Meta.EMPTY, block: (Row<T>) -> Int): IntColumn {
val data = IntArray(rows.size) { block(rows[it]) } val data = IntArray(rows.size) { block(rows[it]) }
return IntColumn(name, data, meta) return IntColumn(name, data, meta)
} }

20
dataforge-tables/src/commonMain/kotlin/hep/dataforge/tables/io/TextRows.kt
View File

@ -32,12 +32,12 @@ private fun readLine(header: ValueTableHeader, line: String): Row<Value> {
* Finite or infinite [Rows] created from a fixed width text binary * Finite or infinite [Rows] created from a fixed width text binary
*/ */
@ExperimentalIoApi @ExperimentalIoApi
class TextRows(override val header: ValueTableHeader, val binary: Binary) : Rows<Value> { public class TextRows(override val header: ValueTableHeader, private val binary: Binary) : Rows<Value> {
/** /**
* A flow of indexes of string start offsets ignoring empty strings * A flow of indexes of string start offsets ignoring empty strings
*/ */
fun indexFlow(): Flow<Int> = binary.read { public fun indexFlow(): Flow<Int> = binary.read {
var counter: Int = 0 var counter: Int = 0
flow { flow {
val string = readUtf8StringUntilDelimiter('\n') val string = readUtf8StringUntilDelimiter('\n')
@ -59,23 +59,23 @@ class TextRows(override val header: ValueTableHeader, val binary: Binary) : Rows
} }
} }
companion object public companion object
} }
/** /**
* Create a row offset index for [TextRows] * Create a row offset index for [TextRows]
*/ */
@ExperimentalIoApi @ExperimentalIoApi
suspend fun TextRows.buildRowIndex(): List<Int> = indexFlow().toList() public suspend fun TextRows.buildRowIndex(): List<Int> = indexFlow().toList()
/** /**
* Finite table created from [RandomAccessBinary] with fixed width text table * Finite table created from [RandomAccessBinary] with fixed width text table
*/ */
@ExperimentalIoApi @ExperimentalIoApi
class TextTable( public class TextTable(
override val header: ValueTableHeader, override val header: ValueTableHeader,
val binary: Binary, private val binary: Binary,
val index: List<Int> public val index: List<Int>
) : Table<Value> { ) : Table<Value> {
override val columns: Collection<Column<Value>> get() = header.map { RowTableColumn(this, it) } override val columns: Collection<Column<Value>> get() = header.map { RowTableColumn(this, it) }
@ -96,8 +96,8 @@ class TextTable(
return readAt(offset)[column] return readAt(offset)[column]
} }
companion object { public companion object {
suspend operator fun invoke(header: ValueTableHeader, binary: Binary): TextTable { public suspend operator fun invoke(header: ValueTableHeader, binary: Binary): TextTable {
val index = TextRows(header, binary).buildRowIndex() val index = TextRows(header, binary).buildRowIndex()
return TextTable(header, binary, index) return TextTable(header, binary, index)
} }
@ -131,7 +131,7 @@ private fun Output.writeValue(value: Value, width: Int, left: Boolean = true) {
/** /**
* Write rows without header to the output * Write rows without header to the output
*/ */
suspend fun Output.writeRows(rows: Rows<Value>) { public suspend fun Output.writeRows(rows: Rows<Value>) {
val widths: List<Int> = rows.header.map { val widths: List<Int> = rows.header.map {
it.textWidth it.textWidth
} }

4
dataforge-tables/src/commonMain/kotlin/hep/dataforge/tables/io/textTableEnvelope.kt
View File

@ -12,7 +12,7 @@ import kotlinx.io.asBinary
@ExperimentalIoApi @ExperimentalIoApi
suspend fun Table<Value>.wrap(): Envelope = Envelope { public suspend fun Table<Value>.wrap(): Envelope = Envelope {
meta { meta {
header.forEachIndexed { index, columnHeader -> header.forEachIndexed { index, columnHeader ->
set("column", index.toString(), Meta { set("column", index.toString(), Meta {
@ -32,7 +32,7 @@ suspend fun Table<Value>.wrap(): Envelope = Envelope {
@DFExperimental @DFExperimental
@ExperimentalIoApi @ExperimentalIoApi
fun TextRows.Companion.readEnvelope(envelope: Envelope): TextRows { public fun TextRows.Companion.readEnvelope(envelope: Envelope): TextRows {
val header = envelope.meta.getIndexed("column") val header = envelope.meta.getIndexed("column")
.entries.sortedBy { it.key?.toInt() } .entries.sortedBy { it.key?.toInt() }
.map { (_, item) -> .map { (_, item) ->

16
dataforge-tables/src/commonMain/kotlin/hep/dataforge/tables/numericColumns.kt
View File

@ -4,9 +4,9 @@ import hep.dataforge.meta.Meta
import kotlin.reflect.KClass import kotlin.reflect.KClass
class RealColumn( public class RealColumn(
override val name: String, override val name: String,
val data: DoubleArray, public val data: DoubleArray,
override val meta: Meta = Meta.EMPTY override val meta: Meta = Meta.EMPTY
) : Column<Double> { ) : Column<Double> {
override val type: KClass<out Double> get() = Double::class override val type: KClass<out Double> get() = Double::class
@ -34,8 +34,8 @@ class RealColumn(
return result return result
} }
companion object { public companion object {
inline operator fun <reified T : Any> invoke( public inline operator fun <reified T : Any> invoke(
name: String, name: String,
data: DoubleArray, data: DoubleArray,
noinline metaBuilder: ColumnScheme.() -> Unit noinline metaBuilder: ColumnScheme.() -> Unit
@ -43,9 +43,9 @@ class RealColumn(
} }
} }
class IntColumn( public class IntColumn(
override val name: String, override val name: String,
val data: IntArray, public val data: IntArray,
override val meta: Meta = Meta.EMPTY override val meta: Meta = Meta.EMPTY
) : Column<Int> { ) : Column<Int> {
override val type: KClass<out Int> get() = Int::class override val type: KClass<out Int> get() = Int::class
@ -73,8 +73,8 @@ class IntColumn(
return result return result
} }
companion object { public companion object {
inline operator fun <reified T : Any> invoke( public inline operator fun <reified T : Any> invoke(
name: String, name: String,
data: IntArray, data: IntArray,
noinline metaBuilder: ColumnScheme.() -> Unit noinline metaBuilder: ColumnScheme.() -> Unit

31
dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/Dependency.kt
View File

@ -4,6 +4,7 @@ import hep.dataforge.data.DataFilter
import hep.dataforge.data.DataNode import hep.dataforge.data.DataNode
import hep.dataforge.data.filter import hep.dataforge.data.filter
import hep.dataforge.meta.Meta import hep.dataforge.meta.Meta
import hep.dataforge.meta.MetaBuilder
import hep.dataforge.meta.MetaRepr import hep.dataforge.meta.MetaRepr
import hep.dataforge.names.Name import hep.dataforge.names.Name
import hep.dataforge.names.asName import hep.dataforge.names.asName
@ -13,11 +14,11 @@ import hep.dataforge.names.plus
/** /**
* A dependency of the task which allows to lazily create a data tree for single dependency * A dependency of the task which allows to lazily create a data tree for single dependency
*/ */
sealed class Dependency : MetaRepr { public sealed class Dependency : MetaRepr {
abstract fun apply(workspace: Workspace): DataNode<Any> public abstract fun apply(workspace: Workspace): DataNode<Any>
} }
class DataDependency(val filter: DataFilter, val placement: Name = Name.EMPTY) : Dependency() { public class DataDependency(private val filter: DataFilter, private val placement: Name = Name.EMPTY) : Dependency() {
override fun apply(workspace: Workspace): DataNode<Any> { override fun apply(workspace: Workspace): DataNode<Any> {
val result = workspace.data.filter(filter) val result = workspace.data.filter(filter)
return if (placement.isEmpty()) { return if (placement.isEmpty()) {
@ -33,29 +34,29 @@ class DataDependency(val filter: DataFilter, val placement: Name = Name.EMPTY) :
} }
} }
class AllDataDependency(val placement: Name = Name.EMPTY) : Dependency() { public class AllDataDependency(private val placement: Name = Name.EMPTY) : Dependency() {
override fun apply(workspace: Workspace): DataNode<Any> = if (placement.isEmpty()) { override fun apply(workspace: Workspace): DataNode<Any> = if (placement.isEmpty()) {
workspace.data workspace.data
} else { } else {
DataNode.invoke(Any::class) { this[placement] = workspace.data } DataNode.invoke(Any::class) { this[placement] = workspace.data }
} }
override fun toMeta() = Meta { override fun toMeta(): MetaBuilder = Meta {
"data" put "@all" "data" put "@all"
"to" put placement.toString() "to" put placement.toString()
} }
} }
abstract class TaskDependency<out T : Any>( public abstract class TaskDependency<out T : Any>(
val meta: Meta, public val meta: Meta,
val placement: Name = Name.EMPTY public val placement: Name = Name.EMPTY
) : Dependency() { ) : Dependency() {
abstract fun resolveTask(workspace: Workspace): Task<T> public abstract fun resolveTask(workspace: Workspace): Task<T>
/** /**
* A name of the dependency for logging and serialization * A name of the dependency for logging and serialization
*/ */
abstract val name: Name public abstract val name: Name
override fun apply(workspace: Workspace): DataNode<T> { override fun apply(workspace: Workspace): DataNode<T> {
val task = resolveTask(workspace) val task = resolveTask(workspace)
@ -75,8 +76,8 @@ abstract class TaskDependency<out T : Any>(
} }
} }
class DirectTaskDependency<T : Any>( public class DirectTaskDependency<T : Any>(
val task: Task<T>, public val task: Task<T>,
meta: Meta, meta: Meta,
placement: Name placement: Name
) : TaskDependency<T>(meta, placement) { ) : TaskDependency<T>(meta, placement) {
@ -84,12 +85,12 @@ class DirectTaskDependency<T : Any>(
override val name: Name get() = DIRECT_TASK_NAME + task.name override val name: Name get() = DIRECT_TASK_NAME + task.name
companion object { public companion object {
val DIRECT_TASK_NAME = "@direct".asName() public val DIRECT_TASK_NAME: Name = "@direct".asName()
} }
} }
class WorkspaceTaskDependency( public class WorkspaceTaskDependency(
override val name: Name, override val name: Name,
meta: Meta, meta: Meta,
placement: Name placement: Name

12
dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/GenericTask.kt
View File

@ -1,8 +1,8 @@
package hep.dataforge.workspace package hep.dataforge.workspace
import hep.dataforge.data.DataNode import hep.dataforge.data.DataNode
import hep.dataforge.meta.descriptors.NodeDescriptor
import hep.dataforge.meta.Meta import hep.dataforge.meta.Meta
import hep.dataforge.meta.descriptors.NodeDescriptor
import hep.dataforge.meta.get import hep.dataforge.meta.get
import hep.dataforge.meta.node import hep.dataforge.meta.node
import hep.dataforge.names.Name import hep.dataforge.names.Name
@ -11,7 +11,7 @@ import kotlin.reflect.KClass
//data class TaskEnv(val workspace: Workspace, val model: TaskModel) //data class TaskEnv(val workspace: Workspace, val model: TaskModel)
class GenericTask<R : Any>( public class GenericTask<R : Any>(
override val name: Name, override val name: Name,
override val type: KClass<out R>, override val type: KClass<out R>,
override val descriptor: NodeDescriptor, override val descriptor: NodeDescriptor,
@ -19,14 +19,6 @@ class GenericTask<R : Any>(
private val dataTransform: Workspace.() -> TaskModel.(DataNode<Any>) -> DataNode<R> private val dataTransform: Workspace.() -> TaskModel.(DataNode<Any>) -> DataNode<R>
) : Task<R> { ) : Task<R> {
// private fun gather(workspace: Workspace, model: TaskModel): DataNode<Any> {
// return DataNode.invoke(Any::class) {
// model.dependencies.forEach { dep ->
// update(dep.apply(workspace))
// }
// }
// }
override fun run(workspace: Workspace, model: TaskModel): DataNode<R> { override fun run(workspace: Workspace, model: TaskModel): DataNode<R> {
//validate model //validate model
validate(model) validate(model)

4
dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/SimpleWorkspace.kt
View File

@ -11,7 +11,7 @@ import hep.dataforge.names.Name
/** /**
* A simple workspace without caching * A simple workspace without caching
*/ */
class SimpleWorkspace( public class SimpleWorkspace(
override val context: Context, override val context: Context,
override val data: DataNode<Any>, override val data: DataNode<Any>,
override val targets: Map<String, Meta>, override val targets: Map<String, Meta>,
@ -22,7 +22,7 @@ class SimpleWorkspace(
context.resolve<Task<*>>(Task.TYPE) + tasks.toMap() context.resolve<Task<*>>(Task.TYPE) + tasks.toMap()
} }
companion object { public companion object {
} }
} }

18
dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/Task.kt
View File

@ -2,23 +2,23 @@ package hep.dataforge.workspace
import hep.dataforge.context.Named import hep.dataforge.context.Named
import hep.dataforge.data.DataNode import hep.dataforge.data.DataNode
import hep.dataforge.meta.descriptors.Described
import hep.dataforge.meta.Meta import hep.dataforge.meta.Meta
import hep.dataforge.meta.descriptors.Described
import hep.dataforge.provider.Type import hep.dataforge.provider.Type
import hep.dataforge.workspace.Task.Companion.TYPE import hep.dataforge.workspace.Task.Companion.TYPE
import kotlin.reflect.KClass import kotlin.reflect.KClass
@Type(TYPE) @Type(TYPE)
interface Task<out R : Any> : Named, Described { public interface Task<out R : Any> : Named, Described {
/** /**
* Terminal task is the one that could not build model lazily * Terminal task is the one that could not build model lazily
*/ */
val isTerminal: Boolean get() = false public val isTerminal: Boolean get() = false
/** /**
* The explicit type of the node returned by the task * The explicit type of the node returned by the task
*/ */
val type: KClass<out R> public val type: KClass<out R>
/** /**
* Build a model for this task * Build a model for this task
@ -27,14 +27,14 @@ interface Task<out R : Any> : Named, Described {
* @param taskConfig * @param taskConfig
* @return * @return
*/ */
fun build(workspace: Workspace, taskConfig: Meta): TaskModel public fun build(workspace: Workspace, taskConfig: Meta): TaskModel
/** /**
* Check if the model is valid and is acceptable by the task. Throw exception if not. * Check if the model is valid and is acceptable by the task. Throw exception if not.
* *
* @param model * @param model
*/ */
fun validate(model: TaskModel) { public fun validate(model: TaskModel) {
if(this.name != model.name) error("The task $name could not be run with model from task ${model.name}") if(this.name != model.name) error("The task $name could not be run with model from task ${model.name}")
} }
@ -46,9 +46,9 @@ interface Task<out R : Any> : Named, Described {
* @param model - a model to be executed * @param model - a model to be executed
* @return * @return
*/ */
fun run(workspace: Workspace, model: TaskModel): DataNode<R> public fun run(workspace: Workspace, model: TaskModel): DataNode<R>
companion object { public companion object {
const val TYPE = "task" public const val TYPE = "task"
} }
} }

35
dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskBuilder.kt
View File

@ -14,11 +14,11 @@ import kotlin.jvm.JvmName
import kotlin.reflect.KClass import kotlin.reflect.KClass
@DFBuilder @DFBuilder
class TaskBuilder<R : Any>(val name: Name, val type: KClass<out R>) { public class TaskBuilder<R : Any>(public val name: Name, public val type: KClass<out R>) {
private var modelTransform: TaskModelBuilder.(Meta) -> Unit = { allData() } private var modelTransform: TaskModelBuilder.(Meta) -> Unit = { allData() }
// private val additionalDependencies = HashSet<Dependency>() // private val additionalDependencies = HashSet<Dependency>()
var descriptor: NodeDescriptor? = null private var descriptor: NodeDescriptor? = null
private val dataTransforms: MutableList<DataTransformation> = ArrayList() private val dataTransforms: MutableList<DataTransformation> = ArrayList()
/** /**
@ -43,7 +43,7 @@ class TaskBuilder<R : Any>(val name: Name, val type: KClass<out R>) {
// additionalDependencies.add(dependency) // additionalDependencies.add(dependency)
// } // }
fun model(modelTransform: TaskModelBuilder.(Meta) -> Unit) { public fun model(modelTransform: TaskModelBuilder.(Meta) -> Unit) {
this.modelTransform = modelTransform this.modelTransform = modelTransform
} }
@ -51,7 +51,7 @@ class TaskBuilder<R : Any>(val name: Name, val type: KClass<out R>) {
* Add a transformation on untyped data * Add a transformation on untyped data
*/ */
@JvmName("rawTransform") @JvmName("rawTransform")
fun transform( public fun transform(
from: String = "", from: String = "",
to: String = "", to: String = "",
block: TaskEnv.(DataNode<*>) -> DataNode<R> block: TaskEnv.(DataNode<*>) -> DataNode<R>
@ -62,7 +62,7 @@ class TaskBuilder<R : Any>(val name: Name, val type: KClass<out R>) {
} }
} }
fun <T : Any> transform( public fun <T : Any> transform(
inputType: KClass<out T>, inputType: KClass<out T>,
from: String = "", from: String = "",
to: String = "", to: String = "",
@ -75,7 +75,7 @@ class TaskBuilder<R : Any>(val name: Name, val type: KClass<out R>) {
} }
} }
inline fun <reified T : Any> transform( public inline fun <reified T : Any> transform(
from: String = "", from: String = "",
to: String = "", to: String = "",
noinline block: TaskEnv.(DataNode<T>) -> DataNode<R> noinline block: TaskEnv.(DataNode<T>) -> DataNode<R>
@ -86,7 +86,7 @@ class TaskBuilder<R : Any>(val name: Name, val type: KClass<out R>) {
/** /**
* Perform given action on data elements in `from` node in input and put the result to `to` node * Perform given action on data elements in `from` node in input and put the result to `to` node
*/ */
inline fun <reified T : Any> action( public inline fun <reified T : Any> action(
from: String = "", from: String = "",
to: String = "", to: String = "",
crossinline block: TaskEnv.() -> Action<T, R> crossinline block: TaskEnv.() -> Action<T, R>
@ -96,8 +96,8 @@ class TaskBuilder<R : Any>(val name: Name, val type: KClass<out R>) {
} }
} }
class TaskEnv(val name: Name, val meta: Meta, val context: Context, val data: DataNode<Any>) { public class TaskEnv(public val name: Name, public val meta: Meta, public val context: Context, public val data: DataNode<Any>) {
operator fun <T : Any> DirectTaskDependency<T>.invoke(): DataNode<T> = if (placement.isEmpty()) { public operator fun <T : Any> DirectTaskDependency<T>.invoke(): DataNode<T> = if (placement.isEmpty()) {
data.cast(task.type) data.cast(task.type)
} else { } else {
data[placement].node?.cast(task.type) data[placement].node?.cast(task.type)
@ -108,7 +108,7 @@ class TaskBuilder<R : Any>(val name: Name, val type: KClass<out R>) {
/** /**
* A customized map action with ability to change meta and name * A customized map action with ability to change meta and name
*/ */
inline fun <reified T : Any> mapAction( public inline fun <reified T : Any> mapAction(
from: String = "", from: String = "",
to: String = "", to: String = "",
crossinline block: MapActionBuilder<T, R>.(TaskEnv) -> Unit crossinline block: MapActionBuilder<T, R>.(TaskEnv) -> Unit
@ -127,7 +127,7 @@ class TaskBuilder<R : Any>(val name: Name, val type: KClass<out R>) {
/** /**
* A simple map action without changing meta or name * A simple map action without changing meta or name
*/ */
inline fun <reified T : Any> map( public inline fun <reified T : Any> map(
from: String = "", from: String = "",
to: String = "", to: String = "",
crossinline block: suspend TaskEnv.(T) -> R crossinline block: suspend TaskEnv.(T) -> R
@ -148,7 +148,7 @@ class TaskBuilder<R : Any>(val name: Name, val type: KClass<out R>) {
/** /**
* Join elements in gathered data by multiple groups * Join elements in gathered data by multiple groups
*/ */
inline fun <reified T : Any> reduceByGroup( public inline fun <reified T : Any> reduceByGroup(
from: String = "", from: String = "",
to: String = "", to: String = "",
crossinline block: ReduceGroupBuilder<T, R>.(TaskEnv) -> Unit //TODO needs KEEP-176 crossinline block: ReduceGroupBuilder<T, R>.(TaskEnv) -> Unit //TODO needs KEEP-176
@ -165,7 +165,7 @@ class TaskBuilder<R : Any>(val name: Name, val type: KClass<out R>) {
/** /**
* Join all elemlents in gathered data matching input type * Join all elemlents in gathered data matching input type
*/ */
inline fun <reified T : Any> reduce( public inline fun <reified T : Any> reduce(
from: String = "", from: String = "",
to: String = "", to: String = "",
crossinline block: suspend TaskEnv.(Map<Name, T>) -> R crossinline block: suspend TaskEnv.(Map<Name, T>) -> R
@ -188,7 +188,7 @@ class TaskBuilder<R : Any>(val name: Name, val type: KClass<out R>) {
/** /**
* Split each element in gathered data into fixed number of fragments * Split each element in gathered data into fixed number of fragments
*/ */
inline fun <reified T : Any> split( public inline fun <reified T : Any> split(
from: String = "", from: String = "",
to: String = "", to: String = "",
crossinline block: SplitBuilder<T, R>.(TaskEnv) -> Unit //TODO needs KEEP-176 crossinline block: SplitBuilder<T, R>.(TaskEnv) -> Unit //TODO needs KEEP-176
@ -205,16 +205,11 @@ class TaskBuilder<R : Any>(val name: Name, val type: KClass<out R>) {
/** /**
* Use DSL to create a descriptor for this task * Use DSL to create a descriptor for this task
*/ */
fun description(transform: NodeDescriptor.() -> Unit) { public fun description(transform: NodeDescriptor.() -> Unit) {
this.descriptor = NodeDescriptor().apply(transform) this.descriptor = NodeDescriptor().apply(transform)
} }
internal fun build(): GenericTask<R> { internal fun build(): GenericTask<R> {
// val actualTransform: TaskModelBuilder.(Meta) -> Unit = {
// modelTransform
// dependencies.addAll(additionalDependencies)
// }
return GenericTask( return GenericTask(
name, name,
type, type,

42
dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskModel.kt
View File

@ -21,7 +21,7 @@ import hep.dataforge.workspace.TaskModel.Companion.MODEL_TARGET_KEY
* @param meta the meta for the task (not for the whole configuration) * @param meta the meta for the task (not for the whole configuration)
* @param dependencies a list of direct dependencies for this task * @param dependencies a list of direct dependencies for this task
*/ */
data class TaskModel( public data class TaskModel(
val name: Name, val name: Name,
val meta: Meta, val meta: Meta,
val dependencies: Collection<Dependency> val dependencies: Collection<Dependency>
@ -45,15 +45,15 @@ data class TaskModel(
} }
} }
companion object { public companion object {
val MODEL_TARGET_KEY = "@target".asName() public val MODEL_TARGET_KEY = "@target".asName()
} }
} }
/** /**
* Build input for the task * Build input for the task
*/ */
fun TaskModel.buildInput(workspace: Workspace): DataTree<Any> { public fun TaskModel.buildInput(workspace: Workspace): DataTree<Any> {
return DataTreeBuilder(Any::class).apply { return DataTreeBuilder(Any::class).apply {
dependencies.forEach { dep -> dependencies.forEach { dep ->
update(dep.apply(workspace)) update(dep.apply(workspace))
@ -61,43 +61,43 @@ fun TaskModel.buildInput(workspace: Workspace): DataTree<Any> {
}.build() }.build()
} }
interface TaskDependencyContainer { public interface TaskDependencyContainer {
val defaultMeta: Meta public val defaultMeta: Meta
fun add(dependency: Dependency) public fun add(dependency: Dependency)
} }
/** /**
* Add dependency for a task defined in a workspace and resolved by * Add dependency for a task defined in a workspace and resolved by
*/ */
fun TaskDependencyContainer.dependsOn( public fun TaskDependencyContainer.dependsOn(
name: Name, name: Name,
placement: Name = Name.EMPTY, placement: Name = Name.EMPTY,
meta: Meta = defaultMeta meta: Meta = defaultMeta
): WorkspaceTaskDependency = ): WorkspaceTaskDependency =
WorkspaceTaskDependency(name, meta, placement).also { add(it) } WorkspaceTaskDependency(name, meta, placement).also { add(it) }
fun TaskDependencyContainer.dependsOn( public fun TaskDependencyContainer.dependsOn(
name: String, name: String,
placement: Name = Name.EMPTY, placement: Name = Name.EMPTY,
meta: Meta = defaultMeta meta: Meta = defaultMeta
): WorkspaceTaskDependency = ): WorkspaceTaskDependency =
dependsOn(name.toName(), placement, meta) dependsOn(name.toName(), placement, meta)
fun <T : Any> TaskDependencyContainer.dependsOn( public fun <T : Any> TaskDependencyContainer.dependsOn(
task: Task<T>, task: Task<T>,
placement: Name = Name.EMPTY, placement: Name = Name.EMPTY,
meta: Meta = defaultMeta meta: Meta = defaultMeta
): DirectTaskDependency<T> = ): DirectTaskDependency<T> =
DirectTaskDependency(task, meta, placement).also { add(it) } DirectTaskDependency(task, meta, placement).also { add(it) }
fun <T : Any> TaskDependencyContainer.dependsOn( public fun <T : Any> TaskDependencyContainer.dependsOn(
task: Task<T>, task: Task<T>,
placement: String, placement: String,
meta: Meta = defaultMeta meta: Meta = defaultMeta
): DirectTaskDependency<T> = ): DirectTaskDependency<T> =
DirectTaskDependency(task, meta, placement.toName()).also { add(it) } DirectTaskDependency(task, meta, placement.toName()).also { add(it) }
fun <T : Any> TaskDependencyContainer.dependsOn( public fun <T : Any> TaskDependencyContainer.dependsOn(
task: Task<T>, task: Task<T>,
placement: Name = Name.EMPTY, placement: Name = Name.EMPTY,
metaBuilder: MetaBuilder.() -> Unit metaBuilder: MetaBuilder.() -> Unit
@ -107,13 +107,13 @@ fun <T : Any> TaskDependencyContainer.dependsOn(
/** /**
* Add custom data dependency * Add custom data dependency
*/ */
fun TaskDependencyContainer.data(action: DataFilter.() -> Unit): DataDependency = public fun TaskDependencyContainer.data(action: DataFilter.() -> Unit): DataDependency =
DataDependency(DataFilter(action)).also { add(it) } DataDependency(DataFilter(action)).also { add(it) }
/** /**
* User-friendly way to add data dependency * User-friendly way to add data dependency
*/ */
fun TaskDependencyContainer.data(pattern: String? = null, from: String? = null, to: String? = null): DataDependency = public fun TaskDependencyContainer.data(pattern: String? = null, from: String? = null, to: String? = null): DataDependency =
data { data {
pattern?.let { this.pattern = it } pattern?.let { this.pattern = it }
from?.let { this.from = it } from?.let { this.from = it }
@ -123,17 +123,17 @@ fun TaskDependencyContainer.data(pattern: String? = null, from: String? = null,
/** /**
* Add all data as root node * Add all data as root node
*/ */
fun TaskDependencyContainer.allData(to: Name = Name.EMPTY) = AllDataDependency(to).also { add(it) } public fun TaskDependencyContainer.allData(to: Name = Name.EMPTY): AllDataDependency = AllDataDependency(to).also { add(it) }
/** /**
* A builder for [TaskModel] * A builder for [TaskModel]
*/ */
class TaskModelBuilder(val name: Name, meta: Meta = Meta.EMPTY) : TaskDependencyContainer { public class TaskModelBuilder(public val name: Name, meta: Meta = Meta.EMPTY) : TaskDependencyContainer {
/** /**
* Meta for current task. By default uses the whole input meta * Meta for current task. By default uses the whole input meta
*/ */
var meta: MetaBuilder = meta.builder() public var meta: MetaBuilder = meta.builder()
val dependencies = HashSet<Dependency>() private val dependencies: HashSet<Dependency> = HashSet<Dependency>()
override val defaultMeta: Meta get() = meta override val defaultMeta: Meta get() = meta
@ -141,11 +141,11 @@ class TaskModelBuilder(val name: Name, meta: Meta = Meta.EMPTY) : TaskDependency
dependencies.add(dependency) dependencies.add(dependency)
} }
var target: String by this.meta.string(key = MODEL_TARGET_KEY, default = "") public var target: String by this.meta.string(key = MODEL_TARGET_KEY, default = "")
fun build(): TaskModel = TaskModel(name, meta.seal(), dependencies) public fun build(): TaskModel = TaskModel(name, meta.seal(), dependencies)
} }
val TaskModel.target get() = meta[MODEL_TARGET_KEY]?.string ?: "" public val TaskModel.target: String get() = meta[MODEL_TARGET_KEY]?.string ?: ""

26
dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/Workspace.kt
View File

@ -15,21 +15,21 @@ import hep.dataforge.provider.Type
@Type(Workspace.TYPE) @Type(Workspace.TYPE)
interface Workspace : ContextAware, Provider { public interface Workspace : ContextAware, Provider {
/** /**
* The whole data node for current workspace * The whole data node for current workspace
*/ */
val data: DataNode<Any> public val data: DataNode<Any>
/** /**
* All targets associated with the workspace * All targets associated with the workspace
*/ */
val targets: Map<String, Meta> public val targets: Map<String, Meta>
/** /**
* All tasks associated with the workspace * All tasks associated with the workspace
*/ */
val tasks: Map<Name, Task<*>> public val tasks: Map<Name, Task<*>>
override fun provideTop(target: String): Map<Name, Any> { override fun provideTop(target: String): Map<Name, Any> {
return when (target) { return when (target) {
@ -44,7 +44,7 @@ interface Workspace : ContextAware, Provider {
/** /**
* Invoke a task in the workspace utilizing caching if possible * Invoke a task in the workspace utilizing caching if possible
*/ */
fun <R : Any> run(task: Task<R>, config: Meta): DataNode<R> { public fun <R : Any> run(task: Task<R>, config: Meta): DataNode<R> {
context.activate(this) context.activate(this)
try { try {
val model = task.build(this, config) val model = task.build(this, config)
@ -55,27 +55,27 @@ interface Workspace : ContextAware, Provider {
} }
} }
companion object { public companion object {
const val TYPE = "workspace" public const val TYPE: String = "workspace"
operator fun invoke(parent: Context = Global, block: SimpleWorkspaceBuilder.() -> Unit): SimpleWorkspace = public operator fun invoke(parent: Context = Global, block: SimpleWorkspaceBuilder.() -> Unit): SimpleWorkspace =
SimpleWorkspaceBuilder(parent).apply(block).build() SimpleWorkspaceBuilder(parent).apply(block).build()
} }
} }
fun Workspace.run(task: Task<*>, target: String): DataNode<Any> { public fun Workspace.run(task: Task<*>, target: String): DataNode<Any> {
val meta = targets[target] ?: error("A target with name $target not found in ${this}") val meta = targets[target] ?: error("A target with name $target not found in ${this}")
return run(task, meta) return run(task, meta)
} }
fun Workspace.run(task: String, target: String) = public fun Workspace.run(task: String, target: String): DataNode<Any> =
tasks[task.toName()]?.let { run(it, target) } ?: error("Task with name $task not found") tasks[task.toName()]?.let { run(it, target) } ?: error("Task with name $task not found")
fun Workspace.run(task: String, meta: Meta) = public fun Workspace.run(task: String, meta: Meta): DataNode<Any> =
tasks[task.toName()]?.let { run(it, meta) } ?: error("Task with name $task not found") tasks[task.toName()]?.let { run(it, meta) } ?: error("Task with name $task not found")
fun Workspace.run(task: String, block: MetaBuilder.() -> Unit = {}) = public fun Workspace.run(task: String, block: MetaBuilder.() -> Unit = {}): DataNode<Any> =
run(task, Meta(block)) run(task, Meta(block))
fun <T: Any> Workspace.run(task: Task<T>, metaBuilder: MetaBuilder.() -> Unit = {}): DataNode<T> = public fun <T: Any> Workspace.run(task: Task<T>, metaBuilder: MetaBuilder.() -> Unit = {}): DataNode<T> =
run(task, Meta(metaBuilder)) run(task, Meta(metaBuilder))

32
dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/WorkspaceBuilder.kt
View File

@ -12,25 +12,25 @@ import kotlin.jvm.JvmName
import kotlin.reflect.KClass import kotlin.reflect.KClass
@DFBuilder @DFBuilder
interface WorkspaceBuilder { public interface WorkspaceBuilder {
val parentContext: Context public val parentContext: Context
var context: Context public var context: Context
var data: DataTreeBuilder<Any> public var data: DataTreeBuilder<Any>
var tasks: MutableSet<Task<Any>> public var tasks: MutableSet<Task<Any>>
var targets: MutableMap<String, Meta> public var targets: MutableMap<String, Meta>
fun build(): Workspace public fun build(): Workspace
} }
/** /**
* Set the context for future workspcace * Set the context for future workspcace
*/ */
fun WorkspaceBuilder.context(name: String = "WORKSPACE", block: ContextBuilder.() -> Unit = {}) { public fun WorkspaceBuilder.context(name: String = "WORKSPACE", block: ContextBuilder.() -> Unit = {}) {
context = ContextBuilder(parentContext, name).apply(block).build() context = ContextBuilder(parentContext, name).apply(block).build()
} }
inline fun <reified T : Any> WorkspaceBuilder.data( public inline fun <reified T : Any> WorkspaceBuilder.data(
name: Name = Name.EMPTY, name: Name = Name.EMPTY,
noinline block: DataTreeBuilder<T>.() -> Unit noinline block: DataTreeBuilder<T>.() -> Unit
): DataNode<T> { ): DataNode<T> {
@ -45,20 +45,20 @@ inline fun <reified T : Any> WorkspaceBuilder.data(
} }
@JvmName("rawData") @JvmName("rawData")
fun WorkspaceBuilder.data( public fun WorkspaceBuilder.data(
name: Name = Name.EMPTY, name: Name = Name.EMPTY,
block: DataTreeBuilder<Any>.() -> Unit block: DataTreeBuilder<Any>.() -> Unit
): DataNode<Any> = data<Any>(name, block) ): DataNode<Any> = data<Any>(name, block)
fun WorkspaceBuilder.target(name: String, block: MetaBuilder.() -> Unit) { public fun WorkspaceBuilder.target(name: String, block: MetaBuilder.() -> Unit) {
targets[name] = Meta(block).seal() targets[name] = Meta(block).seal()
} }
/** /**
* Use existing target as a base updating it with the block * Use existing target as a base updating it with the block
*/ */
fun WorkspaceBuilder.target(name: String, base: String, block: MetaBuilder.() -> Unit) { public fun WorkspaceBuilder.target(name: String, base: String, block: MetaBuilder.() -> Unit) {
val parentTarget = targets[base] ?: error("Base target with name $base not found") val parentTarget = targets[base] ?: error("Base target with name $base not found")
targets[name] = parentTarget.builder() targets[name] = parentTarget.builder()
.apply { "@baseTarget" put base } .apply { "@baseTarget" put base }
@ -66,19 +66,19 @@ fun WorkspaceBuilder.target(name: String, base: String, block: MetaBuilder.() ->
.seal() .seal()
} }
fun <T : Any> WorkspaceBuilder.task( public fun <T : Any> WorkspaceBuilder.task(
name: String, name: String,
type: KClass<out T>, type: KClass<out T>,
builder: TaskBuilder<T>.() -> Unit builder: TaskBuilder<T>.() -> Unit
): Task<T> = TaskBuilder(name.toName(), type).apply(builder).build().also { tasks.add(it) } ): Task<T> = TaskBuilder(name.toName(), type).apply(builder).build().also { tasks.add(it) }
inline fun <reified T : Any> WorkspaceBuilder.task( public inline fun <reified T : Any> WorkspaceBuilder.task(
name: String, name: String,
noinline builder: TaskBuilder<T>.() -> Unit noinline builder: TaskBuilder<T>.() -> Unit
): Task<T> = task(name, T::class, builder) ): Task<T> = task(name, T::class, builder)
@JvmName("rawTask") @JvmName("rawTask")
fun WorkspaceBuilder.task( public fun WorkspaceBuilder.task(
name: String, name: String,
builder: TaskBuilder<Any>.() -> Unit builder: TaskBuilder<Any>.() -> Unit
): Task<Any> = task(name, Any::class, builder) ): Task<Any> = task(name, Any::class, builder)
@ -86,7 +86,7 @@ fun WorkspaceBuilder.task(
/** /**
* A builder for a simple workspace * A builder for a simple workspace
*/ */
class SimpleWorkspaceBuilder(override val parentContext: Context) : WorkspaceBuilder { public class SimpleWorkspaceBuilder(override val parentContext: Context) : WorkspaceBuilder {
override var context: Context = parentContext override var context: Context = parentContext
override var data: DataTreeBuilder<Any> = DataTreeBuilder(Any::class) override var data: DataTreeBuilder<Any> = DataTreeBuilder(Any::class)
override var tasks: MutableSet<Task<Any>> = HashSet() override var tasks: MutableSet<Task<Any>> = HashSet()

10
dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/WorkspacePlugin.kt
View File

@ -9,9 +9,9 @@ import kotlin.reflect.KClass
/** /**
* An abstract plugin with some additional boilerplate to effectively work with workspace context * An abstract plugin with some additional boilerplate to effectively work with workspace context
*/ */
abstract class WorkspacePlugin : AbstractPlugin() { public abstract class WorkspacePlugin : AbstractPlugin() {
private val _tasks = HashSet<Task<*>>() private val _tasks = HashSet<Task<*>>()
val tasks: Collection<Task<*>> get() = _tasks public val tasks: Collection<Task<*>> get() = _tasks
override fun provideTop(target: String): Map<Name, Any> { override fun provideTop(target: String): Map<Name, Any> {
return when (target) { return when (target) {
@ -20,11 +20,11 @@ abstract class WorkspacePlugin : AbstractPlugin() {
} }
} }
fun task(task: Task<*>){ public fun task(task: Task<*>){
_tasks.add(task) _tasks.add(task)
} }
fun <T : Any> task( public fun <T : Any> task(
name: String, name: String,
type: KClass<out T>, type: KClass<out T>,
builder: TaskBuilder<T>.() -> Unit builder: TaskBuilder<T>.() -> Unit
@ -32,7 +32,7 @@ abstract class WorkspacePlugin : AbstractPlugin() {
_tasks.add(it) _tasks.add(it)
} }
inline fun <reified T : Any> task( public inline fun <reified T : Any> task(
name: String, name: String,
noinline builder: TaskBuilder<T>.() -> Unit noinline builder: TaskBuilder<T>.() -> Unit
) = task(name, T::class, builder) ) = task(name, T::class, builder)

4
dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/envelopeData.kt
View File

@ -8,11 +8,11 @@ import kotlin.reflect.KClass
/** /**
* Convert an [Envelope] to a data via given format. The actual parsing is done lazily. * Convert an [Envelope] to a data via given format. The actual parsing is done lazily.
*/ */
fun <T : Any> Envelope.toData(type: KClass<out T>, format: IOFormat<T>): Data<T> = Data(type, meta) { public fun <T : Any> Envelope.toData(type: KClass<out T>, format: IOFormat<T>): Data<T> = Data(type, meta) {
data?.readWith(format) ?: error("Can't convert envelope without data to Data") data?.readWith(format) ?: error("Can't convert envelope without data to Data")
} }
suspend fun <T : Any> Data<T>.toEnvelope(format: IOFormat<T>): Envelope { public suspend fun <T : Any> Data<T>.toEnvelope(format: IOFormat<T>): Envelope {
val obj = await() val obj = await()
val binary = format.toBinary(obj) val binary = format.toBinary(obj)
return SimpleEnvelope(meta, binary) return SimpleEnvelope(meta, binary)