kscience/dataforge-core
8
0
Fork 0
Code Issues Pull Requests 1 Packages Projects Releases 12 Wiki Activity

Merge pull request #23 from mipt-npm/dev

Browse Source 
0.1.4
This commit is contained in:
Alexander Nozik 2019-11-01 21:27:52 +03:00 committed by GitHub
commit 5dc7929475
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
103 changed files with 3478 additions and 1008 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
build.gradle.kts
View File

@ -1,9 +1,10 @@
plugins {
id("scientifik.mpp") version "0.1.4" apply false
id("scientifik.publish") version "0.1.4" apply false
id("scientifik.mpp") version "0.2.1" apply false
id("scientifik.jvm") version "0.2.1" apply false
id("scientifik.publish") version "0.2.1" apply false
}
val dataforgeVersion by extra("0.1.3")
val dataforgeVersion by extra("0.1.4")
val bintrayRepo by extra("dataforge")
val githubProject by extra("dataforge-core")

25
dataforge-context/src/commonMain/kotlin/hep/dataforge/context/AbstractPlugin.kt
View File

@ -3,10 +3,13 @@ package hep.dataforge.context
import hep.dataforge.meta.EmptyMeta
import hep.dataforge.meta.Meta
import hep.dataforge.names.Name
import hep.dataforge.names.toName
import kotlin.properties.ReadOnlyProperty
import kotlin.reflect.KClass
import kotlin.reflect.KProperty
abstract class AbstractPlugin(override val meta: Meta = EmptyMeta) : Plugin {
private var _context: Context? = null
private val dependencies = ArrayList<PluginFactory<*>>()
override val context: Context
get() = _context ?: error("Plugin $tag is not attached")
@ -19,9 +22,23 @@ abstract class AbstractPlugin(override val meta: Meta = EmptyMeta) : Plugin {
this._context = null
}
override fun provideTop(target: String): Map<Name, Any> = emptyMap()
final override fun dependsOn(): List<PluginFactory<*>> = dependencies
companion object{
fun <T: Named> Collection<T>.toMap(): Map<Name, T> = associate { it.name.toName() to it }
/**
* Register plugin dependency and return a delegate which provides lazily initialized reference to dependent plugin
*/
protected fun <P : Plugin> require(factory: PluginFactory<P>): ReadOnlyProperty<AbstractPlugin, P> {
dependencies.add(factory)
return PluginDependencyDelegate(factory.type)
}
override fun provideTop(target: String): Map<Name, Any> = emptyMap()
}
fun <T : Named> Collection<T>.toMap(): Map<Name, T> = associate { it.name to it }
private class PluginDependencyDelegate<P : Plugin>(val type: KClass<out P>) : ReadOnlyProperty<AbstractPlugin, P> {
override fun getValue(thisRef: AbstractPlugin, property: KProperty<*>): P {
return thisRef.context.plugins[type] ?: error("Plugin with type $type not found")
}
}

20
dataforge-context/src/commonMain/kotlin/hep/dataforge/context/Context.kt
View File

@ -2,8 +2,8 @@ package hep.dataforge.context
import hep.dataforge.meta.*
import hep.dataforge.names.Name
import hep.dataforge.names.appendLeft
import hep.dataforge.names.toName
import hep.dataforge.names.asName
import hep.dataforge.names.plus
import hep.dataforge.provider.Provider
import hep.dataforge.provider.top
import hep.dataforge.values.Value
@ -27,7 +27,7 @@ import kotlin.jvm.JvmName
* @author Alexander Nozik
*/
open class Context(
final override val name: String,
final override val name: Name,
val parent: Context? = Global
) : Named, MetaRepr, Provider, CoroutineScope {
@ -45,7 +45,7 @@ open class Context(
/**
* Context logger
*/
val logger: KLogger = KotlinLogging.logger(name)
val logger: KLogger = KotlinLogging.logger(name.toString())
/**
* A [PluginManager] for current context
@ -64,7 +64,7 @@ open class Context(
override fun provideTop(target: String): Map<Name, Any> {
return when (target) {
Value.TYPE -> properties.sequence().toMap()
Plugin.PLUGIN_TARGET -> plugins.sequence(true).associateBy { it.name.toName() }
Plugin.PLUGIN_TARGET -> plugins.sequence(true).associateBy { it.name }
else -> emptyMap()
}
}
@ -105,8 +105,8 @@ open class Context(
override fun toMeta(): Meta = buildMeta {
"parent" to parent?.name
"properties" to properties.seal()
"plugins" to plugins.map { it.toMeta() }
"properties" put properties.seal()
"plugins" put plugins.map { it.toMeta() }
}
}
@ -118,14 +118,14 @@ fun Context.content(target: String): Map<Name, Any> = content<Any>(target)
@JvmName("typedContent")
inline fun <reified T : Any> Context.content(target: String): Map<Name, T> =
plugins.flatMap { plugin ->
plugin.top<T>(target).entries.map { (it.key.appendLeft(plugin.name)) to it.value }
plugin.top<T>(target).entries.map { (plugin.name + it.key) to it.value }
}.associate { it }
/**
* A global root context. Closing [Global] terminates the framework.
*/
object Global : Context("GLOBAL", null) {
object Global : Context("GLOBAL".asName(), null) {
/**
* Closing all contexts
*
@ -173,7 +173,7 @@ interface ContextAware {
val logger: KLogger
get() = if (this is Named) {
KotlinLogging.logger(context.name + "." + (this as Named).name)
KotlinLogging.logger((context.name + this.name).toString())
} else {
context.logger
}

5
dataforge-context/src/commonMain/kotlin/hep/dataforge/context/ContextBuilder.kt
View File

@ -2,11 +2,12 @@ package hep.dataforge.context
import hep.dataforge.meta.MetaBuilder
import hep.dataforge.meta.buildMeta
import hep.dataforge.names.toName
/**
* A convenience builder for context
*/
class ContextBuilder(var name: String = "@anonimous", val parent: Context = Global) {
class ContextBuilder(var name: String = "@anonymous", val parent: Context = Global) {
private val plugins = ArrayList<Plugin>()
private var meta = MetaBuilder()
@ -31,7 +32,7 @@ class ContextBuilder(var name: String = "@anonimous", val parent: Context = Glob
}
fun build(): Context {
return Context(name, parent).apply {
return Context(name.toName(), parent).apply {
this@ContextBuilder.plugins.forEach {
plugins.load(it)
}

8
dataforge-context/src/commonMain/kotlin/hep/dataforge/context/Factory.kt Normal file
View File

@ -0,0 +1,8 @@
package hep.dataforge.context
import hep.dataforge.meta.EmptyMeta
import hep.dataforge.meta.Meta
interface Factory<out T : Any> {
operator fun invoke(meta: Meta = EmptyMeta, context: Context = Global): T
}

13
dataforge-context/src/commonMain/kotlin/hep/dataforge/context/Named.kt
View File

@ -15,6 +15,10 @@
*/
package hep.dataforge.context
import hep.dataforge.names.Name
import hep.dataforge.names.asName
import hep.dataforge.names.isEmpty
/**
* Any object that have name
*
@ -27,10 +31,9 @@ interface Named {
*
* @return
*/
val name: String
val name: Name
companion object {
const val ANONYMOUS = ""
/**
* Get the name of given object. If object is Named its name is used,
@ -39,11 +42,11 @@ interface Named {
* @param obj
* @return
*/
fun nameOf(obj: Any): String {
fun nameOf(obj: Any): Name {
return if (obj is Named) {
obj.name
} else {
obj.toString()
obj.toString().asName()
}
}
}
@ -54,4 +57,4 @@ interface Named {
* @return
*/
val Named.isAnonymous: Boolean
get() = this.name == Named.ANONYMOUS
get() = this.name.isEmpty()

12
dataforge-context/src/commonMain/kotlin/hep/dataforge/context/Plugin.kt
View File

@ -3,6 +3,8 @@ package hep.dataforge.context
import hep.dataforge.meta.Meta
import hep.dataforge.meta.MetaRepr
import hep.dataforge.meta.buildMeta
import hep.dataforge.names.Name
import hep.dataforge.names.toName
import hep.dataforge.provider.Provider
/**
@ -37,7 +39,7 @@ interface Plugin : Named, ContextAware, Provider, MetaRepr {
*
* @return
*/
override val name: String get() = tag.name
override val name: Name get() = tag.name.toName()
/**
* Plugin dependencies which are required to attach this plugin. Plugin
@ -46,7 +48,7 @@ interface Plugin : Named, ContextAware, Provider, MetaRepr {
*
* @return
*/
fun dependsOn(): List<PluginFactory<*>> = emptyList()
fun dependsOn(): Collection<PluginFactory<*>>
/**
* Start this plugin and attach registration info to the context. This method
@ -64,10 +66,10 @@ interface Plugin : Named, ContextAware, Provider, MetaRepr {
fun detach()
override fun toMeta(): Meta = buildMeta {
"context" to context.name
"context" put context.name.toString()
"type" to this::class.simpleName
"tag" to tag
"meta" to meta
"tag" put tag
"meta" put meta
}
companion object {

14
dataforge-context/src/commonMain/kotlin/hep/dataforge/context/PluginManager.kt
View File

@ -38,7 +38,7 @@ class PluginManager(override val context: Context) : ContextAware, Iterable<Plug
* @param recursive search for parent [PluginManager] plugins
* @param predicate condition for the plugin
*/
fun get(recursive: Boolean = true, predicate: (Plugin) -> Boolean): Plugin? = sequence(recursive).find(predicate)
fun find(recursive: Boolean = true, predicate: (Plugin) -> Boolean): Plugin? = sequence(recursive).find(predicate)
/**
@ -47,7 +47,7 @@ class PluginManager(override val context: Context) : ContextAware, Iterable<Plug
* @param tag
* @return
*/
operator fun get(tag: PluginTag, recursive: Boolean = true): Plugin? = get(recursive) { tag.matches(it.tag) }
operator fun get(tag: PluginTag, recursive: Boolean = true): Plugin? = find(recursive) { tag.matches(it.tag) }
/**
@ -63,11 +63,13 @@ class PluginManager(override val context: Context) : ContextAware, Iterable<Plug
*/
@Suppress("UNCHECKED_CAST")
operator fun <T : Any> get(type: KClass<T>, tag: PluginTag? = null, recursive: Boolean = true): T? =
get(recursive) { type.isInstance(it) && (tag == null || tag.matches(it.tag)) } as T?
find(recursive) { type.isInstance(it) && (tag == null || tag.matches(it.tag)) } as T?
inline fun <reified T : Any> get(tag: PluginTag? = null, recursive: Boolean = true): T? =
inline operator fun <reified T : Any> get(tag: PluginTag? = null, recursive: Boolean = true): T? =
get(T::class, tag, recursive)
inline operator fun <reified T : Plugin> get(factory: PluginFactory<T>, recursive: Boolean = true): T? =
get(factory.type, factory.tag, recursive)
/**
* Load given plugin into this manager and return loaded instance.
@ -97,7 +99,7 @@ class PluginManager(override val context: Context) : ContextAware, Iterable<Plug
* Load a plugin using its factory
*/
fun <T : Plugin> load(factory: PluginFactory<T>, meta: Meta = EmptyMeta): T =
load(factory(meta))
load(factory(meta, context))
fun <T : Plugin> load(factory: PluginFactory<T>, metaBuilder: MetaBuilder.() -> Unit): T =
load(factory, buildMeta(metaBuilder))
@ -122,7 +124,7 @@ class PluginManager(override val context: Context) : ContextAware, Iterable<Plug
fun <T : Plugin> fetch(factory: PluginFactory<T>, recursive: Boolean = true, meta: Meta = EmptyMeta): T {
val loaded = get(factory.type, factory.tag, recursive)
return when {
loaded == null -> load(factory(meta))
loaded == null -> load(factory(meta, context))
loaded.meta == meta -> loaded // if meta is the same, return existing plugin
else -> throw RuntimeException("Can't load plugin with tag ${factory.tag}. Plugin with this tag and different configuration already exists in context.")
}

14
dataforge-context/src/commonMain/kotlin/hep/dataforge/context/PluginRepository.kt
View File

@ -4,10 +4,9 @@ import hep.dataforge.meta.EmptyMeta
import hep.dataforge.meta.Meta
import kotlin.reflect.KClass
interface PluginFactory<T : Plugin> {
interface PluginFactory<T : Plugin> : Factory<T> {
val tag: PluginTag
val type: KClass<out T>
operator fun invoke(meta: Meta = EmptyMeta): T
}
expect object PluginRepository {
@ -25,25 +24,26 @@ expect object PluginRepository {
* Fetch specific plugin and instantiate it with given meta
*/
fun PluginRepository.fetch(tag: PluginTag, meta: Meta = EmptyMeta): Plugin =
list().find { it.tag.matches(tag) }?.invoke(meta) ?: error("Plugin with tag $tag not found in the repository")
list().find { it.tag.matches(tag) }?.invoke(meta = meta)
?: error("Plugin with tag $tag not found in the repository")
fun <T : Plugin> PluginRepository.register(
tag: PluginTag,
type: KClass<out T>,
constructor: (Meta) -> T
constructor: (Context, Meta) -> T
): PluginFactory<T> {
val factory = object : PluginFactory<T> {
override val tag: PluginTag = tag
override val type: KClass<out T> = type
override fun invoke(meta: Meta): T = constructor(meta)
override fun invoke(meta: Meta, context: Context): T = constructor(context, meta)
}
register(factory)
return factory
}
inline fun <reified T : Plugin> PluginRepository.register(tag: PluginTag, noinline constructor: (Meta) -> T) =
inline fun <reified T : Plugin> PluginRepository.register(tag: PluginTag, noinline constructor: (Context, Meta) -> T) =
register(tag, T::class, constructor)
fun PluginRepository.register(plugin: Plugin) = register(plugin.tag, plugin::class) { plugin }
fun PluginRepository.register(plugin: Plugin) = register(plugin.tag, plugin::class) { _, _ -> plugin }

6
dataforge-context/src/commonMain/kotlin/hep/dataforge/context/PluginTag.kt
View File

@ -37,9 +37,9 @@ data class PluginTag(
override fun toString(): String = listOf(group, name, version).joinToString(separator = ":")
override fun toMeta(): Meta = buildMeta {
"name" to name
"group" to group
"version" to version
"name" put name
"group" put group
"version" put version
}
companion object {

3
dataforge-context/src/commonTest/kotlin/hep/dataforge/context/ContextTest.kt
View File

@ -5,7 +5,6 @@ import hep.dataforge.names.appendLeft
import hep.dataforge.names.toName
import kotlin.test.Test
import kotlin.test.assertEquals
import kotlin.test.assertTrue
class ContextTest {
@ -26,7 +25,7 @@ class ContextTest {
val members = Global.content<Name>("test")
assertEquals(3, members.count())
members.forEach {
assertTrue{it.key == it.value.appendLeft("test")}
assertEquals(it.key, it.value.appendLeft("test"))
}
}

41
dataforge-data/src/commonMain/kotlin/hep/dataforge/data/Data.kt
View File

@ -1,8 +1,6 @@
package hep.dataforge.data
import hep.dataforge.meta.EmptyMeta
import hep.dataforge.meta.Meta
import hep.dataforge.meta.MetaRepr
import hep.dataforge.meta.*
import kotlinx.coroutines.CoroutineScope
import kotlin.coroutines.CoroutineContext
import kotlin.coroutines.EmptyCoroutineContext
@ -11,7 +9,7 @@ import kotlin.reflect.KClass
/**
* A data element characterized by its meta
*/
interface Data<out T : Any> : Goal<T>, MetaRepr {
interface Data<out T : Any> : Goal<T>, MetaRepr{
/**
* Type marker for the data. The type is known before the calculation takes place so it could be checked.
*/
@ -21,7 +19,12 @@ interface Data<out T : Any> : Goal<T>, MetaRepr {
*/
val meta: Meta
override fun toMeta(): Meta = meta
override fun toMeta(): Meta = buildMeta {
"type" put (type.simpleName?:"undefined")
if(!meta.isEmpty()) {
"meta" put meta
}
}
companion object {
const val TYPE = "data"
@ -34,7 +37,7 @@ interface Data<out T : Any> : Goal<T>, MetaRepr {
block: suspend CoroutineScope.() -> T
): Data<T> = DynamicData(type, meta, context, dependencies, block)
operator inline fun <reified T : Any> invoke(
inline operator fun <reified T : Any> invoke(
meta: Meta = EmptyMeta,
context: CoroutineContext = EmptyCoroutineContext,
dependencies: Collection<Data<*>> = emptyList(),
@ -50,7 +53,7 @@ interface Data<out T : Any> : Goal<T>, MetaRepr {
block: suspend CoroutineScope.() -> T
): Data<T> = NamedData(name, invoke(type, meta, context, dependencies, block))
operator inline fun <reified T : Any> invoke(
inline operator fun <reified T : Any> invoke(
name: String,
meta: Meta = EmptyMeta,
context: CoroutineContext = EmptyCoroutineContext,
@ -65,18 +68,6 @@ interface Data<out T : Any> : Goal<T>, MetaRepr {
}
fun <R : Any, T : R> Data<T>.cast(type: KClass<R>): Data<R> {
return object : Data<R> by this {
override val type: KClass<out R> = type
}
}
/**
* Upcast a [Data] to a supertype
*/
inline fun <reified R : Any, T : R> Data<T>.cast(): Data<R> = cast(R::class)
class DynamicData<T : Any>(
override val type: KClass<out T>,
override val meta: Meta = EmptyMeta,
@ -94,7 +85,7 @@ class StaticData<T : Any>(
class NamedData<out T : Any>(val name: String, data: Data<T>) : Data<T> by data
fun <T : Any, R : Any> Data<T>.pipe(
fun <T : Any, R : Any> Data<T>.map(
outputType: KClass<out R>,
coroutineContext: CoroutineContext = EmptyCoroutineContext,
meta: Meta = this.meta,
@ -107,7 +98,7 @@ fun <T : Any, R : Any> Data<T>.pipe(
/**
* Create a data pipe
*/
inline fun <T : Any, reified R : Any> Data<T>.pipe(
inline fun <T : Any, reified R : Any> Data<T>.map(
coroutineContext: CoroutineContext = EmptyCoroutineContext,
meta: Meta = this.meta,
noinline block: suspend CoroutineScope.(T) -> R
@ -118,7 +109,7 @@ inline fun <T : Any, reified R : Any> Data<T>.pipe(
/**
* Create a joined data.
*/
inline fun <T : Any, reified R : Any> Collection<Data<T>>.join(
inline fun <T : Any, reified R : Any> Collection<Data<T>>.reduce(
coroutineContext: CoroutineContext = EmptyCoroutineContext,
meta: Meta,
noinline block: suspend CoroutineScope.(Collection<T>) -> R
@ -128,10 +119,10 @@ inline fun <T : Any, reified R : Any> Collection<Data<T>>.join(
coroutineContext,
this
) {
block(map { this.run { it.await(this) } })
block(map { run { it.await(this) } })
}
fun <K, T : Any, R : Any> Map<K, Data<T>>.join(
fun <K, T : Any, R : Any> Map<K, Data<T>>.reduce(
outputType: KClass<out R>,
coroutineContext: CoroutineContext = EmptyCoroutineContext,
meta: Meta,
@ -152,7 +143,7 @@ fun <K, T : Any, R : Any> Map<K, Data<T>>.join(
* @param T type of the input goal
* @param R type of the result goal
*/
inline fun <K, T : Any, reified R : Any> Map<K, Data<T>>.join(
inline fun <K, T : Any, reified R : Any> Map<K, Data<T>>.reduce(
coroutineContext: CoroutineContext = EmptyCoroutineContext,
meta: Meta,
noinline block: suspend CoroutineScope.(Map<K, T>) -> R

13
dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataFilter.kt
View File

@ -5,12 +5,23 @@ import hep.dataforge.names.toName
class DataFilter(override val config: Config) : Specific {
/**
* A source node for the filter
*/
var from by string()
/**
* A target placement for the filtered node
*/
var to by string()
var pattern by string("*.")
/**
* A regular expression pattern for the filter
*/
var pattern by string(".*")
// val prefix by string()
// val suffix by string()
fun isEmpty(): Boolean = config.isEmpty()
companion object : Specification<DataFilter> {
override fun wrap(config: Config): DataFilter = DataFilter(config)
}

95
dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataNode.kt
View File

@ -1,5 +1,6 @@
package hep.dataforge.data
import hep.dataforge.meta.*
import hep.dataforge.names.*
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Job
@ -9,22 +10,26 @@ import kotlin.collections.component2
import kotlin.collections.set
import kotlin.reflect.KClass
sealed class DataItem<out T : Any> {
sealed class DataItem<out T : Any> : MetaRepr {
abstract val type: KClass<out T>
class Node<out T : Any>(val value: DataNode<T>) : DataItem<T>() {
override val type: KClass<out T> get() = value.type
override fun toMeta(): Meta = value.toMeta()
}
class Leaf<out T : Any>(val value: Data<T>) : DataItem<T>() {
override val type: KClass<out T> get() = value.type
override fun toMeta(): Meta = value.toMeta()
}
}
/**
* A tree-like data structure grouped into the node. All data inside the node must inherit its type
*/
interface DataNode<out T : Any> {
interface DataNode<out T : Any> : MetaRepr {
/**
* The minimal common ancestor to all data in the node
@ -33,12 +38,24 @@ interface DataNode<out T : Any> {
val items: Map<NameToken, DataItem<T>>
override fun toMeta(): Meta = buildMeta {
"type" put (type.simpleName ?: "undefined")
"items" put {
this@DataNode.items.forEach {
it.key.toString() put it.value.toMeta()
}
}
}
companion object {
const val TYPE = "dataNode"
fun <T : Any> build(type: KClass<out T>, block: DataTreeBuilder<T>.() -> Unit) =
operator fun <T : Any> invoke(type: KClass<out T>, block: DataTreeBuilder<T>.() -> Unit) =
DataTreeBuilder(type).apply(block).build()
inline operator fun <reified T : Any> invoke(noinline block: DataTreeBuilder<T>.() -> Unit) =
DataTreeBuilder(T::class).apply(block).build()
fun <T : Any> builder(type: KClass<out T>) = DataTreeBuilder(type)
}
}
@ -68,10 +85,12 @@ fun DataNode<*>.joinAll(scope: CoroutineScope): Job = scope.launch {
operator fun <T : Any> DataNode<T>.get(name: Name): DataItem<T>? = when (name.length) {
0 -> error("Empty name")
1 -> (items[name.first()] as? DataItem.Leaf)
1 -> items[name.first()]
else -> get(name.first()!!.asName()).node?.get(name.cutFirst())
}
operator fun <T : Any> DataNode<T>.get(name: String): DataItem<T>? = get(name.toName())
/**
* Sequence of all children including nodes
*/
@ -108,7 +127,9 @@ class DataTree<out T : Any> internal constructor(
override val type: KClass<out T>,
override val items: Map<NameToken, DataItem<T>>
) : DataNode<T> {
//TODO add node-level meta?
override fun toString(): String {
return super.toString()
}
}
private sealed class DataTreeBuilderItem<out T : Any> {
@ -119,10 +140,11 @@ private sealed class DataTreeBuilderItem<out T : Any> {
/**
* A builder for a DataTree.
*/
class DataTreeBuilder<T : Any>(private val type: KClass<out T>) {
@DFBuilder
class DataTreeBuilder<T : Any>(val type: KClass<out T>) {
private val map = HashMap<NameToken, DataTreeBuilderItem<T>>()
operator fun set(token: NameToken, node: DataTreeBuilder<T>) {
operator fun set(token: NameToken, node: DataTreeBuilder<out T>) {
if (map.containsKey(token)) error("Tree entry with name $token is not empty")
map[token] = DataTreeBuilderItem.Node(node)
}
@ -134,9 +156,9 @@ class DataTreeBuilder<T : Any>(private val type: KClass<out T>) {
private fun buildNode(token: NameToken): DataTreeBuilder<T> {
return if (!map.containsKey(token)) {
DataTreeBuilder<T>(type).also { map[token] = DataTreeBuilderItem.Node(it) }
DataTreeBuilder(type).also { map[token] = DataTreeBuilderItem.Node(it) }
} else {
(map[token] as? DataTreeBuilderItem.Node ?: error("The node with name $token is occupied by leaf")).tree
(map[token] as? DataTreeBuilderItem.Node<T> ?: error("The node with name $token is occupied by leaf")).tree
}
}
@ -156,7 +178,7 @@ class DataTreeBuilder<T : Any>(private val type: KClass<out T>) {
}
}
operator fun set(name: Name, node: DataTreeBuilder<T>) {
operator fun set(name: Name, node: DataTreeBuilder<out T>) {
when (name.length) {
0 -> error("Can't add data with empty name")
1 -> set(name.first()!!, node)
@ -174,19 +196,19 @@ class DataTreeBuilder<T : Any>(private val type: KClass<out T>) {
/**
* Append data to node
*/
infix fun String.to(data: Data<T>) = set(toName(), data)
infix fun String.put(data: Data<T>) = set(toName(), data)
/**
* Append node
*/
infix fun String.to(node: DataNode<T>) = set(toName(), node)
infix fun String.put(node: DataNode<T>) = set(toName(), node)
infix fun String.to(item: DataItem<T>) = set(toName(), item)
infix fun String.put(item: DataItem<T>) = set(toName(), item)
/**
* Build and append node
*/
infix fun String.to(block: DataTreeBuilder<T>.() -> Unit) = set(toName(), DataTreeBuilder<T>(type).apply(block))
infix fun String.put(block: DataTreeBuilder<T>.() -> Unit) = set(toName(), DataTreeBuilder(type).apply(block))
fun update(node: DataNode<T>) {
@ -207,6 +229,42 @@ class DataTreeBuilder<T : Any>(private val type: KClass<out T>) {
}
}
fun <T : Any> DataTreeBuilder<T>.datum(name: Name, data: Data<T>) {
this[name] = data
}
fun <T : Any> DataTreeBuilder<T>.datum(name: String, data: Data<T>) {
this[name.toName()] = data
}
fun <T : Any> DataTreeBuilder<T>.static(name: Name, data: T, meta: Meta = EmptyMeta) {
this[name] = Data.static(data, meta)
}
fun <T : Any> DataTreeBuilder<T>.static(name: Name, data: T, block: MetaBuilder.() -> Unit = {}) {
this[name] = Data.static(data, buildMeta(block))
}
fun <T : Any> DataTreeBuilder<T>.static(name: String, data: T, block: MetaBuilder.() -> Unit = {}) {
this[name.toName()] = Data.static(data, buildMeta(block))
}
fun <T : Any> DataTreeBuilder<T>.node(name: Name, node: DataNode<T>) {
this[name] = node
}
fun <T : Any> DataTreeBuilder<T>.node(name: String, node: DataNode<T>) {
this[name.toName()] = node
}
inline fun <reified T : Any> DataTreeBuilder<T>.node(name: Name, noinline block: DataTreeBuilder<T>.() -> Unit) {
this[name] = DataNode(T::class, block)
}
inline fun <reified T : Any> DataTreeBuilder<T>.node(name: String, noinline block: DataTreeBuilder<T>.() -> Unit) {
this[name.toName()] = DataNode(T::class, block)
}
/**
* Generate a mutable builder from this node. Node content is not changed
*/
@ -214,7 +272,7 @@ fun <T : Any> DataNode<T>.builder(): DataTreeBuilder<T> = DataTreeBuilder(type).
dataSequence().forEach { (name, data) -> this[name] = data }
}
fun <T : Any> DataNode<T>.filter(predicate: (Name, Data<T>) -> Boolean): DataNode<T> = DataNode.build(type) {
fun <T : Any> DataNode<T>.filter(predicate: (Name, Data<T>) -> Boolean): DataNode<T> = DataNode.invoke(type) {
dataSequence().forEach { (name, data) ->
if (predicate(name, data)) {
this[name] = data
@ -222,9 +280,4 @@ fun <T : Any> DataNode<T>.filter(predicate: (Name, Data<T>) -> Boolean): DataNod
}
}
fun <T : Any> DataNode<T>.first(): Data<T>? = dataSequence().first().second
/**
* Check that node is compatible with given type meaning that each element could be cast to the type
*/
expect fun DataNode<*>.checkType(type: KClass<*>)
fun <T : Any> DataNode<T>.first(): Data<T>? = dataSequence().first().second

8
dataforge-data/src/commonMain/kotlin/hep/dataforge/data/Goal.kt
View File

@ -85,7 +85,7 @@ open class DynamicGoal<T>(
/**
* Create a one-to-one goal based on existing goal
*/
fun <T, R> Goal<T>.pipe(
fun <T, R> Goal<T>.map(
coroutineContext: CoroutineContext = EmptyCoroutineContext,
block: suspend CoroutineScope.(T) -> R
): Goal<R> = DynamicGoal(coroutineContext, listOf(this)) {
@ -95,11 +95,11 @@ fun <T, R> Goal<T>.pipe(
/**
* Create a joining goal.
*/
fun <T, R> Collection<Goal<T>>.join(
fun <T, R> Collection<Goal<T>>.reduce(
coroutineContext: CoroutineContext = EmptyCoroutineContext,
block: suspend CoroutineScope.(Collection<T>) -> R
): Goal<R> = DynamicGoal(coroutineContext, this) {
block(map { this.run { it.await(this) } })
block(map { run { it.await(this) } })
}
/**
@ -108,7 +108,7 @@ fun <T, R> Collection<Goal<T>>.join(
* @param T type of the input goal
* @param R type of the result goal
*/
fun <K, T, R> Map<K, Goal<T>>.join(
fun <K, T, R> Map<K, Goal<T>>.reduce(
coroutineContext: CoroutineContext = EmptyCoroutineContext,
block: suspend CoroutineScope.(Map<K, T>) -> R
): Goal<R> = DynamicGoal(coroutineContext, this.values) {

78
dataforge-data/src/commonMain/kotlin/hep/dataforge/data/MapAction.kt Normal file
View File

@ -0,0 +1,78 @@
package hep.dataforge.data
import hep.dataforge.meta.Meta
import hep.dataforge.meta.MetaBuilder
import hep.dataforge.meta.builder
import hep.dataforge.meta.seal
import hep.dataforge.names.Name
import kotlin.reflect.KClass
/**
* Action environment includes data name, data meta and action configuration meta
*/
data class ActionEnv(
val name: Name,
val meta: Meta,
val actionMeta: Meta
)
/**
* Action environment
*/
class MapActionBuilder<T, R>(var name: Name, var meta: MetaBuilder, val actionMeta: Meta) {
lateinit var result: suspend ActionEnv.(T) -> R
/**
* Calculate the result of goal
*/
fun result(f: suspend ActionEnv.(T) -> R) {
result = f;
}
}
class MapAction<T : Any, out R : Any>(
val inputType: KClass<out T>,
val outputType: KClass<out R>,
private val block: MapActionBuilder<T, R>.() -> Unit
) : Action<T, R> {
override fun invoke(node: DataNode<T>, meta: Meta): DataNode<R> {
node.ensureType(inputType)
return DataNode.invoke(outputType) {
node.dataSequence().forEach { (name, data) ->
/*
* Creating a new environment for action using **old** name, old meta and task meta
*/
val env = ActionEnv(name, data.meta, meta)
//applying transformation from builder
val builder = MapActionBuilder<T, R>(
name,
data.meta.builder(), // using data meta
meta
).apply(block)
//getting new name
val newName = builder.name
//getting new meta
val newMeta = builder.meta.seal()
val newData = data.map(outputType, meta = newMeta) { builder.result(env, it) }
//setting the data node
this[newName] = newData
}
}
}
}
inline fun <reified T : Any, reified R : Any> DataNode<T>.map(
meta: Meta,
noinline action: MapActionBuilder<in T, out R>.() -> Unit
): DataNode<R> = MapAction(T::class, R::class, action).invoke(this, meta)

60
dataforge-data/src/commonMain/kotlin/hep/dataforge/data/PipeAction.kt
View File

@ -1,60 +0,0 @@
package hep.dataforge.data
import hep.dataforge.meta.*
import hep.dataforge.names.Name
import kotlin.reflect.KClass
class ActionEnv(val name: Name, val meta: Meta)
/**
* Action environment
*/
class PipeBuilder<T, R>(var name: Name, var meta: MetaBuilder) {
lateinit var result: suspend ActionEnv.(T) -> R
/**
* Calculate the result of goal
*/
fun result(f: suspend ActionEnv.(T) -> R) {
result = f;
}
}
class PipeAction<T : Any, R : Any>(
val inputType: KClass<T>,
val outputType: KClass<R>,
private val block: PipeBuilder<T, R>.() -> Unit
) : Action<T, R> {
override fun invoke(node: DataNode<T>, meta: Meta): DataNode<R> {
node.checkType(inputType)
return DataNode.build(outputType) {
node.dataSequence().forEach { (name, data) ->
//merging data meta with action meta (data meta is primary)
val oldMeta = meta.builder().apply { update(data.meta) }
// creating environment from old meta and name
val env = ActionEnv(name, oldMeta)
//applying transformation from builder
val builder = PipeBuilder<T, R>(name, oldMeta).apply(block)
//getting new name
val newName = builder.name
//getting new meta
val newMeta = builder.meta.seal()
val newData = data.pipe(outputType, meta = newMeta) { builder.result(env, it) }
//setting the data node
this[newName] = newData
}
}
}
}
inline fun <reified T : Any, reified R : Any> DataNode<T>.pipe(
meta: Meta,
noinline action: PipeBuilder<T, R>.() -> Unit
): DataNode<R> = PipeAction(T::class, R::class, action).invoke(this, meta)

31
dataforge-data/src/commonMain/kotlin/hep/dataforge/data/JoinAction.kt → dataforge-data/src/commonMain/kotlin/hep/dataforge/data/ReduceAction.kt
View File

@ -1,9 +1,7 @@
package hep.dataforge.data
import hep.dataforge.meta.Laminate
import hep.dataforge.meta.Meta
import hep.dataforge.meta.MetaBuilder
import hep.dataforge.meta.builder
import hep.dataforge.names.Name
import hep.dataforge.names.toName
import kotlin.reflect.KClass
@ -21,7 +19,7 @@ class JoinGroup<T : Any, R : Any>(var name: String, internal val node: DataNode<
}
class JoinGroupBuilder<T : Any, R : Any>(val actionMeta: Meta) {
class ReduceGroupBuilder<T : Any, R : Any>(val actionMeta: Meta) {
private val groupRules: MutableList<(DataNode<T>) -> List<JoinGroup<T, R>>> = ArrayList();
/**
@ -73,26 +71,31 @@ class JoinGroupBuilder<T : Any, R : Any>(val actionMeta: Meta) {
/**
* The same rules as for KPipe
*/
class JoinAction<T : Any, R : Any>(
val inputType: KClass<T>,
val outputType: KClass<R>,
private val action: JoinGroupBuilder<T, R>.() -> Unit
class ReduceAction<T : Any, R : Any>(
val inputType: KClass<out T>,
val outputType: KClass<out R>,
private val action: ReduceGroupBuilder<T, R>.() -> Unit
) : Action<T, R> {
override fun invoke(node: DataNode<T>, meta: Meta): DataNode<R> {
node.checkType(inputType)
return DataNode.build(outputType) {
JoinGroupBuilder<T, R>(meta).apply(action).buildGroups(node).forEach { group ->
node.ensureType(inputType)
return DataNode.invoke(outputType) {
ReduceGroupBuilder<T, R>(meta).apply(action).buildGroups(node).forEach { group ->
val laminate = Laminate(group.meta, meta)
//val laminate = Laminate(group.meta, meta)
val dataMap = group.node.dataSequence().associate { it }
val groupName: String = group.name;
val groupName: String = group.name
val env = ActionEnv(groupName.toName(), laminate.builder())
val groupMeta = group.meta
val res: DynamicData<R> = dataMap.join(outputType, meta = laminate) { group.result.invoke(env, it) }
val env = ActionEnv(groupName.toName(), groupMeta, meta)
val res: DynamicData<R> = dataMap.reduce(
outputType,
meta = groupMeta
) { group.result.invoke(env, it) }
set(env.name, res)
}

10
dataforge-data/src/commonMain/kotlin/hep/dataforge/data/SplitAction.kt
View File

@ -33,15 +33,15 @@ class SplitBuilder<T : Any, R : Any>(val name: Name, val meta: Meta) {
}
class SplitAction<T : Any, R : Any>(
val inputType: KClass<T>,
val outputType: KClass<R>,
val inputType: KClass<out T>,
val outputType: KClass<out R>,
private val action: SplitBuilder<T, R>.() -> Unit
) : Action<T, R> {
override fun invoke(node: DataNode<T>, meta: Meta): DataNode<R> {
node.checkType(inputType)
node.ensureType(inputType)
return DataNode.build(outputType) {
return DataNode.invoke(outputType) {
node.dataSequence().forEach { (name, data) ->
val laminate = Laminate(data.meta, meta)
@ -55,7 +55,7 @@ class SplitAction<T : Any, R : Any>(
rule(env)
val res = data.pipe(outputType, meta = env.meta) { env.result(it) }
val res = data.map(outputType, meta = env.meta) { env.result(it) }
set(env.name, res)
}
}

72
dataforge-data/src/commonMain/kotlin/hep/dataforge/data/dataCast.kt Normal file
View File

@ -0,0 +1,72 @@
package hep.dataforge.data
import hep.dataforge.meta.Meta
import hep.dataforge.names.NameToken
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Deferred
import kotlin.reflect.KClass
fun <R : Any, T : R> Data<T>.upcast(type: KClass<out R>): Data<R> {
return object : Data<R> by this {
override val type: KClass<out R> = type
}
}
/**
* Safe upcast a [Data] to a supertype
*/
inline fun <reified R : Any, T : R> Data<T>.upcast(): Data<R> = upcast(R::class)
/**
* Check if node could be safely cast to given class
*/
expect fun <R : Any> DataNode<*>.canCast(type: KClass<out R>): Boolean
/**
* Check if data could be safely cast to given class
*/
expect fun <R : Any> Data<*>.canCast(type: KClass<out R>): Boolean
fun <R : Any> DataItem<*>.canCast(type: KClass<out R>): Boolean = when (this) {
is DataItem.Node -> value.canCast(type)
is DataItem.Leaf -> value.canCast(type)
}
/**
* Unsafe cast of data node
*/
@Suppress("UNCHECKED_CAST")
fun <R : Any> Data<*>.cast(type: KClass<out R>): Data<R> {
return object : Data<R> {
override val meta: Meta get() = this@cast.meta
override val dependencies: Collection<Goal<*>> get() = this@cast.dependencies
override val result: Deferred<R>? get() = this@cast.result as Deferred<R>
override fun startAsync(scope: CoroutineScope): Deferred<R> = this@cast.startAsync(scope) as Deferred<R>
override fun reset() = this@cast.reset()
override val type: KClass<out R> = type
}
}
inline fun <reified R : Any> Data<*>.cast(): Data<R> = cast(R::class)
@Suppress("UNCHECKED_CAST")
fun <R : Any> DataNode<*>.cast(type: KClass<out R>): DataNode<R> {
return object : DataNode<R> {
override val type: KClass<out R> = type
override val items: Map<NameToken, DataItem<R>> get() = this@cast.items as Map<NameToken, DataItem<R>>
}
}
inline fun <reified R : Any> DataNode<*>.cast(): DataNode<R> = cast(R::class)
/**
* Check that node is compatible with given type meaning that each element could be cast to the type
*/
fun <T : Any> DataNode<*>.ensureType(type: KClass<out T>) {
if (!canCast(type)) {
error("$type expected, but $type received")
}
}
//expect fun <T : Any, R : Any> DataNode<T>.cast(type: KClass<out R>): DataNode<R>

32
dataforge-data/src/commonTest/kotlin/hep/dataforge/data/DataTreeBuilderTest.kt Normal file
View File

@ -0,0 +1,32 @@
package hep.dataforge.data
import kotlin.test.Test
import kotlin.test.assertTrue
internal class DataTreeBuilderTest{
@Test
fun testDataUpdate(){
val updateData = DataNode<Any>{
"update" put {
"a" put Data.static("a")
"b" put Data.static("b")
}
}
val node = DataNode<Any>{
node("primary"){
static("a","a")
static("b","b")
}
static("root","root")
update(updateData)
}
println(node.toMeta())
assertTrue { node["update.a"] != null }
assertTrue { node["primary.a"] != null }
}
}

10
dataforge-data/src/jsMain/kotlin/hep/dataforge/data/checkType.kt
View File

@ -1,10 +0,0 @@
package hep.dataforge.data
import kotlin.reflect.KClass
/**
* Check that node is compatible with given type meaning that each element could be cast to the type
*/
actual fun DataNode<*>.checkType(type: KClass<*>) {
//Not supported in js yet
}

16
dataforge-data/src/jsMain/kotlin/hep/dataforge/data/dataJS.kt Normal file
View File

@ -0,0 +1,16 @@
package hep.dataforge.data
import kotlin.reflect.KClass
/**
* Check that node is compatible with given type meaning that each element could be cast to the type
*/
actual fun <R : Any> DataNode<*>.canCast(type: KClass<out R>): Boolean {
//Not supported in js yet
return true
}
actual fun <R : Any> Data<*>.canCast(type: KClass<out R>): Boolean {
//Not supported in js yet
return true
}

55
dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/CastDataNode.kt
View File

@ -1,55 +0,0 @@
package hep.dataforge.data
import hep.dataforge.meta.Meta
import hep.dataforge.names.NameToken
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Deferred
import kotlin.reflect.KClass
import kotlin.reflect.full.isSubclassOf
@Suppress("UNCHECKED_CAST")
fun <T : Any, R : Any> Data<T>.safeCast(type: KClass<out R>): Data<R>? {
return if (this.type.isSubclassOf(type)) {
return object : Data<R> {
override val meta: Meta get() = this@safeCast.meta
override val dependencies: Collection<Goal<*>> get() = this@safeCast.dependencies
override val result: Deferred<R>? get() = this@safeCast.result as Deferred<R>
override fun startAsync(scope: CoroutineScope): Deferred<R> = this@safeCast.startAsync(scope) as Deferred<R>
override fun reset() = this@safeCast.reset()
override val type: KClass<out R> = type
}
} else {
null
}
}
/**
* Filter a node by data and node type. Resulting node and its subnodes is guaranteed to have border type [type],
* but could contain empty nodes
*/
fun <T : Any, R : Any> DataNode<T>.cast(type: KClass<out R>): DataNode<R> {
return if (this is CastDataNode) {
origin.cast(type)
} else {
CastDataNode(this, type)
}
}
inline fun <T : Any, reified R : Any> DataNode<T>.cast(): DataNode<R> = cast(R::class)
class CastDataNode<out T : Any>(val origin: DataNode<Any>, override val type: KClass<out T>) : DataNode<T> {
override val items: Map<NameToken, DataItem<T>> by lazy {
origin.items.mapNotNull { (key, item) ->
when (item) {
is DataItem.Leaf -> {
(item.value.safeCast(type))?.let {
key to DataItem.Leaf(it)
}
}
is DataItem.Node -> {
key to DataItem.Node(item.value.cast(type))
}
}
}.associate { it }
}
}

25
dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/TypeFilteredDataNode.kt Normal file
View File

@ -0,0 +1,25 @@
package hep.dataforge.data
import hep.dataforge.names.NameToken
import kotlin.reflect.KClass
/**
* A zero-copy data node wrapper that returns only children with appropriate type.
*/
class TypeFilteredDataNode<out T : Any>(val origin: DataNode<*>, override val type: KClass<out T>) : DataNode<T> {
override val items: Map<NameToken, DataItem<T>> by lazy {
origin.items.mapNotNull { (key, item) ->
when (item) {
is DataItem.Leaf -> {
(item.value.filterIsInstance(type))?.let {
key to DataItem.Leaf(it)
}
}
is DataItem.Node -> {
key to DataItem.Node(item.value.filterIsInstance(type))
}
}
}.associate { it }
}
}

40
dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/dataJVM.kt
View File

@ -2,6 +2,7 @@ package hep.dataforge.data
import kotlinx.coroutines.runBlocking
import kotlin.reflect.KClass
import kotlin.reflect.full.isSubclassOf
import kotlin.reflect.full.isSuperclassOf
/**
@ -12,8 +13,39 @@ fun <T : Any> Data<T>.get(): T = runBlocking { await() }
/**
* Check that node is compatible with given type meaning that each element could be cast to the type
*/
actual fun DataNode<*>.checkType(type: KClass<*>) {
if (!type.isSuperclassOf(type)) {
error("$type expected, but $type received")
actual fun <R : Any> DataNode<*>.canCast(type: KClass<out R>): Boolean =
type.isSuperclassOf(type)
actual fun <R : Any> Data<*>.canCast(type: KClass<out R>): Boolean =
this.type.isSubclassOf(type)
/**
* Cast the node to given type if the cast is possible or return null
*/
fun <R : Any> Data<*>.filterIsInstance(type: KClass<out R>): Data<R>? =
if (canCast(type)) cast(type) else null
/**
* Filter a node by data and node type. Resulting node and its subnodes is guaranteed to have border type [type],
* but could contain empty nodes
*/
fun <R : Any> DataNode<*>.filterIsInstance(type: KClass<out R>): DataNode<R> {
return if (canCast(type)) {
cast(type)
} else if (this is TypeFilteredDataNode) {
origin.filterIsInstance(type)
} else {
TypeFilteredDataNode(this, type)
}
}
}
/**
* Filter all elements of given data item that could be cast to given type. If no elements are available, return null.
*/
fun <R : Any> DataItem<*>?.filterIsInstance(type: KClass<out R>): DataItem<R>? = when (this) {
null -> null
is DataItem.Node -> DataItem.Node(this.value.filterIsInstance(type))
is DataItem.Leaf -> this.value.filterIsInstance(type)?.let { DataItem.Leaf(it) }
}
inline fun <reified R : Any> DataItem<*>?.filterIsInstance(): DataItem<R>? = this@filterIsInstance.filterIsInstance(R::class)

9
dataforge-io/build.gradle.kts
View File

@ -5,8 +5,8 @@ plugins {
description = "IO module"
scientifik{
serialization = true
io = true
withSerialization()
withIO()
}
@ -17,6 +17,11 @@ kotlin {
api(project(":dataforge-context"))
}
}
jvmMain{
dependencies {
}
}
jsMain{
dependencies{
api(npm("text-encoding"))

12
dataforge-io/dataforge-io-yaml/build.gradle.kts Normal file
View File

@ -0,0 +1,12 @@
plugins {
id("scientifik.jvm")
}
description = "YAML meta IO"
dependencies {
api(project(":dataforge-io"))
api("org.yaml:snakeyaml:1.25")
testImplementation(kotlin("test"))
testImplementation(kotlin("test-junit"))
}

88
dataforge-io/dataforge-io-yaml/src/main/kotlin/hep/dataforge/io/yaml/FrontMatterEnvelopeFormat.kt Normal file
View File

@ -0,0 +1,88 @@
package hep.dataforge.io.yaml
import hep.dataforge.context.Context
import hep.dataforge.io.*
import hep.dataforge.meta.DFExperimental
import hep.dataforge.meta.EmptyMeta
import hep.dataforge.meta.Meta
import kotlinx.io.core.*
import kotlinx.serialization.toUtf8Bytes
@DFExperimental
class FrontMatterEnvelopeFormat(
val io: IOPlugin,
meta: Meta = EmptyMeta
) : EnvelopeFormat {
override fun Input.readPartial(): PartialEnvelope {
var line: String = ""
var offset = 0u
do {
line = readUTF8Line() ?: error("Input does not contain front matter separator")
offset += line.toUtf8Bytes().size.toUInt()
} while (!line.startsWith(SEPARATOR))
val readMetaFormat =
metaTypeRegex.matchEntire(line)?.groupValues?.first()
?.let { io.metaFormat(it) } ?: YamlMetaFormat.default
val metaBlock = buildPacket {
do {
line = readUTF8Line() ?: error("Input does not contain closing front matter separator")
appendln(line)
offset += line.toUtf8Bytes().size.toUInt()
} while (!line.startsWith(SEPARATOR))
}
val meta = readMetaFormat.fromBytes(metaBlock)
return PartialEnvelope(meta, offset, null)
}
override fun Input.readObject(): Envelope {
var line: String = ""
do {
line = readUTF8Line() ?: error("Input does not contain front matter separator")
} while (!line.startsWith(SEPARATOR))
val readMetaFormat =
metaTypeRegex.matchEntire(line)?.groupValues?.first()
?.let { io.metaFormat(it) } ?: YamlMetaFormat.default
val metaBlock = buildPacket {
do {
appendln(readUTF8Line() ?: error("Input does not contain closing front matter separator"))
} while (!line.startsWith(SEPARATOR))
}
val meta = readMetaFormat.fromBytes(metaBlock)
val bytes = readBytes()
val data = bytes.asBinary()
return SimpleEnvelope(meta, data)
}
override fun Output.writeEnvelope(envelope: Envelope, metaFormatFactory: MetaFormatFactory, formatMeta: Meta) {
val metaFormat = metaFormatFactory(formatMeta, io.context)
writeText("$SEPARATOR\r\n")
metaFormat.run { writeObject(envelope.meta) }
writeText("$SEPARATOR\r\n")
envelope.data?.read { copyTo(this@writeEnvelope) }
}
companion object : EnvelopeFormatFactory {
const val SEPARATOR = "---"
private val metaTypeRegex = "---(\\w*)\\s*".toRegex()
override fun invoke(meta: Meta, context: Context): EnvelopeFormat {
return FrontMatterEnvelopeFormat(context.io, meta)
}
override fun peekFormat(io: IOPlugin, input: Input): EnvelopeFormat? {
val line = input.readUTF8Line(3, 30)
return if (line != null && line.startsWith("---")) {
invoke()
} else {
null
}
}
}
}

56
dataforge-io/dataforge-io-yaml/src/main/kotlin/hep/dataforge/io/yaml/YamlMetaFormat.kt Normal file
View File

@ -0,0 +1,56 @@
package hep.dataforge.io.yaml
import hep.dataforge.context.Context
import hep.dataforge.descriptors.NodeDescriptor
import hep.dataforge.io.MetaFormat
import hep.dataforge.io.MetaFormatFactory
import hep.dataforge.meta.DFExperimental
import hep.dataforge.meta.Meta
import hep.dataforge.meta.toMap
import hep.dataforge.meta.toMeta
import hep.dataforge.names.Name
import hep.dataforge.names.plus
import kotlinx.io.core.Input
import kotlinx.io.core.Output
import kotlinx.io.core.readUByte
import kotlinx.io.core.writeText
import org.yaml.snakeyaml.Yaml
import java.io.InputStream
private class InputAsStream(val input: Input) : InputStream() {
override fun read(): Int {
if (input.endOfInput) return -1
return input.readUByte().toInt()
}
override fun close() {
input.close()
}
}
private fun Input.asStream() = InputAsStream(this)
@DFExperimental
class YamlMetaFormat(val meta: Meta) : MetaFormat {
private val yaml = Yaml()
override fun Output.writeMeta(meta: Meta, descriptor: NodeDescriptor?) {
val string = yaml.dump(meta.toMap(descriptor))
writeText(string)
}
override fun Input.readMeta(descriptor: NodeDescriptor?): Meta {
val map: Map<String, Any?> = yaml.load(asStream())
return map.toMeta(descriptor)
}
companion object : MetaFormatFactory {
val default = YamlMetaFormat()
override fun invoke(meta: Meta, context: Context): MetaFormat = YamlMetaFormat(meta)
override val name: Name = super.name + "yaml"
override val key: Short = 0x594d //YM
}
}

43
dataforge-io/dataforge-io-yaml/src/test/kotlin/hep/dataforge/io/yaml/YamlMetaFormatTest.kt Normal file
View File

@ -0,0 +1,43 @@
package hep.dataforge.io.yaml
import hep.dataforge.io.parse
import hep.dataforge.io.toString
import hep.dataforge.meta.Meta
import hep.dataforge.meta.buildMeta
import hep.dataforge.meta.get
import hep.dataforge.meta.seal
import org.junit.Test
import kotlin.test.assertEquals
class YamlMetaFormatTest{
@Test
fun testYamlMetaFormat(){
val meta = buildMeta {
"a" put 22
"node" put {
"b" put "DDD"
"c" put 11.1
"d" put {
"d1" put {
"d11" put "aaa"
"d12" put "bbb"
}
"d2" put 2
}
"array" put doubleArrayOf(1.0, 2.0, 3.0)
}
}
val string = meta.toString(YamlMetaFormat)
println(string)
val result = YamlMetaFormat.parse(string)
assertEquals<Meta>(meta, meta.seal())
meta.items.keys.forEach {
if (meta[it] != result[it]) error("${meta[it]} != ${result[it]}")
}
assertEquals(meta, result)
}
}

36
dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Binary.kt
View File

@ -1,9 +1,7 @@
package hep.dataforge.io
import kotlinx.io.core.ByteReadPacket
import kotlinx.io.core.Input
import kotlinx.io.core.buildPacket
import kotlinx.io.core.readBytes
import kotlinx.io.core.*
import kotlin.math.min
/**
* A source of binary data
@ -30,19 +28,21 @@ interface RandomAccessBinary : Binary {
/**
* Read at most [size] of bytes starting at [from] offset from the beginning of the binary.
* This method could be called multiple times simultaneously.
*
* If size
*/
fun <R> read(from: UInt, size: UInt = UInt.MAX_VALUE, block: Input.() -> R): R
override fun <R> read(block: Input.() -> R): R = read(0.toUInt(), UInt.MAX_VALUE, block)
}
fun Binary.readAll(): ByteReadPacket = read {
ByteReadPacket(this.readBytes())
fun Binary.toBytes(): ByteArray = read {
this.readBytes()
}
@ExperimentalUnsignedTypes
fun RandomAccessBinary.readPacket(from: UInt, size: UInt): ByteReadPacket = read(from, size) {
ByteReadPacket(this.readBytes())
buildPacket { copyTo(this) }
}
@ExperimentalUnsignedTypes
@ -53,33 +53,35 @@ object EmptyBinary : RandomAccessBinary {
override fun <R> read(from: UInt, size: UInt, block: Input.() -> R): R {
error("The binary is empty")
}
}
@ExperimentalUnsignedTypes
class ArrayBinary(val array: ByteArray) : RandomAccessBinary {
inline class ArrayBinary(val array: ByteArray) : RandomAccessBinary {
override val size: ULong get() = array.size.toULong()
override fun <R> read(from: UInt, size: UInt, block: Input.() -> R): R {
return ByteReadPacket(array, from.toInt(), size.toInt()).block()
val theSize = min(size, array.size.toUInt() - from)
return buildPacket {
writeFully(array, from.toInt(), theSize.toInt())
}.block()
}
}
fun ByteArray.asBinary() = ArrayBinary(this)
/**
* Read given binary as object using given format
*/
fun <T : Any> Binary.readWith(format: IOFormat<T>): T = format.run {
read {
readThis()
readObject()
}
}
/**
* Write this object to a binary
* TODO make a lazy binary that does not use intermediate array
*/
fun <T: Any> T.writeWith(format: IOFormat<T>): Binary = format.run{
fun <T : Any> IOFormat<T>.writeBinary(obj: T): Binary {
val packet = buildPacket {
writeThis(this@writeWith)
writeObject(obj)
}
return@run ArrayBinary(packet.readBytes())
return ArrayBinary(packet.readBytes())
}

15
dataforge-io/src/commonMain/kotlin/hep/dataforge/io/BinaryMetaFormat.kt
View File

@ -1,17 +1,22 @@
package hep.dataforge.io
import hep.dataforge.context.Context
import hep.dataforge.descriptors.NodeDescriptor
import hep.dataforge.meta.*
import hep.dataforge.names.Name
import hep.dataforge.names.plus
import hep.dataforge.values.*
import kotlinx.io.core.Input
import kotlinx.io.core.Output
import kotlinx.io.core.readText
import kotlinx.io.core.writeText
object BinaryMetaFormat : MetaFormat {
override val name: String = "bin"
object BinaryMetaFormat : MetaFormat, MetaFormatFactory {
override val name: Name = super.name + "bin"
override val key: Short = 0x4249//BI
override fun invoke(meta: Meta, context: Context): MetaFormat = this
override fun Input.readMeta(descriptor: NodeDescriptor?): Meta {
return (readMetaItem() as MetaItem.NodeItem).node
}
@ -23,7 +28,7 @@ object BinaryMetaFormat : MetaFormat {
writeText(str)
}
private fun Output.writeValue(value: Value) {
fun Output.writeValue(value: Value) {
if (value.isList()) {
writeChar('L')
writeInt(value.list.size)
@ -80,7 +85,7 @@ object BinaryMetaFormat : MetaFormat {
writeValue(item.value)
}
is MetaItem.NodeItem -> {
writeThis(item.node)
writeObject(item.node)
}
}
}
@ -92,7 +97,7 @@ object BinaryMetaFormat : MetaFormat {
}
@Suppress("UNCHECKED_CAST")
private fun Input.readMetaItem(): MetaItem<MetaBuilder> {
fun Input.readMetaItem(): MetaItem<MetaBuilder> {
return when (val keyChar = readByte().toChar()) {
'S' -> MetaItem.ValueItem(StringValue(readString()))
'N' -> MetaItem.ValueItem(Null)

72
dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Envelope.kt
View File

@ -1,9 +1,11 @@
package hep.dataforge.io
import hep.dataforge.meta.Laminate
import hep.dataforge.meta.Meta
import hep.dataforge.meta.get
import hep.dataforge.meta.string
import hep.dataforge.meta.*
import hep.dataforge.names.asName
import hep.dataforge.names.plus
import kotlinx.io.core.Output
import kotlinx.io.core.buildPacket
import kotlinx.io.core.readBytes
interface Envelope {
val meta: Meta
@ -14,12 +16,17 @@ interface Envelope {
/**
* meta keys
*/
const val ENVELOPE_NODE = "@envelope"
const val ENVELOPE_TYPE_KEY = "$ENVELOPE_NODE.type"
const val ENVELOPE_DATA_TYPE_KEY = "$ENVELOPE_NODE.dataType"
const val ENVELOPE_DESCRIPTION_KEY = "$ENVELOPE_NODE.description"
val ENVELOPE_NODE_KEY = "@envelope".asName()
val ENVELOPE_TYPE_KEY = ENVELOPE_NODE_KEY + "type"
val ENVELOPE_DATA_TYPE_KEY = ENVELOPE_NODE_KEY + "dataType"
val ENVELOPE_DATA_ID_KEY = ENVELOPE_NODE_KEY + "dataID"
val ENVELOPE_DESCRIPTION_KEY = ENVELOPE_NODE_KEY + "description"
//const val ENVELOPE_TIME_KEY = "@envelope.time"
/**
* Build a static envelope using provided builder
*/
operator fun invoke(block: EnvelopeBuilder.() -> Unit) = EnvelopeBuilder().apply(block).build()
}
}
@ -28,24 +35,35 @@ class SimpleEnvelope(override val meta: Meta, override val data: Binary?) : Enve
/**
* The purpose of the envelope
*
* @return
*/
val Envelope.type: String? get() = meta[Envelope.ENVELOPE_TYPE_KEY].string
/**
* The type of data encoding
*
* @return
*/
val Envelope.dataType: String? get() = meta[Envelope.ENVELOPE_DATA_TYPE_KEY].string
/**
* Textual user friendly description
*
* @return
*/
val Envelope.description: String? get() = meta[Envelope.ENVELOPE_DESCRIPTION_KEY].string
/**
* An optional unique identifier that is used for data comparison. Data without identifier could not be compared to another data.
*/
val Envelope.dataID: String? get() = meta[Envelope.ENVELOPE_DATA_ID_KEY].string
fun Envelope.metaEquals(other: Envelope): Boolean = this.meta == other.meta
fun Envelope.dataEquals(other: Envelope): Boolean = this.dataID != null && this.dataID == other.dataID
fun Envelope.contentEquals(other: Envelope): Boolean {
return (this === other || (metaEquals(other) && dataEquals(other)))
}
/**
* An envelope, which wraps existing envelope and adds one or several additional layers of meta
*/
@ -55,7 +73,7 @@ class ProxyEnvelope(val source: Envelope, vararg meta: Meta) : Envelope {
}
/**
* Add few meta layers to existing envelope
* Add few meta layers to existing envelope (on top of existing meta)
*/
fun Envelope.withMetaLayers(vararg layers: Meta): Envelope {
return when {
@ -63,4 +81,34 @@ fun Envelope.withMetaLayers(vararg layers: Meta): Envelope {
this is ProxyEnvelope -> ProxyEnvelope(source, *layers, *this.meta.layers.toTypedArray())
else -> ProxyEnvelope(this, *layers)
}
}
class EnvelopeBuilder {
private val metaBuilder = MetaBuilder()
var data: Binary? = null
fun meta(block: MetaBuilder.() -> Unit) {
metaBuilder.apply(block)
}
fun meta(meta: Meta) {
metaBuilder.update(meta)
}
var type by metaBuilder.string(key = Envelope.ENVELOPE_TYPE_KEY)
var dataType by metaBuilder.string(key = Envelope.ENVELOPE_DATA_TYPE_KEY)
var dataID by metaBuilder.string(key = Envelope.ENVELOPE_DATA_ID_KEY)
var description by metaBuilder.string(key = Envelope.ENVELOPE_DESCRIPTION_KEY)
/**
* Construct a binary and transform it into byte-array based buffer
*/
fun data(block: Output.() -> Unit) {
val bytes = buildPacket {
block()
}
data = ArrayBinary(bytes.readBytes())
}
internal fun build() = SimpleEnvelope(metaBuilder.seal(), data)
}

40
dataforge-io/src/commonMain/kotlin/hep/dataforge/io/EnvelopeFormat.kt
View File

@ -1,11 +1,15 @@
package hep.dataforge.io
import hep.dataforge.context.Named
import hep.dataforge.io.EnvelopeFormat.Companion.ENVELOPE_FORMAT_TYPE
import hep.dataforge.context.Context
import hep.dataforge.io.EnvelopeFormatFactory.Companion.ENVELOPE_FORMAT_TYPE
import hep.dataforge.meta.EmptyMeta
import hep.dataforge.meta.Meta
import hep.dataforge.names.Name
import hep.dataforge.names.asName
import hep.dataforge.provider.Type
import kotlinx.io.core.Input
import kotlinx.io.core.Output
import kotlin.reflect.KClass
/**
* A partially read envelope with meta, but without data
@ -13,19 +17,33 @@ import kotlinx.io.core.Output
@ExperimentalUnsignedTypes
data class PartialEnvelope(val meta: Meta, val dataOffset: UInt, val dataSize: ULong?)
interface EnvelopeFormat : IOFormat<Envelope> {
val defaultMetaFormat: MetaFormatFactory get() = JsonMetaFormat
fun Input.readPartial(): PartialEnvelope
fun Output.writeEnvelope(envelope: Envelope, metaFormatFactory: MetaFormatFactory, formatMeta: Meta = EmptyMeta)
override fun Input.readObject(): Envelope
override fun Output.writeObject(obj: Envelope): Unit = writeEnvelope(obj, defaultMetaFormat)
}
@Type(ENVELOPE_FORMAT_TYPE)
interface EnvelopeFormat : IOFormat<Envelope>, Named {
fun Input.readPartial(formats: Collection<MetaFormat> = IOPlugin.defaultMetaFormats): PartialEnvelope
interface EnvelopeFormatFactory : IOFormatFactory<Envelope> {
override val name: Name get() = "envelope".asName()
override val type: KClass<out Envelope> get() = Envelope::class
fun Input.readEnvelope(formats: Collection<MetaFormat> = IOPlugin.defaultMetaFormats): Envelope
override fun invoke(meta: Meta, context: Context): EnvelopeFormat
override fun Input.readThis(): Envelope = readEnvelope()
fun Output.writeEnvelope(envelope: Envelope, format: MetaFormat = JsonMetaFormat)
override fun Output.writeThis(obj: Envelope) = writeEnvelope(obj)
/**
* Try to infer specific format from input and return null if the attempt is failed.
* This method does **not** return Input into initial state.
*/
fun peekFormat(io: IOPlugin, input: Input): EnvelopeFormat?
companion object {
const val ENVELOPE_FORMAT_TYPE = "envelopeFormat"
const val ENVELOPE_FORMAT_TYPE = "io.format.envelope"
}
}

38
dataforge-io/src/commonMain/kotlin/hep/dataforge/io/FunctionServer.kt
View File

@ -1,38 +0,0 @@
package hep.dataforge.io
import kotlin.reflect.KClass
/**
* A descriptor for specific type of functions
*/
interface FunctionSpec<T : Any, R : Any> {
val inputType: KClass<T>
val outputType: KClass<R>
}
/**
* A server that could produce asynchronous function values
*/
interface FunctionServer {
/**
* Call a function with given name and descriptor
*/
suspend fun <T : Any, R : Any, D : FunctionSpec<T, R>> call(name: String, descriptor: D, arg: T): R
/**
* Resolve a function descriptor for given types
*/
fun <T : Any, R : Any> resolveType(inputType: KClass<out T>, outputType: KClass<out R>): FunctionSpec<T, R>
/**
* Get a generic suspended function with given name and descriptor
*/
operator fun <T : Any, R : Any, D : FunctionSpec<T, R>> get(name: String, descriptor: D): (suspend (T) -> R) =
{ call(name, descriptor, it) }
}
suspend inline fun <reified T : Any, reified R : Any> FunctionServer.call(name: String, arg: T): R =
call(name, resolveType(T::class, R::class), arg)
inline operator fun <reified T : Any, reified R : Any> FunctionServer.get(name: String): (suspend (T) -> R) =
get(name, resolveType(T::class, R::class))

160
dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOFormat.kt
View File

@ -1,14 +1,164 @@
package hep.dataforge.io
import hep.dataforge.context.Context
import hep.dataforge.context.Factory
import hep.dataforge.context.Named
import hep.dataforge.io.IOFormatFactory.Companion.IO_FORMAT_TYPE
import hep.dataforge.meta.Meta
import hep.dataforge.meta.MetaItem
import hep.dataforge.names.Name
import hep.dataforge.names.asName
import hep.dataforge.provider.Type
import hep.dataforge.values.Value
import kotlinx.io.core.*
import kotlinx.io.pool.ObjectPool
import kotlinx.serialization.ImplicitReflectionSerializer
import kotlinx.serialization.KSerializer
import kotlinx.serialization.cbor.Cbor
import kotlinx.serialization.serializer
import kotlin.math.min
import kotlin.reflect.KClass
/**
* And interface for serialization facilities
* And interface for reading and writing objects into with IO streams
*/
interface IOFormat<T : Any> {
fun Output.writeThis(obj: T)
fun Input.readThis(): T
fun Output.writeObject(obj: T)
fun Input.readObject(): T
}
fun <T : Any> IOFormat<T>.writePacket(obj: T): ByteReadPacket = buildPacket { writeThis(obj) }
fun <T : Any> IOFormat<T>.writeBytes(obj: T): ByteArray = buildPacket { writeThis(obj) }.readBytes()
fun <T : Any> Input.readWith(format: IOFormat<T>): T = format.run { readObject() }
fun <T : Any> Output.writeWith(format: IOFormat<T>, obj: T) = format.run { writeObject(obj) }
class ListIOFormat<T : Any>(val format: IOFormat<T>) : IOFormat<List<T>> {
override fun Output.writeObject(obj: List<T>) {
writeInt(obj.size)
format.run {
obj.forEach {
writeObject(it)
}
}
}
override fun Input.readObject(): List<T> {
val size = readInt()
return format.run {
List(size) { readObject() }
}
}
}
val <T : Any> IOFormat<T>.list get() = ListIOFormat(this)
fun ObjectPool<IoBuffer>.fill(block: IoBuffer.() -> Unit): IoBuffer {
val buffer = borrow()
return try {
buffer.apply(block)
} catch (ex: Exception) {
//recycle(buffer)
throw ex
}
}
@Type(IO_FORMAT_TYPE)
interface IOFormatFactory<T : Any> : Factory<IOFormat<T>>, Named {
/**
* Explicit type for dynamic type checks
*/
val type: KClass<out T>
companion object {
const val IO_FORMAT_TYPE = "io.format"
}
}
@Deprecated("To be removed in io-2")
inline fun buildPacketWithoutPool(headerSizeHint: Int = 0, block: BytePacketBuilder.() -> Unit): ByteReadPacket {
val builder = BytePacketBuilder(headerSizeHint, IoBuffer.NoPool)
block(builder)
return builder.build()
}
fun <T : Any> IOFormat<T>.writePacket(obj: T): ByteReadPacket = buildPacket { writeObject(obj) }
fun <T : Any> IOFormat<T>.writeBytes(obj: T): ByteArray = buildPacket { writeObject(obj) }.readBytes()
fun <T : Any> IOFormat<T>.readBytes(array: ByteArray): T {
//= ByteReadPacket(array).readThis()
val byteArrayInput: Input = object : AbstractInput(
IoBuffer.Pool.borrow(),
remaining = array.size.toLong(),
pool = IoBuffer.Pool
) {
var written = 0
override fun closeSource() {
// do nothing
}
override fun fill(): IoBuffer? {
if (array.size - written <= 0) return null
return IoBuffer.Pool.fill {
reserveEndGap(IoBuffer.ReservedSize)
val toWrite = min(capacity, array.size - written)
writeFully(array, written, toWrite)
written += toWrite
}
}
}
return byteArrayInput.readObject()
}
object DoubleIOFormat : IOFormat<Double>, IOFormatFactory<Double> {
override fun invoke(meta: Meta, context: Context): IOFormat<Double> = this
override val name: Name = "double".asName()
override val type: KClass<out Double> get() = Double::class
override fun Output.writeObject(obj: Double) {
writeDouble(obj)
}
override fun Input.readObject(): Double = readDouble()
}
object ValueIOFormat : IOFormat<Value>, IOFormatFactory<Value> {
override fun invoke(meta: Meta, context: Context): IOFormat<Value> = this
override val name: Name = "value".asName()
override val type: KClass<out Value> get() = Value::class
override fun Output.writeObject(obj: Value) {
BinaryMetaFormat.run { writeValue(obj) }
}
override fun Input.readObject(): Value {
return (BinaryMetaFormat.run { readMetaItem() } as? MetaItem.ValueItem)?.value
?: error("The item is not a value")
}
}
/**
* Experimental
*/
@ImplicitReflectionSerializer
class SerializerIOFormat<T : Any>(
type: KClass<T>,
val serializer: KSerializer<T> = type.serializer()
) : IOFormat<T> {
//override val name: Name = type.simpleName?.toName() ?: EmptyName
override fun Output.writeObject(obj: T) {
val bytes = Cbor.plain.dump(serializer, obj)
writeFully(bytes)
}
override fun Input.readObject(): T {
//FIXME reads the whole input
val bytes = readBytes()
return Cbor.plain.load(serializer, bytes)
}
}

52
dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOPlugin.kt
View File

@ -1,37 +1,61 @@
package hep.dataforge.io
import hep.dataforge.context.AbstractPlugin
import hep.dataforge.context.PluginFactory
import hep.dataforge.context.PluginTag
import hep.dataforge.context.content
import hep.dataforge.meta.Meta
import hep.dataforge.context.*
import hep.dataforge.io.EnvelopeFormatFactory.Companion.ENVELOPE_FORMAT_TYPE
import hep.dataforge.io.IOFormatFactory.Companion.IO_FORMAT_TYPE
import hep.dataforge.io.MetaFormatFactory.Companion.META_FORMAT_TYPE
import hep.dataforge.meta.*
import hep.dataforge.names.Name
import hep.dataforge.names.get
import kotlin.reflect.KClass
class IOPlugin(meta: Meta) : AbstractPlugin(meta) {
override val tag: PluginTag get() = Companion.tag
val metaFormats by lazy {
context.content<MetaFormat>(MetaFormat.META_FORMAT_TYPE).values
val metaFormatFactories by lazy {
context.content<MetaFormatFactory>(META_FORMAT_TYPE).values
}
fun metaFormat(key: Short): MetaFormat? = metaFormats.find { it.key == key }
fun metaFormat(name: String): MetaFormat? = metaFormats.find { it.name == name }
fun metaFormat(key: Short, meta: Meta = EmptyMeta): MetaFormat? =
metaFormatFactories.find { it.key == key }?.invoke(meta)
fun metaFormat(name: String, meta: Meta = EmptyMeta): MetaFormat? =
metaFormatFactories.find { it.name.toString() == name }?.invoke(meta)
val envelopeFormatFactories by lazy {
context.content<EnvelopeFormatFactory>(ENVELOPE_FORMAT_TYPE).values
}
override fun provideTop(target: String): Map<Name, Any> {
return when (target) {
MetaFormat.META_FORMAT_TYPE -> defaultMetaFormats.toMap()
EnvelopeFormat.ENVELOPE_FORMAT_TYPE -> defaultEnvelopeFormats.toMap()
META_FORMAT_TYPE -> defaultMetaFormats.toMap()
ENVELOPE_FORMAT_TYPE -> defaultEnvelopeFormats.toMap()
else -> super.provideTop(target)
}
}
val ioFormats: Map<Name, IOFormatFactory<*>> by lazy {
context.content<IOFormatFactory<*>>(IO_FORMAT_TYPE)
}
fun <T : Any> resolveIOFormat(item: MetaItem<*>, type: KClass<out T>): IOFormat<T>? {
val key = item.string ?: item.node["name"]?.string ?: error("Format name not defined")
return ioFormats[key]?.let {
@Suppress("UNCHECKED_CAST")
if (it.type != type) error("Format type ${it.type} is not the same as requested type $type")
else it.invoke(item.node["meta"].node ?: EmptyMeta, context) as IOFormat<T>
}
}
companion object : PluginFactory<IOPlugin> {
val defaultMetaFormats: List<MetaFormat> = listOf(JsonMetaFormat, BinaryMetaFormat)
val defaultMetaFormats: List<MetaFormatFactory> = listOf(JsonMetaFormat, BinaryMetaFormat)
val defaultEnvelopeFormats = listOf(TaggedEnvelopeFormat)
override val tag: PluginTag = PluginTag("io", group = PluginTag.DATAFORGE_GROUP)
override val type: KClass<out IOPlugin> = IOPlugin::class
override fun invoke(meta: Meta): IOPlugin = IOPlugin(meta)
override fun invoke(meta: Meta, context: Context): IOPlugin = IOPlugin(meta)
}
}
}
val Context.io: IOPlugin get() = plugins.fetch(IOPlugin)

46
dataforge-io/src/commonMain/kotlin/hep/dataforge/io/JsonMetaFormat.kt
View File

@ -1,12 +1,17 @@
@file:Suppress("UNUSED_PARAMETER")
package hep.dataforge.io
import hep.dataforge.context.Context
import hep.dataforge.descriptors.ItemDescriptor
import hep.dataforge.descriptors.NodeDescriptor
import hep.dataforge.descriptors.ValueDescriptor
import hep.dataforge.meta.Meta
import hep.dataforge.meta.MetaBase
import hep.dataforge.meta.MetaItem
import hep.dataforge.names.Name
import hep.dataforge.names.NameToken
import hep.dataforge.names.plus
import hep.dataforge.names.toName
import hep.dataforge.values.*
import kotlinx.io.core.Input
@ -19,28 +24,32 @@ import kotlin.collections.component2
import kotlin.collections.set
object JsonMetaFormat : MetaFormat {
override val name: String = "json"
override val key: Short = 0x4a53//"JS"
class JsonMetaFormat(private val json: Json = Json.indented) : MetaFormat {
override fun Output.writeMeta(meta: Meta, descriptor: NodeDescriptor?) {
val json = meta.toJson(descriptor)
writeText(json.toString())
val jsonObject = meta.toJson(descriptor)
writeText(json.stringify(JsonObjectSerializer, jsonObject))
}
override fun Input.readMeta(descriptor: NodeDescriptor?): Meta {
val str = readText()
val json = Json.plain.parseJson(str)
val jsonElement = json.parseJson(str)
return jsonElement.toMeta()
}
if (json is JsonObject) {
return json.toMeta()
} else {
TODO("Non-object root not supported")
}
companion object : MetaFormatFactory {
val default = JsonMetaFormat()
override fun invoke(meta: Meta, context: Context): MetaFormat = default
override val name: Name = super.name + "json"
override val key: Short = 0x4a53//"JS"
}
}
/**
* @param descriptor reserved for custom serialization in future
*/
fun Value.toJson(descriptor: ValueDescriptor? = null): JsonElement {
return if (isList()) {
JsonArray(list.map { it.toJson() })
@ -54,7 +63,7 @@ fun Value.toJson(descriptor: ValueDescriptor? = null): JsonElement {
}
}
//Use theese methods to customize JSON key mapping
//Use these methods to customize JSON key mapping
private fun NameToken.toJsonKey(descriptor: ItemDescriptor?) = toString()
private fun NodeDescriptor?.getDescriptor(key: String) = this?.items?.get(key)
@ -78,7 +87,12 @@ fun Meta.toJson(descriptor: NodeDescriptor? = null): JsonObject {
return JsonObject(map)
}
fun JsonObject.toMeta(descriptor: NodeDescriptor? = null): JsonMeta = JsonMeta(this, descriptor)
fun JsonElement.toMeta(descriptor: NodeDescriptor? = null): Meta {
return when (val item = toMetaItem(descriptor)) {
is MetaItem.NodeItem<*> -> item.node
is MetaItem.ValueItem ->item.value.toMeta()
}
}
fun JsonPrimitive.toValue(descriptor: ValueDescriptor?): Value {
return when (this) {
@ -93,7 +107,7 @@ fun JsonElement.toMetaItem(descriptor: ItemDescriptor? = null): MetaItem<JsonMet
MetaItem.ValueItem(value)
}
is JsonObject -> {
val meta = toMeta(descriptor as? NodeDescriptor)
val meta = JsonMeta(this, descriptor as? NodeDescriptor)
MetaItem.NodeItem(meta)
}
is JsonArray -> {
@ -129,7 +143,7 @@ class JsonMeta(val json: JsonObject, val descriptor: NodeDescriptor? = null) : M
this[name] = MetaItem.ValueItem(value.toValue(itemDescriptor as? ValueDescriptor)) as MetaItem<JsonMeta>
}
is JsonObject -> {
this[name] = MetaItem.NodeItem(value.toMeta(itemDescriptor as? NodeDescriptor))
this[name] = MetaItem.NodeItem(JsonMeta(value, itemDescriptor as? NodeDescriptor))
}
is JsonArray -> {
when {

47
dataforge-io/src/commonMain/kotlin/hep/dataforge/io/MetaFormat.kt
View File

@ -1,49 +1,64 @@
package hep.dataforge.io
import hep.dataforge.context.Named
import hep.dataforge.context.Context
import hep.dataforge.descriptors.NodeDescriptor
import hep.dataforge.io.MetaFormat.Companion.META_FORMAT_TYPE
import hep.dataforge.io.MetaFormatFactory.Companion.META_FORMAT_TYPE
import hep.dataforge.meta.Meta
import hep.dataforge.names.Name
import hep.dataforge.names.asName
import hep.dataforge.provider.Type
import kotlinx.io.core.*
import kotlin.reflect.KClass
/**
* A format for meta serialization
*/
@Type(META_FORMAT_TYPE)
interface MetaFormat : IOFormat<Meta>, Named {
override val name: String
val key: Short
override fun Output.writeThis(obj: Meta) {
interface MetaFormat : IOFormat<Meta> {
override fun Output.writeObject(obj: Meta) {
writeMeta(obj, null)
}
override fun Input.readThis(): Meta = readMeta(null)
override fun Input.readObject(): Meta = readMeta()
fun Output.writeMeta(meta: Meta, descriptor: NodeDescriptor? = null)
fun Input.readMeta(descriptor: NodeDescriptor? = null): Meta
}
companion object{
const val META_FORMAT_TYPE = "metaFormat"
@Type(META_FORMAT_TYPE)
interface MetaFormatFactory : IOFormatFactory<Meta> {
override val name: Name get() = "meta".asName()
override val type: KClass<out Meta> get() = Meta::class
val key: Short
override operator fun invoke(meta: Meta, context: Context): MetaFormat
companion object {
const val META_FORMAT_TYPE = "io.format.meta"
}
}
fun Meta.toString(format: MetaFormat): String = buildPacket {
format.run { writeThis(this@toString) }
format.run { writeObject(this@toString) }
}.readText()
fun Meta.toBytes(format: MetaFormat = JsonMetaFormat): ByteReadPacket = buildPacket {
format.run { writeThis(this@toBytes) }
}
fun Meta.toString(formatFactory: MetaFormatFactory): String = toString(formatFactory())
fun Meta.toBytes(format: MetaFormat = JsonMetaFormat.default): ByteReadPacket = buildPacket {
format.run { writeObject(this@toBytes) }
}
fun MetaFormat.parse(str: String): Meta {
return ByteReadPacket(str.toByteArray()).readThis()
return buildPacket { writeText(str) }.readObject()
}
fun MetaFormatFactory.parse(str: String): Meta = invoke().parse(str)
fun MetaFormat.fromBytes(packet: ByteReadPacket): Meta {
return packet.readThis()
return packet.readObject()
}

53
dataforge-io/src/commonMain/kotlin/hep/dataforge/io/MetaSerializer.kt
View File

@ -1,53 +0,0 @@
package hep.dataforge.io
import hep.dataforge.meta.Config
import hep.dataforge.meta.Meta
import hep.dataforge.meta.toConfig
import hep.dataforge.names.Name
import hep.dataforge.names.toName
import kotlinx.serialization.*
import kotlinx.serialization.internal.StringDescriptor
import kotlinx.serialization.json.JsonObjectSerializer
@Serializer(Name::class)
object NameSerializer : KSerializer<Name> {
override val descriptor: SerialDescriptor = StringDescriptor
override fun deserialize(decoder: Decoder): Name {
return decoder.decodeString().toName()
}
override fun serialize(encoder: Encoder, obj: Name) {
encoder.encodeString(obj.toString())
}
}
/**
* Serialized for meta
*/
@Serializer(Meta::class)
object MetaSerializer : KSerializer<Meta> {
override val descriptor: SerialDescriptor = JsonObjectSerializer.descriptor
override fun deserialize(decoder: Decoder): Meta {
//currently just delegates serialization to json serializer
return JsonObjectSerializer.deserialize(decoder).toMeta()
}
override fun serialize(encoder: Encoder, obj: Meta) {
JsonObjectSerializer.serialize(encoder, obj.toJson())
}
}
@Serializer(Config::class)
object ConfigSerializer : KSerializer<Config> {
override val descriptor: SerialDescriptor = JsonObjectSerializer.descriptor
override fun deserialize(decoder: Decoder): Config {
return JsonObjectSerializer.deserialize(decoder).toMeta().toConfig()
}
override fun serialize(encoder: Encoder, obj: Config) {
JsonObjectSerializer.serialize(encoder, obj.toJson())
}
}

8
dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Responder.kt Normal file
View File

@ -0,0 +1,8 @@
package hep.dataforge.io
interface Responder {
/**
* Send a request and wait for response for this specific request
*/
suspend fun respond(request: Envelope): Envelope
}

125
dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt
View File

@ -1,43 +1,50 @@
package hep.dataforge.io
import hep.dataforge.context.Context
import hep.dataforge.meta.Meta
import hep.dataforge.meta.get
import hep.dataforge.meta.string
import hep.dataforge.names.Name
import hep.dataforge.names.plus
import hep.dataforge.names.toName
import kotlinx.io.charsets.Charsets
import kotlinx.io.core.*
@ExperimentalUnsignedTypes
object TaggedEnvelopeFormat : EnvelopeFormat {
const val VERSION = "DF03"
private const val START_SEQUENCE = "#~"
private const val END_SEQUENCE = "~#\r\n"
private const val TAG_SIZE = 26u
class TaggedEnvelopeFormat(
val io: IOPlugin,
val version: VERSION = TaggedEnvelopeFormat.VERSION.DF02
) : EnvelopeFormat {
// private val metaFormat = io.metaFormat(metaFormatKey)
// ?: error("Meta format with key $metaFormatKey could not be resolved in $io")
override val name: String get() = VERSION
private fun Tag.toBytes(): ByteReadPacket = buildPacket(24) {
writeText(START_SEQUENCE)
writeText(VERSION)
writeText(version.name)
writeShort(metaFormatKey)
writeUInt(metaSize)
writeULong(dataSize)
when (version) {
TaggedEnvelopeFormat.VERSION.DF02 -> {
writeUInt(dataSize.toUInt())
}
TaggedEnvelopeFormat.VERSION.DF03 -> {
writeULong(dataSize)
}
}
writeText(END_SEQUENCE)
}
private fun Input.readTag(): Tag {
val start = readTextExactBytes(2)
if (start != START_SEQUENCE) error("The input is not an envelope")
val version = readTextExactBytes(4)
if (version != VERSION) error("Wrong version of DataForge: expected $VERSION but found $version")
val metaFormatKey = readShort()
val metaLength = readUInt()
val dataLength = readULong()
return Tag(metaFormatKey, metaLength, dataLength)
}
override fun Output.writeEnvelope(envelope: Envelope, format: MetaFormat) {
val metaBytes = format.writeBytes(envelope.meta)
val tag = Tag(format.key, metaBytes.size.toUInt(), envelope.data?.size ?: 0.toULong())
override fun Output.writeEnvelope(envelope: Envelope, metaFormatFactory: MetaFormatFactory, formatMeta: Meta) {
val metaFormat = metaFormatFactory.invoke(formatMeta, io.context)
val metaBytes = metaFormat.writeBytes(envelope.meta)
val tag = Tag(metaFormatFactory.key, metaBytes.size.toUInt() + 2u, envelope.data?.size ?: 0.toULong())
writePacket(tag.toBytes())
writeFully(metaBytes)
writeText("\r\n")
envelope.data?.read { copyTo(this@writeEnvelope) }
flush()
}
/**
@ -46,30 +53,29 @@ object TaggedEnvelopeFormat : EnvelopeFormat {
* @param input an input to read from
* @param formats a collection of meta formats to resolve
*/
override fun Input.readEnvelope(formats: Collection<MetaFormat>): Envelope {
val tag = readTag()
override fun Input.readObject(): Envelope {
val tag = readTag(version)
val metaFormat = formats.find { it.key == tag.metaFormatKey }
val metaFormat = io.metaFormat(tag.metaFormatKey)
?: error("Meta format with key ${tag.metaFormatKey} not found")
val metaPacket = ByteReadPacket(readBytes(tag.metaSize.toInt()))
val meta = metaFormat.run { metaPacket.readThis() }
val dataBytes = readBytes(tag.dataSize.toInt())
val meta = metaFormat.run { metaPacket.readObject() }
return SimpleEnvelope(meta, ArrayBinary(dataBytes))
}
override fun Input.readPartial(formats: Collection<MetaFormat>): PartialEnvelope {
val tag = readTag()
override fun Input.readPartial(): PartialEnvelope {
val tag = readTag(version)
val metaFormat = formats.find { it.key == tag.metaFormatKey }
val metaFormat = io.metaFormat(tag.metaFormatKey)
?: error("Meta format with key ${tag.metaFormatKey} not found")
val metaPacket = ByteReadPacket(readBytes(tag.metaSize.toInt()))
val meta = metaFormat.run { metaPacket.readThis() }
val meta = metaFormat.run { metaPacket.readObject() }
return PartialEnvelope(meta, TAG_SIZE + tag.metaSize, tag.dataSize)
return PartialEnvelope(meta, version.tagSize + tag.metaSize, tag.dataSize)
}
private data class Tag(
@ -78,4 +84,57 @@ object TaggedEnvelopeFormat : EnvelopeFormat {
val dataSize: ULong
)
enum class VERSION(val tagSize: UInt) {
DF02(20u),
DF03(24u)
}
companion object : EnvelopeFormatFactory {
private const val START_SEQUENCE = "#~"
private const val END_SEQUENCE = "~#\r\n"
override val name: Name = super.name + "tagged"
override fun invoke(meta: Meta, context: Context): EnvelopeFormat {
val io = context.io
val metaFormatName = meta["name"].string?.toName() ?: JsonMetaFormat.name
val metaFormatFactory = io.metaFormatFactories.find { it.name == metaFormatName }
?: error("Meta format could not be resolved")
return TaggedEnvelopeFormat(io)
}
private fun Input.readTag(version: VERSION): Tag {
val start = readTextExactBytes(2, charset = Charsets.ISO_8859_1)
if (start != START_SEQUENCE) error("The input is not an envelope")
val versionString = readTextExactBytes(4, charset = Charsets.ISO_8859_1)
if (version.name != versionString) error("Wrong version of DataForge: expected $version but found $versionString")
val metaFormatKey = readShort()
val metaLength = readUInt()
val dataLength: ULong = when (version) {
VERSION.DF02 -> readUInt().toULong()
VERSION.DF03 -> readULong()
}
val end = readTextExactBytes(4, charset = Charsets.ISO_8859_1)
if (end != END_SEQUENCE) error("The input is not an envelope")
return Tag(metaFormatKey, metaLength, dataLength)
}
override fun peekFormat(io: IOPlugin, input: Input): EnvelopeFormat? {
return try {
val header = input.readTextExactBytes(6)
when (header.substring(2..5)) {
VERSION.DF02.name -> TaggedEnvelopeFormat(io, VERSION.DF02)
VERSION.DF03.name -> TaggedEnvelopeFormat(io, VERSION.DF03)
else -> null
}
} catch (ex: Exception) {
null
}
}
val default by lazy { invoke() }
}
}

189
dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaglessEnvelopeFormat.kt Normal file
View File

@ -0,0 +1,189 @@
package hep.dataforge.io
import hep.dataforge.context.Context
import hep.dataforge.meta.*
import hep.dataforge.names.asName
import kotlinx.io.core.*
import kotlinx.serialization.toUtf8Bytes
class TaglessEnvelopeFormat(
val io: IOPlugin,
meta: Meta = EmptyMeta
) : EnvelopeFormat {
private val metaStart = meta[META_START_PROPERTY].string ?: DEFAULT_META_START
private val dataStart = meta[DATA_START_PROPERTY].string ?: DEFAULT_DATA_START
private fun Output.writeProperty(key: String, value: Any) {
writeText("#? $key: $value;\r\n")
}
override fun Output.writeEnvelope(envelope: Envelope, metaFormatFactory: MetaFormatFactory, formatMeta: Meta) {
val metaFormat = metaFormatFactory(formatMeta, io.context)
//printing header
writeText(TAGLESS_ENVELOPE_HEADER + "\r\n")
//printing all properties
writeProperty(META_TYPE_PROPERTY, metaFormatFactory.type)
//TODO add optional metaFormat properties
writeProperty(DATA_LENGTH_PROPERTY, envelope.data?.size ?: 0)
//Printing meta
if (!envelope.meta.isEmpty()) {
val metaBytes = metaFormat.writeBytes(envelope.meta)
writeProperty(META_LENGTH_PROPERTY, metaBytes.size)
writeText(metaStart + "\r\n")
writeFully(metaBytes)
writeText("\r\n")
}
//Printing data
envelope.data?.let { data ->
writeText(dataStart + "\r\n")
writeFully(data.toBytes())
}
}
override fun Input.readObject(): Envelope {
var line: String = ""
do {
line = readUTF8Line() ?: error("Input does not contain tagless envelope header")
} while (!line.startsWith(TAGLESS_ENVELOPE_HEADER))
val properties = HashMap<String, String>()
line = ""
while (line.isBlank() || line.startsWith("#?")) {
if (line.startsWith("#?")) {
val match = propertyPattern.find(line)
?: error("Line $line does not match property declaration pattern")
val (key, value) = match.destructured
properties[key] = value
}
line = readUTF8Line() ?: return SimpleEnvelope(Meta.empty, null)
}
var meta: Meta = EmptyMeta
if (line.startsWith(metaStart)) {
val metaFormat = properties[META_TYPE_PROPERTY]?.let { io.metaFormat(it) } ?: JsonMetaFormat.default
val metaSize = properties.get(META_LENGTH_PROPERTY)?.toInt()
meta = if (metaSize != null) {
val metaPacket = buildPacket {
writeFully(readBytes(metaSize))
}
metaFormat.run { metaPacket.readObject() }
} else {
metaFormat.run {
readObject()
}
}
}
do {
line = readUTF8Line() ?: return SimpleEnvelope(meta, null)
//returning an Envelope without data if end of input is reached
} while (!line.startsWith(dataStart))
val data: Binary? = if (properties.containsKey(DATA_LENGTH_PROPERTY)) {
val bytes = ByteArray(properties[DATA_LENGTH_PROPERTY]!!.toInt())
readFully(bytes)
bytes.asBinary()
} else {
val bytes = readBytes()
bytes.asBinary()
}
return SimpleEnvelope(meta, data)
}
override fun Input.readPartial(): PartialEnvelope {
var offset = 0u
var line: String = ""
do {
line = readUTF8Line() ?: error("Input does not contain tagless envelope header")
offset += line.toUtf8Bytes().size.toUInt()
} while (!line.startsWith(TAGLESS_ENVELOPE_HEADER))
val properties = HashMap<String, String>()
line = ""
while (line.isBlank() || line.startsWith("#?")) {
if (line.startsWith("#?")) {
val match = propertyPattern.find(line)
?: error("Line $line does not match property declaration pattern")
val (key, value) = match.destructured
properties[key] = value
}
line = readUTF8Line() ?: return PartialEnvelope(Meta.empty, offset.toUInt(), 0.toULong())
offset += line.toUtf8Bytes().size.toUInt()
}
var meta: Meta = EmptyMeta
if (line.startsWith(metaStart)) {
val metaFormat = properties[META_TYPE_PROPERTY]?.let { io.metaFormat(it) } ?: JsonMetaFormat.default
val metaSize = properties.get(META_LENGTH_PROPERTY)?.toInt()
meta = if (metaSize != null) {
val metaPacket = buildPacket {
writeFully(readBytes(metaSize))
}
offset += metaSize.toUInt()
metaFormat.run { metaPacket.readObject() }
} else {
error("Can't partially read an envelope with undefined meta size")
}
}
do {
line = readUTF8Line() ?: return PartialEnvelope(Meta.empty, offset.toUInt(), 0.toULong())
offset += line.toUtf8Bytes().size.toUInt()
//returning an Envelope without data if end of input is reached
} while (!line.startsWith(dataStart))
val dataSize = properties[DATA_LENGTH_PROPERTY]?.toULong()
return PartialEnvelope(meta, offset, dataSize)
}
companion object : EnvelopeFormatFactory {
private val propertyPattern = "#\\?\\s*(?<key>[\\w.]*)\\s*:\\s*(?<value>[^;]*);?".toRegex()
const val META_TYPE_PROPERTY = "metaType"
const val META_LENGTH_PROPERTY = "metaLength"
const val DATA_LENGTH_PROPERTY = "dataLength"
const val TAGLESS_ENVELOPE_TYPE = "tagless"
const val TAGLESS_ENVELOPE_HEADER = "#~DFTL~#"
const val META_START_PROPERTY = "metaSeparator"
const val DEFAULT_META_START = "#~META~#"
const val DATA_START_PROPERTY = "dataSeparator"
const val DEFAULT_DATA_START = "#~DATA~#"
const val code: Int = 0x4446544c //DFTL
override val name = TAGLESS_ENVELOPE_TYPE.asName()
override fun invoke(meta: Meta, context: Context): EnvelopeFormat {
return TaglessEnvelopeFormat(context.io, meta)
}
val default by lazy { invoke() }
override fun peekFormat(io: IOPlugin, input: Input): EnvelopeFormat? {
return try {
val buffer = ByteArray(TAGLESS_ENVELOPE_HEADER.length)
input.readFully(buffer)
return if (buffer.toString() == TAGLESS_ENVELOPE_HEADER) {
TaglessEnvelopeFormat(io)
} else {
null
}
} catch (ex: Exception) {
null
}
}
}
}

75
dataforge-io/src/commonMain/kotlin/hep/dataforge/io/functions/FunctionServer.kt Normal file
View File

@ -0,0 +1,75 @@
package hep.dataforge.io.functions
import hep.dataforge.context.ContextAware
import hep.dataforge.io.IOFormat
import hep.dataforge.io.IOPlugin
import hep.dataforge.meta.Meta
import hep.dataforge.meta.get
import hep.dataforge.names.asName
import hep.dataforge.names.plus
import kotlin.reflect.KClass
/**
* A server that could produce asynchronous function values
*/
interface FunctionServer : ContextAware {
/**
* Call a function with given name and descriptor
*/
suspend fun <T : Any, R : Any> call(meta: Meta, arg: T, inputType: KClass<out T>, outputType: KClass<out R>): R
suspend fun <T : Any, R : Any> callMany(
meta: Meta,
arg: List<T>,
inputType: KClass<out T>,
outputType: KClass<out R>
): List<R> = List(arg.size) {
call<T, R>(meta, arg[it], inputType, outputType)
}
/**
* Get a generic suspended function with given name and descriptor
*/
fun <T : Any, R : Any> function(
meta: Meta,
inputType: KClass<out T>,
outputType: KClass<out R>
): (suspend (T) -> R) = { call(meta, it, inputType, outputType) }
companion object {
const val FUNCTION_NAME_KEY = "function"
val FORMAT_KEY = "format".asName()
val INPUT_FORMAT_KEY = FORMAT_KEY + "input"
val OUTPUT_FORMAT_KEY = FORMAT_KEY + "output"
}
}
suspend inline fun <reified T : Any, reified R : Any> FunctionServer.call(meta: Meta, arg: T) =
call(meta, arg, T::class, R::class)
suspend inline fun <reified T : Any, reified R : Any> FunctionServer.callMany(meta: Meta, arg: List<T>) =
callMany(meta, arg, T::class, R::class)
inline fun <reified T : Any, reified R : Any> FunctionServer.function(meta: Meta) =
function(meta, T::class, R::class)
fun <T : Any> IOPlugin.getInputFormat(meta: Meta, type: KClass<out T>): IOFormat<T> {
return meta[FunctionServer.INPUT_FORMAT_KEY]?.let {
resolveIOFormat<T>(it, type)
} ?: error("Input format not resolved")
}
fun <R : Any> IOPlugin.getOutputFormat(meta: Meta, type: KClass<out R>): IOFormat<R> {
return meta[FunctionServer.OUTPUT_FORMAT_KEY]?.let {
resolveIOFormat<R>(it, type)
} ?: error("Input format not resolved")
}
inline fun <reified T : Any> IOPlugin.getInputFormat(meta: Meta): IOFormat<T> =
getInputFormat(meta, T::class)
inline fun <reified R : Any> IOPlugin.getOutputFormat(meta: Meta): IOFormat<R> =
getOutputFormat(meta, R::class)

98
dataforge-io/src/commonMain/kotlin/hep/dataforge/io/functions/RemoteFunctionClient.kt Normal file
View File

@ -0,0 +1,98 @@
package hep.dataforge.io.functions
import hep.dataforge.context.Context
import hep.dataforge.context.ContextAware
import hep.dataforge.io.*
import hep.dataforge.meta.Meta
import hep.dataforge.meta.get
import hep.dataforge.meta.int
import kotlin.reflect.KClass
class RemoteFunctionClient(override val context: Context, val responder: Responder) : FunctionServer, ContextAware {
private fun <T : Any> IOPlugin.encodeOne(
meta: Meta,
value: T,
valueType: KClass<out T> = value::class
): Envelope = Envelope.invoke {
meta(meta)
type = REQUEST_TYPE
data {
val inputFormat: IOFormat<T> = getInputFormat(meta, valueType)
inputFormat.run {
writeObject(value)
}
}
}
private fun <T : Any> IOPlugin.encodeMany(
meta: Meta,
values: List<T>,
valueType: KClass<out T>
): Envelope = Envelope.invoke {
meta(meta)
type = REQUEST_TYPE
meta {
SIZE_KEY put values.size
}
data {
val inputFormat: IOFormat<T> = getInputFormat(meta, valueType)
inputFormat.run {
values.forEach {
writeObject(it)
}
}
}
}
private fun <R : Any> IOPlugin.decode(envelope: Envelope, valueType: KClass<out R>): List<R> {
require(envelope.type == RESPONSE_TYPE) { "Unexpected message type: ${envelope.type}" }
val size = envelope.meta[SIZE_KEY].int ?: 1
return if (size == 0) {
emptyList()
} else {
val outputFormat: IOFormat<R> = getOutputFormat(envelope.meta, valueType)
envelope.data?.read {
List<R>(size) {
outputFormat.run {
readObject()
}
}
} ?: error("Message does not contain data")
}
}
private val plugin by lazy {
context.plugins.load(IOPlugin)
}
override suspend fun <T : Any, R : Any> call(
meta: Meta,
arg: T,
inputType: KClass<out T>,
outputType: KClass<out R>
): R = plugin.run {
val request = encodeOne(meta, arg)
val response = responder.respond(request)
return decode<R>(response, outputType).first()
}
override suspend fun <T : Any, R : Any> callMany(
meta: Meta,
arg: List<T>,
inputType: KClass<out T>,
outputType: KClass<out R>
): List<R> = plugin.run {
val request = encodeMany(meta, arg, inputType)
val response = responder.respond(request)
return decode<R>(response, outputType)
}
companion object {
const val REQUEST_TYPE = "function.request"
const val RESPONSE_TYPE = "function.response"
const val SIZE_KEY = "size"
}
}

58
dataforge-io/src/commonMain/kotlin/hep/dataforge/io/functions/RemoteFunctionServer.kt Normal file
View File

@ -0,0 +1,58 @@
package hep.dataforge.io.functions
import hep.dataforge.context.Context
import hep.dataforge.context.ContextAware
import hep.dataforge.io.Envelope
import hep.dataforge.io.IOPlugin
import hep.dataforge.io.Responder
import hep.dataforge.io.type
import hep.dataforge.meta.get
import hep.dataforge.meta.int
class RemoteFunctionServer(
override val context: Context,
val functionServer: FunctionServer
) : ContextAware, Responder {
private val plugin by lazy {
context.plugins.load(IOPlugin)
}
override suspend fun respond(request: Envelope): Envelope {
require(request.type == RemoteFunctionClient.REQUEST_TYPE) { "Unexpected message type: ${request.type}" }
val inputFormat = plugin.getInputFormat<Any>(request.meta)
val outputFormat = plugin.getOutputFormat<Any>(request.meta)
val size = request.meta[RemoteFunctionClient.SIZE_KEY].int ?: 1
val input = request.data?.read {
inputFormat.run {
List(size) {
readObject()
}
}
} ?: error("Input is empty")
val output = functionServer.callMany<Any, Any>(
request.meta,
input
)
return Envelope.invoke {
meta {
meta(request.meta)
}
type = RemoteFunctionClient.RESPONSE_TYPE
data {
outputFormat.run {
output.forEach {
writeObject(it)
}
}
}
}
}
}

141
dataforge-io/src/commonMain/kotlin/hep/dataforge/io/serialization/MetaSerializer.kt Normal file
View File

@ -0,0 +1,141 @@
package hep.dataforge.io.serialization
import hep.dataforge.io.toJson
import hep.dataforge.io.toMeta
import hep.dataforge.meta.*
import hep.dataforge.names.NameToken
import hep.dataforge.values.*
import kotlinx.serialization.*
import kotlinx.serialization.internal.*
import kotlinx.serialization.json.JsonInput
import kotlinx.serialization.json.JsonObjectSerializer
import kotlinx.serialization.json.JsonOutput
@Serializer(Value::class)
object ValueSerializer : KSerializer<Value> {
private val valueTypeSerializer = EnumSerializer(ValueType::class)
private val listSerializer by lazy { ArrayListSerializer(ValueSerializer) }
override val descriptor: SerialDescriptor = descriptor("hep.dataforge.values.Value") {
boolean("isList")
enum<ValueType>("valueType")
element("value", null)
}
private fun Decoder.decodeValue(): Value {
return when (decode(valueTypeSerializer)) {
ValueType.NULL -> Null
ValueType.NUMBER -> decodeDouble().asValue() //TODO differentiate?
ValueType.BOOLEAN -> decodeBoolean().asValue()
ValueType.STRING -> decodeString().asValue()
else -> decodeString().parseValue()
}
}
override fun deserialize(decoder: Decoder): Value {
val isList = decoder.decodeBoolean()
return if (isList) {
listSerializer.deserialize(decoder).asValue()
} else {
decoder.decodeValue()
}
}
private fun Encoder.encodeValue(value: Value) {
encode(valueTypeSerializer, value.type)
when (value.type) {
ValueType.NULL -> {
// do nothing
}
ValueType.NUMBER -> encodeDouble(value.double)
ValueType.BOOLEAN -> encodeBoolean(value.boolean)
ValueType.STRING -> encodeString(value.string)
else -> encodeString(value.string)
}
}
override fun serialize(encoder: Encoder, obj: Value) {
encoder.encodeBoolean(obj.isList())
if (obj.isList()) {
listSerializer.serialize(encoder, obj.list)
} else {
encoder.encodeValue(obj)
}
}
}
@Serializer(MetaItem::class)
object MetaItemSerializer : KSerializer<MetaItem<*>> {
override val descriptor: SerialDescriptor = descriptor("MetaItem") {
boolean("isNode")
element("value", null)
}
override fun deserialize(decoder: Decoder): MetaItem<*> {
val isNode = decoder.decodeBoolean()
return if (isNode) {
MetaItem.NodeItem(decoder.decode(MetaSerializer))
} else {
MetaItem.ValueItem(decoder.decode(ValueSerializer))
}
}
override fun serialize(encoder: Encoder, obj: MetaItem<*>) {
encoder.encodeBoolean(obj is MetaItem.NodeItem)
when (obj) {
is MetaItem.NodeItem -> MetaSerializer.serialize(encoder, obj.node)
is MetaItem.ValueItem -> ValueSerializer.serialize(encoder, obj.value)
}
}
}
private class DeserializedMeta(override val items: Map<NameToken, MetaItem<*>>) : MetaBase()
/**
* Serialized for meta
*/
@Serializer(Meta::class)
object MetaSerializer : KSerializer<Meta> {
private val mapSerializer = HashMapSerializer(
StringSerializer,
MetaItemSerializer
)
override val descriptor: SerialDescriptor = NamedMapClassDescriptor(
"hep.dataforge.meta.Meta",
StringSerializer.descriptor,
MetaItemSerializer.descriptor
)
override fun deserialize(decoder: Decoder): Meta {
return if (decoder is JsonInput) {
JsonObjectSerializer.deserialize(decoder).toMeta()
} else {
DeserializedMeta(mapSerializer.deserialize(decoder).mapKeys { NameToken(it.key) })
}
}
override fun serialize(encoder: Encoder, obj: Meta) {
if (encoder is JsonOutput) {
JsonObjectSerializer.serialize(encoder, obj.toJson())
} else {
mapSerializer.serialize(encoder, obj.items.mapKeys { it.key.toString() })
}
}
}
@Serializer(Config::class)
object ConfigSerializer : KSerializer<Config> {
override val descriptor: SerialDescriptor = MetaSerializer.descriptor
override fun deserialize(decoder: Decoder): Config {
return MetaSerializer.deserialize(decoder).toConfig()
}
override fun serialize(encoder: Encoder, obj: Config) {
MetaSerializer.serialize(encoder, obj)
}
}

33
dataforge-io/src/commonMain/kotlin/hep/dataforge/io/serialization/nameSerializers.kt Normal file
View File

@ -0,0 +1,33 @@
package hep.dataforge.io.serialization
import hep.dataforge.names.Name
import hep.dataforge.names.NameToken
import hep.dataforge.names.toName
import kotlinx.serialization.*
import kotlinx.serialization.internal.StringDescriptor
@Serializer(Name::class)
object NameSerializer : KSerializer<Name> {
override val descriptor: SerialDescriptor = StringDescriptor.withName("Name")
override fun deserialize(decoder: Decoder): Name {
return decoder.decodeString().toName()
}
override fun serialize(encoder: Encoder, obj: Name) {
encoder.encodeString(obj.toString())
}
}
@Serializer(NameToken::class)
object NameTokenSerializer : KSerializer<NameToken> {
override val descriptor: SerialDescriptor = StringDescriptor.withName("NameToken")
override fun deserialize(decoder: Decoder): NameToken {
return decoder.decodeString().toName().first()!!
}
override fun serialize(encoder: Encoder, obj: NameToken) {
encoder.encodeString(obj.toString())
}
}

80
dataforge-io/src/commonMain/kotlin/hep/dataforge/io/serialization/serializationUtils.kt Normal file
View File

@ -0,0 +1,80 @@
package hep.dataforge.io.serialization
import kotlinx.serialization.CompositeDecoder
import kotlinx.serialization.Decoder
import kotlinx.serialization.KSerializer
import kotlinx.serialization.SerialDescriptor
import kotlinx.serialization.internal.*
/**
* A convenience builder for serial descriptors
*/
inline class SerialDescriptorBuilder(private val impl: SerialClassDescImpl) {
fun element(
name: String,
descriptor: SerialDescriptor?,
isOptional: Boolean = false,
vararg annotations: Annotation
) {
impl.addElement(name, isOptional)
descriptor?.let { impl.pushDescriptor(descriptor) }
annotations.forEach {
impl.pushAnnotation(it)
}
}
fun element(
name: String,
isOptional: Boolean = false,
vararg annotations: Annotation,
block: SerialDescriptorBuilder.() -> Unit
) {
impl.addElement(name, isOptional)
impl.pushDescriptor(SerialDescriptorBuilder(SerialClassDescImpl(name)).apply(block).build())
annotations.forEach {
impl.pushAnnotation(it)
}
}
fun boolean(name: String, isOptional: Boolean = false, vararg annotations: Annotation) =
element(name, BooleanDescriptor, isOptional, *annotations)
fun string(name: String, isOptional: Boolean = false, vararg annotations: Annotation) =
element(name, StringDescriptor, isOptional, *annotations)
fun int(name: String, isOptional: Boolean = false, vararg annotations: Annotation) =
element(name, IntDescriptor, isOptional, *annotations)
fun double(name: String, isOptional: Boolean = false, vararg annotations: Annotation) =
element(name, DoubleDescriptor, isOptional, *annotations)
fun float(name: String, isOptional: Boolean = false, vararg annotations: Annotation) =
element(name, FloatDescriptor, isOptional, *annotations)
fun long(name: String, isOptional: Boolean = false, vararg annotations: Annotation) =
element(name, LongDescriptor, isOptional, *annotations)
fun doubleArray(name: String, isOptional: Boolean = false, vararg annotations: Annotation) =
element(name, DoubleArraySerializer.descriptor, isOptional, *annotations)
inline fun <reified E : Enum<E>> enum(name: String, isOptional: Boolean = false, vararg annotations: Annotation) =
element(name, EnumSerializer(E::class).descriptor, isOptional, *annotations)
fun classAnnotation(a: Annotation) = impl.pushClassAnnotation(a)
fun build(): SerialDescriptor = impl
}
inline fun <reified T : Any> KSerializer<T>.descriptor(
name: String,
block: SerialDescriptorBuilder.() -> Unit
): SerialDescriptor =
SerialDescriptorBuilder(SerialClassDescImpl(name)).apply(block).build()
fun Decoder.decodeStructure(
desc: SerialDescriptor,
vararg typeParams: KSerializer<*> = emptyArray(),
block: CompositeDecoder.() -> Unit
) {
beginStructure(desc, *typeParams).apply(block).endStructure(desc)
}

46
dataforge-io/src/commonTest/kotlin/hep/dataforge/io/EnvelopeFormatTest.kt Normal file
View File

@ -0,0 +1,46 @@
package hep.dataforge.io
import kotlin.test.Test
import kotlin.test.assertEquals
class EnvelopeFormatTest {
val envelope = Envelope.invoke {
type = "test.format"
meta{
"d" put 22.2
}
data{
writeDouble(22.2)
}
}
@ExperimentalStdlibApi
@Test
fun testTaggedFormat(){
TaggedEnvelopeFormat.default.run {
val bytes = writeBytes(envelope)
println(bytes.decodeToString())
val res = readBytes(bytes)
assertEquals(envelope.meta,res.meta)
val double = res.data?.read {
readDouble()
}
assertEquals(22.2, double)
}
}
@Test
fun testTaglessFormat(){
TaglessEnvelopeFormat.default.run {
val bytes = writeBytes(envelope)
println(bytes.decodeToString())
val res = readBytes(bytes)
assertEquals(envelope.meta,res.meta)
val double = res.data?.read {
readDouble()
}
assertEquals(22.2, double)
}
}
}

20
dataforge-io/src/commonTest/kotlin/hep/dataforge/io/MetaFormatTest.kt
View File

@ -11,11 +11,11 @@ class MetaFormatTest {
@Test
fun testBinaryMetaFormat() {
val meta = buildMeta {
"a" to 22
"node" to {
"b" to "DDD"
"c" to 11.1
"array" to doubleArrayOf(1.0, 2.0, 3.0)
"a" put 22
"node" put {
"b" put "DDD"
"c" put 11.1
"array" put doubleArrayOf(1.0, 2.0, 3.0)
}
}
val bytes = meta.toBytes(BinaryMetaFormat)
@ -26,11 +26,11 @@ class MetaFormatTest {
@Test
fun testJsonMetaFormat() {
val meta = buildMeta {
"a" to 22
"node" to {
"b" to "DDD"
"c" to 11.1
"array" to doubleArrayOf(1.0, 2.0, 3.0)
"a" put 22
"node" put {
"b" put "DDD"
"c" put 11.1
"array" put doubleArrayOf(1.0, 2.0, 3.0)
}
}
val string = meta.toString(JsonMetaFormat)

38
dataforge-io/src/commonTest/kotlin/hep/dataforge/io/MetaSerializerTest.kt
View File

@ -1,7 +1,13 @@
package hep.dataforge.io
import hep.dataforge.io.serialization.MetaItemSerializer
import hep.dataforge.io.serialization.MetaSerializer
import hep.dataforge.io.serialization.NameSerializer
import hep.dataforge.meta.buildMeta
import hep.dataforge.names.toName
import kotlinx.io.charsets.Charsets
import kotlinx.io.core.String
import kotlinx.serialization.cbor.Cbor
import kotlinx.serialization.json.Json
import kotlin.test.Test
import kotlin.test.assertEquals
@ -10,11 +16,11 @@ class MetaSerializerTest {
@Test
fun testMetaSerialization() {
val meta = buildMeta {
"a" to 22
"node" to {
"b" to "DDD"
"c" to 11.1
"array" to doubleArrayOf(1.0, 2.0, 3.0)
"a" put 22
"node" put {
"b" put "DDD"
"c" put 11.1
"array" put doubleArrayOf(1.0, 2.0, 3.0)
}
}
@ -23,6 +29,23 @@ class MetaSerializerTest {
assertEquals(restored, meta)
}
@Test
fun testCborSerialization() {
val meta = buildMeta {
"a" put 22
"node" put {
"b" put "DDD"
"c" put 11.1
"array" put doubleArrayOf(1.0, 2.0, 3.0)
}
}
val bytes = Cbor.dump(MetaSerializer, meta)
println(String(bytes, charset = Charsets.ISO_8859_1))
val restored = Cbor.load(MetaSerializer, bytes)
assertEquals(restored, meta)
}
@Test
fun testNameSerialization() {
val name = "a.b.c".toName()
@ -30,4 +53,9 @@ class MetaSerializerTest {
val restored = Json.plain.parse(NameSerializer, string)
assertEquals(restored, name)
}
@Test
fun testMetaItemDescriptor(){
val descriptor = MetaItemSerializer.descriptor.getElementDescriptor(0)
}
}

22
dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileBinary.kt
View File

@ -1,21 +1,31 @@
package hep.dataforge.io
import kotlinx.io.core.ByteReadPacket
import kotlinx.io.core.Input
import kotlinx.io.core.buildPacket
import java.nio.channels.FileChannel
import java.nio.file.Files
import java.nio.file.Path
import java.nio.file.StandardOpenOption
import kotlin.math.min
@ExperimentalUnsignedTypes
class FileBinary(val path: Path, private val offset: UInt = 0u, size: ULong? = null) : RandomAccessBinary {
override val size: ULong = size ?: (Files.size(path).toULong() - offset).toULong()
override fun <R> read(from: UInt, size: UInt, block: Input.() -> R): R {
FileChannel.open(path, StandardOpenOption.READ).use {
val buffer = it.map(FileChannel.MapMode.READ_ONLY, (from + offset).toLong(), size.toLong())
return ByteReadPacket(buffer).block()
init {
if( size != null && Files.size(path) < offset.toLong() + size.toLong()){
error("Can't read binary from file. File is to short.")
}
}
}
override fun <R> read(from: UInt, size: UInt, block: Input.() -> R): R {
FileChannel.open(path, StandardOpenOption.READ).use {
val theSize: UInt = min(size, Files.size(path).toUInt() - offset)
val buffer = it.map(FileChannel.MapMode.READ_ONLY, (from + offset).toLong(), theSize.toLong())
return buildPacket { writeFully(buffer) }.block()
}
}
}
fun Path.asBinary(offset: UInt = 0u, size: ULong? = null): FileBinary = FileBinary(this, offset, size)

25
dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileEnvelope.kt
View File

@ -1,5 +1,6 @@
package hep.dataforge.io
import hep.dataforge.meta.EmptyMeta
import hep.dataforge.meta.Meta
import kotlinx.io.nio.asInput
import kotlinx.io.nio.asOutput
@ -14,7 +15,7 @@ class FileEnvelope internal constructor(val path: Path, val format: EnvelopeForm
init {
val input = Files.newByteChannel(path, StandardOpenOption.READ).asInput()
partialEnvelope = format.run { input.readPartial() }
partialEnvelope = format.run { input.use { it.readPartial()} }
}
override val meta: Meta get() = partialEnvelope.meta
@ -22,22 +23,30 @@ class FileEnvelope internal constructor(val path: Path, val format: EnvelopeForm
override val data: Binary? = FileBinary(path, partialEnvelope.dataOffset, partialEnvelope.dataSize)
}
fun Path.readEnvelope(format: EnvelopeFormat) = FileEnvelope(this, format)
fun IOPlugin.readEnvelopeFile(
path: Path,
formatFactory: EnvelopeFormatFactory = TaggedEnvelopeFormat,
formatMeta: Meta = EmptyMeta
): FileEnvelope {
val format = formatFactory(formatMeta, context)
return FileEnvelope(path, format)
}
fun Path.writeEnvelope(
fun IOPlugin.writeEnvelopeFile(
path: Path,
envelope: Envelope,
format: EnvelopeFormat = TaggedEnvelopeFormat,
metaFormat: MetaFormat = JsonMetaFormat
formatFactory: EnvelopeFormatFactory = TaggedEnvelopeFormat,
formatMeta: Meta = EmptyMeta
) {
val output = Files.newByteChannel(
this,
path,
StandardOpenOption.WRITE,
StandardOpenOption.CREATE,
StandardOpenOption.TRUNCATE_EXISTING
).asOutput()
with(format) {
output.writeEnvelope(envelope, metaFormat)
with(formatFactory(formatMeta, context)) {
output.writeObject(envelope)
}
}

51
dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/ioFormatsJVM.kt Normal file
View File

@ -0,0 +1,51 @@
package hep.dataforge.io
import hep.dataforge.descriptors.NodeDescriptor
import hep.dataforge.io.functions.FunctionServer
import hep.dataforge.io.functions.FunctionServer.Companion.FUNCTION_NAME_KEY
import hep.dataforge.io.functions.FunctionServer.Companion.INPUT_FORMAT_KEY
import hep.dataforge.io.functions.FunctionServer.Companion.OUTPUT_FORMAT_KEY
import hep.dataforge.io.functions.function
import hep.dataforge.meta.Meta
import hep.dataforge.meta.buildMeta
import hep.dataforge.names.Name
import kotlinx.io.nio.asOutput
import java.nio.file.Files
import java.nio.file.Path
import java.nio.file.StandardOpenOption
import kotlin.reflect.KClass
import kotlin.reflect.full.isSuperclassOf
inline fun <reified T : Any> IOPlugin.resolveIOFormat(): IOFormat<T>? {
return ioFormats.values.find { it.type.isSuperclassOf(T::class) } as IOFormat<T>?
}
fun IOPlugin.resolveIOFormatName(type: KClass<*>): Name {
return ioFormats.entries.find { it.value.type.isSuperclassOf(type) }?.key
?: error("Can't resolve IOFormat for type $type")
}
inline fun <reified T : Any, reified R : Any> IOPlugin.generateFunctionMeta(functionName: String): Meta = buildMeta {
FUNCTION_NAME_KEY put functionName
INPUT_FORMAT_KEY put resolveIOFormatName(T::class).toString()
OUTPUT_FORMAT_KEY put resolveIOFormatName(R::class).toString()
}
inline fun <reified T : Any, reified R : Any> FunctionServer.function(
functionName: String
): (suspend (T) -> R) {
val plugin = context.plugins.get<IOPlugin>() ?: error("IO plugin not loaded")
val meta = plugin.generateFunctionMeta<T, R>(functionName)
return function(meta)
}
/**
* Write meta to file in a given [format]
*/
fun Meta.write(path: Path, format: MetaFormat, descriptor: NodeDescriptor? = null) {
format.run {
Files.newByteChannel(path, StandardOpenOption.WRITE, StandardOpenOption.CREATE_NEW)
.asOutput()
.writeMeta(this@write, descriptor)
}
}

67
dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/tcp/EnvelopeClient.kt Normal file
View File

@ -0,0 +1,67 @@
package hep.dataforge.io.tcp
import hep.dataforge.context.Context
import hep.dataforge.context.ContextAware
import hep.dataforge.io.*
import hep.dataforge.meta.EmptyMeta
import hep.dataforge.meta.Meta
import kotlinx.coroutines.asCoroutineDispatcher
import kotlinx.coroutines.withContext
import kotlinx.io.streams.writePacket
import java.net.Socket
import java.util.concurrent.Executors
import kotlin.time.ExperimentalTime
@ExperimentalTime
class EnvelopeClient(
override val context: Context,
val host: String,
val port: Int,
formatFactory: EnvelopeFormatFactory = TaggedEnvelopeFormat,
formatMeta: Meta = EmptyMeta
) : Responder, ContextAware {
private val dispatcher = Executors.newSingleThreadExecutor().asCoroutineDispatcher()
private val format = formatFactory(formatMeta, context = context)
// private var socket: SocketChannel? = null
//
// private fun getSocket(): Socket {
// val socket = socket ?: Socket(host, port).also { this.socket = it }
// return if (socket.isConnected) {
// socket
// } else {
// Socket(host, port).also { this.socket = it }
// }
// }
suspend fun close() {
try {
respond(
Envelope.invoke {
type = EnvelopeServer.SHUTDOWN_ENVELOPE_TYPE
}
)
} catch (ex: Exception) {
logger.error { ex }
}
}
override suspend fun respond(request: Envelope): Envelope = withContext(dispatcher) {
//val address = InetSocketAddress(host,port)
val socket = Socket(host, port)
val input = socket.getInputStream().asInput()
val output = socket.getOutputStream()
format.run {
output.writePacket {
writeObject(request)
}
logger.debug { "Sent request with type ${request.type} to ${socket.remoteSocketAddress}" }
val res = input.readObject()
logger.debug { "Received response with type ${res.type} from ${socket.remoteSocketAddress}" }
return@withContext res
}
}
}

102
dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/tcp/EnvelopeServer.kt Normal file
View File

@ -0,0 +1,102 @@
package hep.dataforge.io.tcp
import hep.dataforge.context.Context
import hep.dataforge.context.ContextAware
import hep.dataforge.io.EnvelopeFormatFactory
import hep.dataforge.io.Responder
import hep.dataforge.io.TaggedEnvelopeFormat
import hep.dataforge.io.type
import hep.dataforge.meta.EmptyMeta
import hep.dataforge.meta.Meta
import kotlinx.coroutines.*
import kotlinx.io.streams.writePacket
import java.net.ServerSocket
import java.net.Socket
import kotlin.concurrent.thread
class EnvelopeServer(
override val context: Context,
val port: Int,
val responder: Responder,
val scope: CoroutineScope,
formatFactory: EnvelopeFormatFactory = TaggedEnvelopeFormat,
formatMeta: Meta = EmptyMeta
) : ContextAware {
private var job: Job? = null
private val format = formatFactory(formatMeta, context = context)
fun start() {
if (job == null) {
logger.info { "Starting envelope server on port $port" }
val job = scope.launch(Dispatchers.IO) {
val serverSocket = ServerSocket(port)
//TODO add handshake and format negotiation
while (isActive && !serverSocket.isClosed) {
val socket = serverSocket.accept()
logger.info { "Accepted connection from ${socket.remoteSocketAddress}" }
readSocket(socket)
}
}
}
}
fun stop() {
logger.info { "Stopping envelope server on port $port" }
job?.cancel()
job = null
}
// private fun CoroutineScope.readSocket(socket: Socket) {
// launch(Dispatchers.IO) {
// val input = socket.getInputStream().asInput()
// val output = socket.getOutputStream().asOutput()
// format.run {
// while (isActive && socket.isConnected) {
// val request = input.readThis()
// logger.debug { "Accepted request with type ${request.type} from ${socket.remoteSocketAddress}" }
// if (request.type == SHUTDOWN_ENVELOPE_TYPE) {
// //Echo shutdown command
// logger.info { "Accepted graceful shutdown signal from ${socket.inetAddress}" }
// socket.close()
// cancel("Graceful connection shutdown requested by client")
// }
// val response = responder.respond(request)
// output.writeThis(response)
// }
// }
// }
// }
private fun readSocket(socket: Socket) {
thread {
val input = socket.getInputStream().asInput()
val outputStream = socket.getOutputStream()
format.run {
while (socket.isConnected) {
val request = input.readObject()
logger.debug { "Accepted request with type ${request.type} from ${socket.remoteSocketAddress}" }
if (request.type == SHUTDOWN_ENVELOPE_TYPE) {
//Echo shutdown command
logger.info { "Accepted graceful shutdown signal from ${socket.inetAddress}" }
socket.close()
return@thread
// cancel("Graceful connection shutdown requested by client")
}
runBlocking {
val response = responder.respond(request)
outputStream.writePacket {
writeObject(response)
}
logger.debug { "Sent response with type ${response.type} to ${socket.remoteSocketAddress}" }
}
}
}
}
}
companion object {
const val SHUTDOWN_ENVELOPE_TYPE = "@shutdown"
}
}

33
dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/tcp/InputStreamAsInput.kt Normal file
View File

@ -0,0 +1,33 @@
package hep.dataforge.io.tcp
import kotlinx.io.core.AbstractInput
import kotlinx.io.core.Input
import kotlinx.io.core.IoBuffer
import kotlinx.io.core.IoBuffer.Companion.NoPool
import kotlinx.io.core.writePacket
import kotlinx.io.streams.readPacketAtMost
import java.io.InputStream
/**
* Modified version of InputStream to Input converter that supports waiting for input
*/
internal class InputStreamAsInput(
private val stream: InputStream
) : AbstractInput(pool = NoPool) {
override fun fill(): IoBuffer? {
val packet = stream.readPacketAtMost(4096)
return pool.borrow().apply {
resetForWrite(4096)
writePacket(packet)
}
}
override fun closeSource() {
stream.close()
}
}
fun InputStream.asInput(): Input =
InputStreamAsInput(this)

56
dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/FileBinaryTest.kt Normal file
View File

@ -0,0 +1,56 @@
package hep.dataforge.io
import hep.dataforge.context.Global
import java.nio.file.Files
import kotlin.test.Test
import kotlin.test.assertEquals
class FileBinaryTest {
val envelope = Envelope {
meta {
"a" put "AAA"
"b" put 22.2
}
dataType = "hep.dataforge.test"
dataID = "myData" // добавил только что
data {
writeDouble(16.7)
}
}
@Test
fun testSize() {
val binary = envelope.data
assertEquals(binary?.size?.toInt(), binary?.toBytes()?.size)
}
@Test
fun testFileData(){
val dataFile = Files.createTempFile("dataforge_test_bin", ".bin")
dataFile.toFile().writeText("This is my binary")
val envelopeFromFile = Envelope {
meta {
"a" put "AAA"
"b" put 22.2
}
dataType = "hep.dataforge.satellite"
dataID = "cellDepositTest" // добавил только что
data = dataFile.asBinary()
}
val binary = envelopeFromFile.data!!
println(binary.toBytes().size)
assertEquals(binary.size.toInt(), binary.toBytes().size)
}
@Test
fun testFileDataSizeRewriting() {
println(System.getProperty("user.dir"))
val tmpPath = Files.createTempFile("dataforge_test", ".df")
Global.io.writeEnvelopeFile(tmpPath, envelope)
val binary = Global.io.readEnvelopeFile(tmpPath).data!!
assertEquals(binary.size.toInt(), binary.toBytes().size)
}
}

31
dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/FileEnvelopeTest.kt Normal file
View File

@ -0,0 +1,31 @@
package hep.dataforge.io
import hep.dataforge.context.Global
import java.nio.file.Files
import kotlin.test.Test
import kotlin.test.assertTrue
class FileEnvelopeTest {
val envelope = Envelope {
meta {
"a" put "AAA"
"b" put 22.2
}
dataType = "hep.dataforge.test"
dataID = "myData" // добавил только что
data {
writeDouble(16.7)
}
}
@Test
fun testFileWriteRead() {
val tmpPath = Files.createTempFile("dataforge_test", ".df")
Global.io.writeEnvelopeFile(tmpPath,envelope)
println(tmpPath.toUri())
val restored: Envelope = Global.io.readEnvelopeFile(tmpPath)
assertTrue { envelope.contentEquals(restored) }
}
}

67
dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/tcp/EnvelopeServerTest.kt Normal file
View File

@ -0,0 +1,67 @@
package hep.dataforge.io.tcp
import hep.dataforge.context.Global
import hep.dataforge.io.Envelope
import hep.dataforge.io.Responder
import hep.dataforge.io.TaggedEnvelopeFormat
import hep.dataforge.io.writeBytes
import kotlinx.coroutines.GlobalScope
import kotlinx.coroutines.runBlocking
import org.junit.AfterClass
import org.junit.BeforeClass
import org.junit.Test
import kotlin.test.assertEquals
import kotlin.time.ExperimentalTime
@ExperimentalStdlibApi
object EchoResponder : Responder {
override suspend fun respond(request: Envelope): Envelope {
val string = TaggedEnvelopeFormat().run { writeBytes(request).decodeToString() }
println("ECHO:")
println(string)
return request
}
}
@ExperimentalTime
@ExperimentalStdlibApi
class EnvelopeServerTest {
companion object {
@JvmStatic
val echoEnvelopeServer = EnvelopeServer(Global, 7778, EchoResponder, GlobalScope)
@BeforeClass
@JvmStatic
fun start() {
echoEnvelopeServer.start()
}
@AfterClass
@JvmStatic
fun close() {
echoEnvelopeServer.stop()
}
}
@Test
fun doEchoTest() {
val request = Envelope.invoke {
type = "test.echo"
meta {
"test.value" put 22
}
data {
writeDouble(22.7)
}
}
val client = EnvelopeClient(Global, host = "localhost", port = 7778)
runBlocking {
val response = client.respond(request)
assertEquals(request.meta, response.meta)
// assertEquals(request.data?.toBytes()?.decodeToString(), response.data?.toBytes()?.decodeToString())
client.close()
}
}
}

22
dataforge-meta/src/commonMain/kotlin/hep/dataforge/descriptors/ItemDescriptor.kt
View File

@ -32,13 +32,11 @@ sealed class ItemDescriptor(override val config: Config) : Specific {
var info: String? by string()
/**
* A list of tags for this item. Tags used to customize item usage
* Additional attributes of an item. For example validation and widget parameters
*
* @return
*/
var tags: List<String> by value { value ->
value?.list?.map { it.string } ?: emptyList()
}
var attributes by node()
/**
* True if the item is required
@ -54,7 +52,7 @@ sealed class ItemDescriptor(override val config: Config) : Specific {
*
* @author Alexander Nozik
*/
class NodeDescriptor(config: Config) : ItemDescriptor(config){
class NodeDescriptor(config: Config) : ItemDescriptor(config) {
/**
* True if the node is required
@ -68,18 +66,18 @@ class NodeDescriptor(config: Config) : ItemDescriptor(config){
*
* @return
*/
var default: Meta? by node()
var default: Config? by node()
/**
* The list of value descriptors
*/
val values: Map<String, ValueDescriptor>
get() = config.getAll(VALUE_KEY.toName()).entries.associate { (name, node) ->
get() = config.getIndexed(VALUE_KEY.toName()).entries.associate { (name, node) ->
name to ValueDescriptor.wrap(node.node ?: error("Value descriptor must be a node"))
}
fun value(name: String, descriptor: ValueDescriptor) {
if(items.keys.contains(name)) error("The key $name already exists in descriptor")
if (items.keys.contains(name)) error("The key $name already exists in descriptor")
val token = NameToken(VALUE_KEY, name)
config[token] = descriptor.config
}
@ -95,13 +93,13 @@ class NodeDescriptor(config: Config) : ItemDescriptor(config){
* The map of children node descriptors
*/
val nodes: Map<String, NodeDescriptor>
get() = config.getAll(NODE_KEY.toName()).entries.associate { (name, node) ->
get() = config.getIndexed(NODE_KEY.toName()).entries.associate { (name, node) ->
name to wrap(node.node ?: error("Node descriptor must be a node"))
}
fun node(name: String, descriptor: NodeDescriptor) {
if(items.keys.contains(name)) error("The key $name already exists in descriptor")
if (items.keys.contains(name)) error("The key $name already exists in descriptor")
val token = NameToken(NODE_KEY, name)
config[token] = descriptor.config
}
@ -117,7 +115,7 @@ class NodeDescriptor(config: Config) : ItemDescriptor(config){
companion object : Specification<NodeDescriptor> {
// const val ITEM_KEY = "item"
// const val ITEM_KEY = "item"
const val NODE_KEY = "node"
const val VALUE_KEY = "value"
@ -135,7 +133,7 @@ class NodeDescriptor(config: Config) : ItemDescriptor(config){
*
* @author Alexander Nozik
*/
class ValueDescriptor(config: Config) : ItemDescriptor(config){
class ValueDescriptor(config: Config) : ItemDescriptor(config) {
/**

4
dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Laminate.kt
View File

@ -4,8 +4,6 @@ import hep.dataforge.names.NameToken
/**
* A meta laminate consisting of multiple immutable meta layers. For mutable front layer, use [Styled].
*
*
*/
class Laminate(layers: List<Meta>) : MetaBase() {
@ -63,7 +61,7 @@ class Laminate(layers: List<Meta>) : MetaBase() {
}
else -> map {
when (it) {
is MetaItem.ValueItem -> MetaItem.NodeItem(buildMeta { Meta.VALUE_KEY to it.value })
is MetaItem.ValueItem -> MetaItem.NodeItem(buildMeta { Meta.VALUE_KEY put it.value })
is MetaItem.NodeItem -> it
}
}.merge()

73
dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Meta.kt
View File

@ -59,6 +59,8 @@ interface Meta : MetaRepr {
* A key for single value node
*/
const val VALUE_KEY = "@value"
val empty: EmptyMeta = EmptyMeta
}
}
@ -78,27 +80,6 @@ operator fun Meta?.get(name: Name): MetaItem<*>? {
operator fun Meta?.get(token: NameToken): MetaItem<*>? = this?.items?.get(token)
operator fun Meta?.get(key: String): MetaItem<*>? = get(key.toName())
/**
* Get all items matching given name.
*/
fun Meta.getAll(name: Name): Map<String, MetaItem<*>> {
val root = when (name.length) {
0 -> error("Can't use empty name for that")
1 -> this
else -> (this[name.cutLast()] as? NodeItem<*>)?.node
}
val (body, index) = name.last()!!
val regex = index.toRegex()
return root?.items
?.filter { it.key.body == body && (index.isEmpty() || regex.matches(it.key.index)) }
?.mapKeys { it.key.index }
?: emptyMap()
}
fun Meta.getAll(name: String): Map<String, MetaItem<*>> = getAll(name.toName())
/**
* Get a sequence of [Name]-[Value] pairs
*/
@ -136,27 +117,6 @@ interface MetaNode<M : MetaNode<M>> : Meta {
override val items: Map<NameToken, MetaItem<M>>
}
/**
* Get all items matching given name.
*/
fun <M : MetaNode<M>> M.getAll(name: Name): Map<String, MetaItem<M>> {
val root: MetaNode<M>? = when (name.length) {
0 -> error("Can't use empty name for that")
1 -> this
else -> (this[name.cutLast()] as? NodeItem<M>)?.node
}
val (body, index) = name.last()!!
val regex = index.toRegex()
return root?.items
?.filter { it.key.body == body && (index.isEmpty() || regex.matches(it.key.index)) }
?.mapKeys { it.key.index }
?: emptyMap()
}
fun <M : MetaNode<M>> M.getAll(name: String): Map<String, MetaItem<M>> = getAll(name.toName())
operator fun <M : MetaNode<M>> MetaNode<M>?.get(name: Name): MetaItem<M>? {
if (this == null) return null
return name.first()?.let { token ->
@ -183,20 +143,15 @@ operator fun <M : MetaNode<M>> MetaNode<M>?.get(key: NameToken): MetaItem<M>? =
/**
* Equals, hashcode and to string for any meta
*/
abstract class MetaBase: Meta{
abstract class MetaBase : Meta {
override fun equals(other: Any?): Boolean = if(other is Meta) {
override fun equals(other: Any?): Boolean = if (other is Meta) {
this.items == other.items
// val items = items
// val otherItems = other.items
// (items.keys == otherItems.keys) && items.keys.all {
// items[it] == otherItems[it]
// }
} else {
false
}
override fun hashCode(): Int = items.hashCode()
override fun hashCode(): Int = items.hashCode()
override fun toString(): String = items.toString()
}
@ -232,8 +187,7 @@ object EmptyMeta : MetaBase() {
/**
* Unsafe methods to access values and nodes directly from [MetaItem]
*/
val MetaItem<*>?.value
val MetaItem<*>?.value: Value?
get() = (this as? ValueItem)?.value
?: (this?.node?.get(VALUE_KEY) as? ValueItem)?.value
@ -247,7 +201,7 @@ val MetaItem<*>?.long get() = number?.toLong()
val MetaItem<*>?.short get() = number?.toShort()
inline fun <reified E : Enum<E>> MetaItem<*>?.enum() = if (this is ValueItem && this.value is EnumValue<*>) {
this.value as E
this.value.value as E
} else {
string?.let { enumValueOf<E>(it) }
}
@ -257,17 +211,8 @@ val MetaItem<*>?.stringList get() = value?.list?.map { it.string } ?: emptyList(
val <M : Meta> MetaItem<M>?.node: M?
get() = when (this) {
null -> null
is ValueItem -> error("Trying to interpret value meta item as node item")
is ValueItem -> null//error("Trying to interpret value meta item as node item")
is NodeItem -> node
}
/**
* Generic meta-holder object
*/
interface Metoid {
val meta: Meta
}
fun Value.toMeta() = buildMeta { Meta.VALUE_KEY to this }
fun Meta.isEmpty() = this === EmptyMeta || this.items.isEmpty()
fun Meta.isEmpty() = this === EmptyMeta || this.items.isEmpty()

105
dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MetaBuilder.kt
View File

@ -2,50 +2,118 @@ package hep.dataforge.meta
import hep.dataforge.names.Name
import hep.dataforge.names.asName
import hep.dataforge.values.EnumValue
import hep.dataforge.values.Value
import hep.dataforge.values.asValue
import kotlin.jvm.JvmName
/**
* DSL builder for meta. Is not intended to store mutable state
*/
@DFBuilder
class MetaBuilder : AbstractMutableMeta<MetaBuilder>() {
override fun wrapNode(meta: Meta): MetaBuilder = if (meta is MetaBuilder) meta else meta.builder()
override fun empty(): MetaBuilder = MetaBuilder()
infix fun String.to(value: Any) {
if (value is Meta) {
this@MetaBuilder[this] = value
}
this@MetaBuilder[this] = Value.of(value)
infix fun String.put(value: Value?) {
set(this, value)
}
infix fun String.to(meta: Meta) {
infix fun String.put(string: String?) {
set(this, string?.asValue())
}
infix fun String.put(number: Number?) {
set(this, number?.asValue())
}
infix fun String.put(boolean: Boolean?) {
set(this, boolean?.asValue())
}
infix fun String.put(enum: Enum<*>) {
set(this, EnumValue(enum))
}
@JvmName("putValues")
infix fun String.put(iterable: Iterable<Value>) {
set(this, iterable.asValue())
}
@JvmName("putNumbers")
infix fun String.put(iterable: Iterable<Number>) {
set(this, iterable.map { it.asValue() }.asValue())
}
@JvmName("putStrings")
infix fun String.put(iterable: Iterable<String>) {
set(this, iterable.map { it.asValue() }.asValue())
}
infix fun String.put(array: DoubleArray) {
set(this, array.asValue())
}
infix fun String.putValue(any: Any?) {
set(this, Value.of(any))
}
infix fun String.put(meta: Meta?) {
this@MetaBuilder[this] = meta
}
infix fun String.to(value: Iterable<Meta>) {
infix fun String.put(repr: MetaRepr?) {
set(this, repr?.toMeta())
}
@JvmName("putMetas")
infix fun String.put(value: Iterable<Meta>) {
this@MetaBuilder[this] = value.toList()
}
infix fun String.to(metaBuilder: MetaBuilder.() -> Unit) {
infix fun String.put(metaBuilder: MetaBuilder.() -> Unit) {
this@MetaBuilder[this] = MetaBuilder().apply(metaBuilder)
}
infix fun Name.to(value: Any) {
if (value is Meta) {
this@MetaBuilder[this] = value
}
this@MetaBuilder[this] = Value.of(value)
infix fun Name.put(value: Value?) {
set(this, value)
}
infix fun Name.to(meta: Meta) {
infix fun Name.put(string: String?) {
set(this, string?.asValue())
}
infix fun Name.put(number: Number?) {
set(this, number?.asValue())
}
infix fun Name.put(boolean: Boolean?) {
set(this, boolean?.asValue())
}
infix fun Name.put(enum: Enum<*>) {
set(this, EnumValue(enum))
}
@JvmName("putValues")
infix fun Name.put(iterable: Iterable<Value>) {
set(this, iterable.asValue())
}
infix fun Name.put(meta: Meta?) {
this@MetaBuilder[this] = meta
}
infix fun Name.to(value: Iterable<Meta>) {
infix fun Name.put(repr: MetaRepr?) {
set(this, repr?.toMeta())
}
@JvmName("putMetas")
infix fun Name.put(value: Iterable<Meta>) {
this@MetaBuilder[this] = value.toList()
}
infix fun Name.to(metaBuilder: MetaBuilder.() -> Unit) {
infix fun Name.put(metaBuilder: MetaBuilder.() -> Unit) {
this@MetaBuilder[this] = MetaBuilder().apply(metaBuilder)
}
}
@ -65,6 +133,11 @@ fun Meta.builder(): MetaBuilder {
}
}
/**
* Create a deep copy of this meta and apply builder to it
*/
fun Meta.edit(builder: MetaBuilder.() -> Unit): MetaBuilder = builder().apply(builder)
/**
* Build a [MetaBuilder] using given transformation
*/

2
dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MutableMeta.kt
View File

@ -164,7 +164,7 @@ fun MutableMeta<*>.append(name: Name, value: Any?) {
if (newIndex.isNotEmpty()) {
set(name, value)
} else {
val index = (getAll(name).keys.mapNotNull { it.toIntOrNull() }.max() ?: -1) + 1
val index = (getIndexed(name).keys.mapNotNull { it.toIntOrNull() }.max() ?: -1) + 1
set(name.withIndex(index.toString()), value)
}
}

12
dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Specific.kt
View File

@ -1,5 +1,8 @@
package hep.dataforge.meta
import hep.dataforge.names.Name
import kotlin.jvm.JvmName
/**
* Marker interface for classes with specifications
*/
@ -61,5 +64,10 @@ fun <C : Specific, S : Specification<C>> S.createStyle(action: C.() -> Unit): Me
fun <C : Specific> Specific.spec(
spec: Specification<C>,
key: String? = null
): MutableMorphDelegate<Config, C> = MutableMorphDelegate(config, key) { spec.wrap(it) }
key: Name? = null
): MutableMorphDelegate<Config, C> = MutableMorphDelegate(config, key) { spec.wrap(it) }
fun <T: Specific> MetaItem<*>.spec(spec: Specification<T>): T? = node?.let { spec.wrap(it)}
@JvmName("configSpec")
fun <T: Specific> MetaItem<Config>.spec(spec: Specification<T>): T? = node?.let { spec.wrap(it)}

11
dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/annotations.kt Normal file
View File

@ -0,0 +1,11 @@
package hep.dataforge.meta
/**
* General marker for dataforge builders
*/
@DslMarker
annotation class DFBuilder
@Experimental(level = Experimental.Level.WARNING)
@Retention(AnnotationRetention.BINARY)
annotation class DFExperimental

63
dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/configDelegates.kt
View File

@ -1,5 +1,6 @@
package hep.dataforge.meta
import hep.dataforge.names.Name
import hep.dataforge.values.DoubleArrayValue
import hep.dataforge.values.Null
import hep.dataforge.values.Value
@ -11,126 +12,126 @@ import kotlin.jvm.JvmName
/**
* A property delegate that uses custom key
*/
fun Configurable.value(default: Any = Null, key: String? = null): MutableValueDelegate<Config> =
fun Configurable.value(default: Any = Null, key: Name? = null): MutableValueDelegate<Config> =
MutableValueDelegate(config, key, Value.of(default))
fun <T> Configurable.value(
default: T? = null,
key: String? = null,
key: Name? = null,
writer: (T) -> Value = { Value.of(it) },
reader: (Value?) -> T
): ReadWriteDelegateWrapper<Value?, T> =
MutableValueDelegate(config, key, default?.let { Value.of(it) }).transform(reader = reader, writer = writer)
fun Configurable.string(default: String? = null, key: String? = null): MutableStringDelegate<Config> =
fun Configurable.string(default: String? = null, key: Name? = null): MutableStringDelegate<Config> =
MutableStringDelegate(config, key, default)
fun Configurable.boolean(default: Boolean? = null, key: String? = null): MutableBooleanDelegate<Config> =
fun Configurable.boolean(default: Boolean? = null, key: Name? = null): MutableBooleanDelegate<Config> =
MutableBooleanDelegate(config, key, default)
fun Configurable.number(default: Number? = null, key: String? = null): MutableNumberDelegate<Config> =
fun Configurable.number(default: Number? = null, key: Name? = null): MutableNumberDelegate<Config> =
MutableNumberDelegate(config, key, default)
/* Number delegates*/
fun Configurable.int(default: Int? = null, key: String? = null) =
fun Configurable.int(default: Int? = null, key: Name? = null) =
number(default, key).int
fun Configurable.double(default: Double? = null, key: String? = null) =
fun Configurable.double(default: Double? = null, key: Name? = null) =
number(default, key).double
fun Configurable.long(default: Long? = null, key: String? = null) =
fun Configurable.long(default: Long? = null, key: Name? = null) =
number(default, key).long
fun Configurable.short(default: Short? = null, key: String? = null) =
fun Configurable.short(default: Short? = null, key: Name? = null) =
number(default, key).short
fun Configurable.float(default: Float? = null, key: String? = null) =
fun Configurable.float(default: Float? = null, key: Name? = null) =
number(default, key).float
@JvmName("safeString")
fun Configurable.string(default: String, key: String? = null) =
fun Configurable.string(default: String, key: Name? = null) =
MutableSafeStringDelegate(config, key) { default }
@JvmName("safeBoolean")
fun Configurable.boolean(default: Boolean, key: String? = null) =
fun Configurable.boolean(default: Boolean, key: Name? = null) =
MutableSafeBooleanDelegate(config, key) { default }
@JvmName("safeNumber")
fun Configurable.number(default: Number, key: String? = null) =
fun Configurable.number(default: Number, key: Name? = null) =
MutableSafeNumberDelegate(config, key) { default }
@JvmName("safeString")
fun Configurable.string(key: String? = null, default: () -> String) =
fun Configurable.string(key: Name? = null, default: () -> String) =
MutableSafeStringDelegate(config, key, default)
@JvmName("safeBoolean")
fun Configurable.boolean(key: String? = null, default: () -> Boolean) =
fun Configurable.boolean(key: Name? = null, default: () -> Boolean) =
MutableSafeBooleanDelegate(config, key, default)
@JvmName("safeNumber")
fun Configurable.number(key: String? = null, default: () -> Number) =
fun Configurable.number(key: Name? = null, default: () -> Number) =
MutableSafeNumberDelegate(config, key, default)
/* Safe number delegates*/
@JvmName("safeInt")
fun Configurable.int(default: Int, key: String? = null) =
fun Configurable.int(default: Int, key: Name? = null) =
number(default, key).int
@JvmName("safeDouble")
fun Configurable.double(default: Double, key: String? = null) =
fun Configurable.double(default: Double, key: Name? = null) =
number(default, key).double
@JvmName("safeLong")
fun Configurable.long(default: Long, key: String? = null) =
fun Configurable.long(default: Long, key: Name? = null) =
number(default, key).long
@JvmName("safeShort")
fun Configurable.short(default: Short, key: String? = null) =
fun Configurable.short(default: Short, key: Name? = null) =
number(default, key).short
@JvmName("safeFloat")
fun Configurable.float(default: Float, key: String? = null) =
fun Configurable.float(default: Float, key: Name? = null) =
number(default, key).float
/**
* Enum delegate
*/
inline fun <reified E : Enum<E>> Configurable.enum(default: E, key: String? = null) =
inline fun <reified E : Enum<E>> Configurable.enum(default: E, key: Name? = null) =
MutableSafeEnumvDelegate(config, key, default) { enumValueOf(it) }
/* Node delegates */
fun Configurable.node(key: String? = null) = MutableNodeDelegate(config, key)
fun Configurable.node(key: Name? = null): MutableNodeDelegate<Config> = MutableNodeDelegate(config, key)
fun <T : Specific> Configurable.spec(spec: Specification<T>, key: String? = null) =
fun <T : Specific> Configurable.spec(spec: Specification<T>, key: Name? = null) =
MutableMorphDelegate(config, key) { spec.wrap(it) }
fun <T : Specific> Configurable.spec(builder: (Config) -> T, key: String? = null) =
fun <T : Specific> Configurable.spec(builder: (Config) -> T, key: Name? = null) =
MutableMorphDelegate(config, key) { specification(builder).wrap(it) }
/*
* Extra delegates for special cases
*/
fun Configurable.stringList(key: String? = null): ReadWriteDelegateWrapper<Value?, List<String>> =
value(emptyList(), key) { it?.list?.map { value -> value.string } ?: emptyList() }
fun Configurable.stringList(vararg strings: String, key: Name? = null): ReadWriteDelegateWrapper<Value?, List<String>> =
value(strings.asList(), key) { it?.list?.map { value -> value.string } ?: emptyList() }
fun Configurable.numberList(key: String? = null): ReadWriteDelegateWrapper<Value?, List<Number>> =
value(emptyList(), key) { it?.list?.map { value -> value.number } ?: emptyList() }
fun Configurable.numberList(vararg numbers: Number, key: Name? = null): ReadWriteDelegateWrapper<Value?, List<Number>> =
value(numbers.asList(), key) { it?.list?.map { value -> value.number } ?: emptyList() }
/**
* A special delegate for double arrays
*/
fun Configurable.doubleArray(key: String? = null): ReadWriteDelegateWrapper<Value?, DoubleArray> =
fun Configurable.doubleArray(key: Name? = null): ReadWriteDelegateWrapper<Value?, DoubleArray> =
value(doubleArrayOf(), key) {
(it as? DoubleArrayValue)?.value
?: it?.list?.map { value -> value.number.toDouble() }?.toDoubleArray()
?: doubleArrayOf()
}
fun <T : Configurable> Configurable.child(key: String? = null, converter: (Meta) -> T) =
fun <T : Configurable> Configurable.node(key: Name? = null, converter: (Meta) -> T) =
MutableMorphDelegate(config, key, converter)

39
dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/mapMeta.kt Normal file
View File

@ -0,0 +1,39 @@
package hep.dataforge.meta
import hep.dataforge.descriptors.NodeDescriptor
import hep.dataforge.values.Value
///**
// * Find all elements with given body
// */
//private fun Meta.byBody(body: String): Map<String, MetaItem<*>> =
// items.filter { it.key.body == body }.mapKeys { it.key.index }
//
//private fun Meta.distinctNames() = items.keys.map { it.body }.distinct()
/**
* Convert meta to map of maps
*/
@DFExperimental
fun Meta.toMap(descriptor: NodeDescriptor? = null): Map<String, Any?> {
return items.entries.associate { (token, item) ->
token.toString() to when (item) {
is MetaItem.NodeItem -> item.node.toMap()
is MetaItem.ValueItem -> item.value.value
}
}
}
/**
* Convert map of maps to meta
*/
@DFExperimental
fun Map<String, Any?>.toMeta(descriptor: NodeDescriptor? = null): Meta = buildMeta {
entries.forEach { (key, value) ->
@Suppress("UNCHECKED_CAST")
when (value) {
is Map<*, *> -> setNode(key, (value as Map<String, Any?>).toMeta())
else -> setValue(key, Value.of(value))
}
}
}

116
dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/metaDelegates.kt
View File

@ -1,5 +1,7 @@
package hep.dataforge.meta
import hep.dataforge.names.Name
import hep.dataforge.names.asName
import hep.dataforge.values.Null
import hep.dataforge.values.Value
import hep.dataforge.values.asValue
@ -26,15 +28,21 @@ class StringDelegate(val meta: Meta, private val key: String? = null, private va
}
}
class BooleanDelegate(val meta: Meta, private val key: String? = null, private val default: Boolean? = null) :
ReadOnlyProperty<Metoid, Boolean?> {
override fun getValue(thisRef: Metoid, property: KProperty<*>): Boolean? {
class BooleanDelegate(
val meta: Meta,
private val key: String? = null,
private val default: Boolean? = null
) : ReadOnlyProperty<Any?, Boolean?> {
override fun getValue(thisRef: Any?, property: KProperty<*>): Boolean? {
return meta[key ?: property.name]?.boolean ?: default
}
}
class NumberDelegate(val meta: Meta, private val key: String? = null, private val default: Number? = null) :
ReadOnlyProperty<Any?, Number?> {
class NumberDelegate(
val meta: Meta,
private val key: String? = null,
private val default: Number? = null
) : ReadOnlyProperty<Any?, Number?> {
override fun getValue(thisRef: Any?, property: KProperty<*>): Number? {
return meta[key ?: property.name]?.number ?: default
}
@ -136,7 +144,7 @@ fun Meta.boolean(default: Boolean? = null, key: String? = null) = BooleanDelegat
fun Meta.number(default: Number? = null, key: String? = null) = NumberDelegate(this, key, default)
fun Meta.child(key: String? = null) = ChildDelegate(this, key) { it }
fun Meta.node(key: String? = null) = ChildDelegate(this, key) { it }
@JvmName("safeString")
fun Meta.string(default: String, key: String? = null) =
@ -166,22 +174,19 @@ fun Meta.number(key: String? = null, default: () -> Number) =
inline fun <reified E : Enum<E>> Meta.enum(default: E, key: String? = null) =
SafeEnumDelegate(this, key, default) { enumValueOf(it) }
fun <T : Metoid> Metoid.child(key: String? = null, converter: (Meta) -> T) = ChildDelegate(meta, key, converter)
/* Read-write delegates */
class MutableValueDelegate<M : MutableMeta<M>>(
val meta: M,
private val key: String? = null,
private val key: Name? = null,
private val default: Value? = null
) : ReadWriteProperty<Any?, Value?> {
override fun getValue(thisRef: Any?, property: KProperty<*>): Value? {
return meta[key ?: property.name]?.value ?: default
return meta[key ?: property.name.asName()]?.value ?: default
}
override fun setValue(thisRef: Any?, property: KProperty<*>, value: Value?) {
val name = key ?: property.name
val name = key ?: property.name.asName()
if (value == null) {
meta.remove(name)
} else {
@ -195,15 +200,15 @@ class MutableValueDelegate<M : MutableMeta<M>>(
class MutableStringDelegate<M : MutableMeta<M>>(
val meta: M,
private val key: String? = null,
private val key: Name? = null,
private val default: String? = null
) : ReadWriteProperty<Any?, String?> {
override fun getValue(thisRef: Any?, property: KProperty<*>): String? {
return meta[key ?: property.name]?.string ?: default
return meta[key ?: property.name.asName()]?.string ?: default
}
override fun setValue(thisRef: Any?, property: KProperty<*>, value: String?) {
val name = key ?: property.name
val name = key ?: property.name.asName()
if (value == null) {
meta.remove(name)
} else {
@ -214,15 +219,15 @@ class MutableStringDelegate<M : MutableMeta<M>>(
class MutableBooleanDelegate<M : MutableMeta<M>>(
val meta: M,
private val key: String? = null,
private val key: Name? = null,
private val default: Boolean? = null
) : ReadWriteProperty<Any?, Boolean?> {
override fun getValue(thisRef: Any?, property: KProperty<*>): Boolean? {
return meta[key ?: property.name]?.boolean ?: default
return meta[key ?: property.name.asName()]?.boolean ?: default
}
override fun setValue(thisRef: Any?, property: KProperty<*>, value: Boolean?) {
val name = key ?: property.name
val name = key ?: property.name.asName()
if (value == null) {
meta.remove(name)
} else {
@ -233,15 +238,15 @@ class MutableBooleanDelegate<M : MutableMeta<M>>(
class MutableNumberDelegate<M : MutableMeta<M>>(
val meta: M,
private val key: String? = null,
private val key: Name? = null,
private val default: Number? = null
) : ReadWriteProperty<Any?, Number?> {
override fun getValue(thisRef: Any?, property: KProperty<*>): Number? {
return meta[key ?: property.name]?.number ?: default
return meta[key ?: property.name.asName()]?.number ?: default
}
override fun setValue(thisRef: Any?, property: KProperty<*>, value: Number?) {
val name = key ?: property.name
val name = key ?: property.name.asName()
if (value == null) {
meta.remove(name)
} else {
@ -260,52 +265,52 @@ class MutableNumberDelegate<M : MutableMeta<M>>(
class MutableSafeStringDelegate<M : MutableMeta<M>>(
val meta: M,
private val key: String? = null,
private val key: Name? = null,
default: () -> String
) : ReadWriteProperty<Any?, String> {
private val default: String by lazy(default)
override fun getValue(thisRef: Any?, property: KProperty<*>): String {
return meta[key ?: property.name]?.string ?: default
return meta[key ?: property.name.asName()]?.string ?: default
}
override fun setValue(thisRef: Any?, property: KProperty<*>, value: String) {
meta.setValue(key ?: property.name, value.asValue())
meta.setValue(key ?: property.name.asName(), value.asValue())
}
}
class MutableSafeBooleanDelegate<M : MutableMeta<M>>(
val meta: M,
private val key: String? = null,
private val key: Name? = null,
default: () -> Boolean
) : ReadWriteProperty<Any?, Boolean> {
private val default: Boolean by lazy(default)
override fun getValue(thisRef: Any?, property: KProperty<*>): Boolean {
return meta[key ?: property.name]?.boolean ?: default
return meta[key ?: property.name.asName()]?.boolean ?: default
}
override fun setValue(thisRef: Any?, property: KProperty<*>, value: Boolean) {
meta.setValue(key ?: property.name, value.asValue())
meta.setValue(key ?: property.name.asName(), value.asValue())
}
}
class MutableSafeNumberDelegate<M : MutableMeta<M>>(
val meta: M,
private val key: String? = null,
private val key: Name? = null,
default: () -> Number
) : ReadWriteProperty<Any?, Number> {
private val default: Number by lazy(default)
override fun getValue(thisRef: Any?, property: KProperty<*>): Number {
return meta[key ?: property.name]?.number ?: default
return meta[key ?: property.name.asName()]?.number ?: default
}
override fun setValue(thisRef: Any?, property: KProperty<*>, value: Number) {
meta.setValue(key ?: property.name, value.asValue())
meta.setValue(key ?: property.name.asName(), value.asValue())
}
val double get() = ReadWriteDelegateWrapper(this, reader = { it.toDouble() }, writer = { it })
@ -317,16 +322,16 @@ class MutableSafeNumberDelegate<M : MutableMeta<M>>(
class MutableSafeEnumvDelegate<M : MutableMeta<M>, E : Enum<E>>(
val meta: M,
private val key: String? = null,
private val key: Name? = null,
private val default: E,
private val resolver: (String) -> E
) : ReadWriteProperty<Any?, E> {
override fun getValue(thisRef: Any?, property: KProperty<*>): E {
return (meta[key ?: property.name]?.string)?.let { resolver(it) } ?: default
return (meta[key ?: property.name.asName()]?.string)?.let { resolver(it) } ?: default
}
override fun setValue(thisRef: Any?, property: KProperty<*>, value: E) {
meta.setValue(key ?: property.name, value.name.asValue())
meta.setValue(key ?: property.name.asName(), value.name.asValue())
}
}
@ -334,31 +339,31 @@ class MutableSafeEnumvDelegate<M : MutableMeta<M>, E : Enum<E>>(
class MutableNodeDelegate<M : MutableMeta<M>>(
val meta: M,
private val key: String? = null
) : ReadWriteProperty<Any?, Meta?> {
override fun getValue(thisRef: Any?, property: KProperty<*>): Meta? {
return meta[key ?: property.name]?.node
private val key: Name? = null
) : ReadWriteProperty<Any?, M?> {
override fun getValue(thisRef: Any?, property: KProperty<*>): M? {
return meta[key ?: property.name.asName()]?.node
}
override fun setValue(thisRef: Any?, property: KProperty<*>, value: Meta?) {
meta[key ?: property.name] = value
override fun setValue(thisRef: Any?, property: KProperty<*>, value: M?) {
meta[key ?: property.name.asName()] = value
}
}
class MutableMorphDelegate<M : MutableMeta<M>, T : Configurable>(
val meta: M,
private val key: String? = null,
private val key: Name? = null,
private val converter: (Meta) -> T
) : ReadWriteProperty<Any?, T?> {
override fun getValue(thisRef: Any?, property: KProperty<*>): T? {
return meta[key ?: property.name]?.node?.let(converter)
return meta[key ?: property.name.asName()]?.node?.let(converter)
}
override fun setValue(thisRef: Any?, property: KProperty<*>, value: T?) {
if (value == null) {
meta.remove(key ?: property.name)
meta.remove(key ?: property.name.asName())
} else {
meta[key ?: property.name] = value.config
meta[key ?: property.name.asName()] = value.config
}
}
}
@ -383,44 +388,45 @@ class ReadWriteDelegateWrapper<T, R>(
/**
* A property delegate that uses custom key
*/
fun <M : MutableMeta<M>> M.value(default: Value = Null, key: String? = null) =
fun <M : MutableMeta<M>> M.value(default: Value = Null, key: Name? = null) =
MutableValueDelegate(this, key, default)
fun <M : MutableMeta<M>> M.string(default: String? = null, key: String? = null) =
fun <M : MutableMeta<M>> M.string(default: String? = null, key: Name? = null) =
MutableStringDelegate(this, key, default)
fun <M : MutableMeta<M>> M.boolean(default: Boolean? = null, key: String? = null) =
fun <M : MutableMeta<M>> M.boolean(default: Boolean? = null, key: Name? = null) =
MutableBooleanDelegate(this, key, default)
fun <M : MutableMeta<M>> M.number(default: Number? = null, key: String? = null) =
fun <M : MutableMeta<M>> M.number(default: Number? = null, key: Name? = null) =
MutableNumberDelegate(this, key, default)
fun <M : MutableMeta<M>> M.node(key: String? = null) = MutableNodeDelegate(this, key)
fun <M : MutableMeta<M>> M.node(key: Name? = null) =
MutableNodeDelegate(this, key)
@JvmName("safeString")
fun <M : MutableMeta<M>> M.string(default: String, key: String? = null) =
fun <M : MutableMeta<M>> M.string(default: String, key: Name? = null) =
MutableSafeStringDelegate(this, key) { default }
@JvmName("safeBoolean")
fun <M : MutableMeta<M>> M.boolean(default: Boolean, key: String? = null) =
fun <M : MutableMeta<M>> M.boolean(default: Boolean, key: Name? = null) =
MutableSafeBooleanDelegate(this, key) { default }
@JvmName("safeNumber")
fun <M : MutableMeta<M>> M.number(default: Number, key: String? = null) =
fun <M : MutableMeta<M>> M.number(default: Number, key: Name? = null) =
MutableSafeNumberDelegate(this, key) { default }
@JvmName("safeString")
fun <M : MutableMeta<M>> M.string(key: String? = null, default: () -> String) =
fun <M : MutableMeta<M>> M.string(key: Name? = null, default: () -> String) =
MutableSafeStringDelegate(this, key, default)
@JvmName("safeBoolean")
fun <M : MutableMeta<M>> M.boolean(key: String? = null, default: () -> Boolean) =
fun <M : MutableMeta<M>> M.boolean(key: Name? = null, default: () -> Boolean) =
MutableSafeBooleanDelegate(this, key, default)
@JvmName("safeNumber")
fun <M : MutableMeta<M>> M.number(key: String? = null, default: () -> Number) =
fun <M : MutableMeta<M>> M.number(key: Name? = null, default: () -> Number) =
MutableSafeNumberDelegate(this, key, default)
inline fun <M : MutableMeta<M>, reified E : Enum<E>> M.enum(default: E, key: String? = null) =
inline fun <M : MutableMeta<M>, reified E : Enum<E>> M.enum(default: E, key: Name? = null) =
MutableSafeEnumvDelegate(this, key, default) { enumValueOf(it) }

51
dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/metaMatcher.kt Normal file
View File

@ -0,0 +1,51 @@
package hep.dataforge.meta
import hep.dataforge.names.Name
import hep.dataforge.names.toName
/**
* Get all items matching given name.
*/
@DFExperimental
fun Meta.getIndexed(name: Name): Map<String, MetaItem<*>> {
val root = when (name.length) {
0 -> error("Can't use empty name for that")
1 -> this
else -> (this[name.cutLast()] as? MetaItem.NodeItem<*>)?.node
}
val (body, index) = name.last()!!
val regex = index.toRegex()
return root?.items
?.filter { it.key.body == body && (index.isEmpty() || regex.matches(it.key.index)) }
?.mapKeys { it.key.index }
?: emptyMap()
}
@DFExperimental
fun Meta.getIndexed(name: String): Map<String, MetaItem<*>> = this@getIndexed.getIndexed(name.toName())
/**
* Get all items matching given name.
*/
@DFExperimental
fun <M : MetaNode<M>> M.getIndexed(name: Name): Map<String, MetaItem<M>> {
val root: MetaNode<M>? = when (name.length) {
0 -> error("Can't use empty name for that")
1 -> this
else -> (this[name.cutLast()] as? MetaItem.NodeItem<M>)?.node
}
val (body, index) = name.last()!!
val regex = index.toRegex()
return root?.items
?.filter { it.key.body == body && (index.isEmpty() || regex.matches(it.key.index)) }
?.mapKeys { it.key.index }
?: emptyMap()
}
@DFExperimental
fun <M : MetaNode<M>> M.getIndexed(name: String): Map<String, MetaItem<M>> = getIndexed(name.toName())

6
dataforge-meta/src/commonMain/kotlin/hep/dataforge/names/Name.kt
View File

@ -139,9 +139,7 @@ fun String.toName(): Name {
* Convert the [String] to a [Name] by simply wrapping it in a single name token without parsing.
* The input string could contain dots and braces, but they are just escaped, not parsed.
*/
fun String.asName(): Name {
return NameToken(this).asName()
}
fun String.asName(): Name = if (isBlank()) EmptyName else NameToken(this).asName()
operator fun NameToken.plus(other: Name): Name = Name(listOf(this) + other.tokens)
@ -149,6 +147,8 @@ operator fun Name.plus(other: Name): Name = Name(this.tokens + other.tokens)
operator fun Name.plus(other: String): Name = this + other.toName()
operator fun Name.plus(other: NameToken): Name = Name(tokens + other)
fun Name.appendLeft(other: String): Name = NameToken(other) + this
fun NameToken.asName() = Name(listOf(this))

58
dataforge-meta/src/commonMain/kotlin/hep/dataforge/values/Value.kt
View File

@ -41,10 +41,12 @@ interface Value {
* get this value represented as List
*/
val list: List<Value>
get() = listOf(this)
get() = if(this == Null) emptyList() else listOf(this)
override fun equals(other: Any?): Boolean
override fun hashCode(): Int
companion object {
const val TYPE = "value"
@ -58,7 +60,14 @@ interface Value {
true -> True
false -> False
is Number -> value.asValue()
is Iterable<*> -> ListValue(value.map { of(it) })
is Iterable<*> -> {
val list = value.map { of(it) }
if (list.isEmpty()) {
Null
} else {
ListValue(list)
}
}
is DoubleArray -> value.asValue()
is IntArray -> value.asValue()
is FloatArray -> value.asValue()
@ -86,14 +95,9 @@ object Null : Value {
override fun toString(): String = value.toString()
override fun equals(other: Any?): Boolean = other === Null
override fun hashCode(): Int = 0
}
/**
* Check if value is null
*/
fun Value.isNull(): Boolean = this == Null
/**
* Singleton true value
*/
@ -106,7 +110,7 @@ object True : Value {
override fun toString(): String = value.toString()
override fun equals(other: Any?): Boolean = other === True
override fun hashCode(): Int = 1
}
/**
@ -121,10 +125,9 @@ object False : Value {
override fun toString(): String = True.value.toString()
override fun equals(other: Any?): Boolean = other === False
override fun hashCode(): Int = -1
}
val Value.boolean get() = this == True || this.list.firstOrNull() == True || (type == ValueType.STRING && string.toBoolean())
class NumberValue(override val number: Number) : Value {
override val value: Any? get() = number
override val type: ValueType get() = ValueType.NUMBER
@ -178,23 +181,21 @@ class EnumValue<E : Enum<*>>(override val value: E) : Value {
class ListValue(override val list: List<Value>) : Value {
init {
if (list.isEmpty()) {
throw IllegalArgumentException("Can't create list value from empty list")
}
require(list.isNotEmpty()) { "Can't create list value from empty list" }
}
override val value: Any? get() = list
override val value: List<Value> get() = list
override val type: ValueType get() = list.first().type
override val number: Number get() = list.first().number
override val string: String get() = list.first().string
override fun toString(): String = list.joinToString (prefix = "[", postfix = "]")
override fun toString(): String = list.joinToString(prefix = "[", postfix = "]")
override fun equals(other: Any?): Boolean {
if (this === other) return true
if (other !is Value) return false
if( other is DoubleArrayValue){
if (other is DoubleArrayValue) {
return DoubleArray(list.size) { list[it].number.toDouble() }.contentEquals(other.value)
}
return list == other.list
}
@ -206,29 +207,26 @@ class ListValue(override val list: List<Value>) : Value {
}
/**
* Check if value is list
*/
fun Value.isList(): Boolean = this.list.size > 1
fun Number.asValue(): Value = NumberValue(this)
fun Boolean.asValue(): Value = if (this) True else False
fun String.asValue(): Value = StringValue(this)
fun Iterable<Value>.asValue(): Value = ListValue(this.toList())
fun Iterable<Value>.asValue(): Value {
val list = toList()
return if (list.isEmpty()) Null else ListValue(this.toList())
}
fun IntArray.asValue(): Value = ListValue(map{NumberValue(it)})
fun IntArray.asValue(): Value = if (isEmpty()) Null else ListValue(map { NumberValue(it) })
fun LongArray.asValue(): Value = ListValue(map{NumberValue(it)})
fun LongArray.asValue(): Value = if (isEmpty()) Null else ListValue(map { NumberValue(it) })
fun ShortArray.asValue(): Value = ListValue(map{NumberValue(it)})
fun ShortArray.asValue(): Value = if (isEmpty()) Null else ListValue(map { NumberValue(it) })
fun FloatArray.asValue(): Value = ListValue(map{NumberValue(it)})
fun FloatArray.asValue(): Value = if (isEmpty()) Null else ListValue(map { NumberValue(it) })
fun ByteArray.asValue(): Value = ListValue(map{NumberValue(it)})
fun ByteArray.asValue(): Value = if (isEmpty()) Null else ListValue(map { NumberValue(it) })
/**

4
dataforge-meta/src/commonMain/kotlin/hep/dataforge/values/exoticValues.kt
View File

@ -14,6 +14,8 @@ class LazyParsedValue(override val string: String) : Value {
override fun toString(): String = string
override fun equals(other: Any?): Boolean = other is Value && this.parsedValue == other
override fun hashCode(): Int = string.hashCode()
}
fun String.lazyParseValue(): LazyParsedValue = LazyParsedValue(this)
@ -44,4 +46,4 @@ class DoubleArrayValue(override val value: DoubleArray) : Value {
override fun toString(): String = list.joinToString (prefix = "[", postfix = "]")
}
fun DoubleArray.asValue(): DoubleArrayValue = DoubleArrayValue(this)
fun DoubleArray.asValue(): Value = if(isEmpty()) Null else DoubleArrayValue(this)

37
dataforge-meta/src/commonMain/kotlin/hep/dataforge/values/valueExtensions.kt Normal file
View File

@ -0,0 +1,37 @@
package hep.dataforge.values
import hep.dataforge.meta.Meta
import hep.dataforge.meta.buildMeta
/**
* Check if value is null
*/
fun Value.isNull(): Boolean = this == Null
/**
* Check if value is list
*/
fun Value.isList(): Boolean = this.list.size > 1
val Value.boolean
get() = this == True
|| this.list.firstOrNull() == True
|| (type == ValueType.STRING && string.toBoolean())
val Value.int get() = number.toInt()
val Value.double get() = number.toDouble()
val Value.float get() = number.toFloat()
val Value.long get() = number.toLong()
val Value.stringList: List<String> get() = list.map { it.string }
val Value.doubleArray: DoubleArray
get() = if (this is DoubleArrayValue) {
value
} else {
DoubleArray(list.size) { list[it].double }
}
fun Value.toMeta() = buildMeta { Meta.VALUE_KEY put this }

16
dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/MetaBuilderTest.kt
View File

@ -9,13 +9,13 @@ class MetaBuilderTest {
@Test
fun testBuilder() {
val meta = buildMeta {
"a" to 22
"b" to listOf(1, 2, 3)
"a" put 22
"b" put listOf(1, 2, 3)
this["c"] = "myValue".asValue()
"node" to {
"e" to 12.2
"childNode" to {
"f" to true
"node" put {
"e" put 12.2
"childNode" put {
"f" put true
}
}
}
@ -27,12 +27,12 @@ class MetaBuilderTest {
fun testSNS(){
val meta = buildMeta {
repeat(10){
"b.a[$it]" to it
"b.a[$it]" put it
}
}.seal()
assertEquals(10, meta.values().count())
val nodes = meta.getAll("b.a")
val nodes = meta.getIndexed("b.a")
assertEquals(3, nodes["3"]?.int)
}

23
dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/MetaExtensionTest.kt Normal file
View File

@ -0,0 +1,23 @@
package hep.dataforge.meta
import kotlin.test.Test
class MetaExtensionTest {
enum class TestEnum{
test
}
@Test
fun testEnum(){
val meta = buildMeta{"enum" put TestEnum.test}
meta["enum"].enum<TestEnum>()
}
@Test
fun testEnumByString(){
val meta = buildMeta{"enum" put TestEnum.test.name}
println(meta["enum"].enum<TestEnum>())
}
}

31
dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/MetaTest.kt
View File

@ -17,17 +17,36 @@ class MetaTest {
@Test
fun metaEqualityTest() {
val meta1 = buildMeta {
"a" to 22
"b" to {
"c" to "ddd"
"a" put 22
"b" put {
"c" put "ddd"
}
}
val meta2 = buildMeta {
"b" to {
"c" to "ddd"
"b" put {
"c" put "ddd"
}
"a" to 22
"a" put 22
}.seal()
assertEquals<Meta>(meta1, meta2)
}
@Test
fun metaToMap(){
val meta = buildMeta {
"a" put 22
"b" put {
"c" put "ddd"
}
"list" put (0..4).map {
buildMeta {
"value" put it
}
}
}
val map = meta.toMap()
val reconstructed = map.toMeta()
assertEquals(meta,reconstructed)
}
}

10
dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/MutableMetaTest.kt
View File

@ -7,12 +7,12 @@ class MutableMetaTest{
@Test
fun testRemove(){
val meta = buildMeta {
"aNode" to {
"innerNode" to {
"innerValue" to true
"aNode" put {
"innerNode" put {
"innerValue" put true
}
"b" to 22
"c" to "StringValue"
"b" put 22
"c" put "StringValue"
}
}.toConfig()

23
dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/SpecificationTest.kt Normal file
View File

@ -0,0 +1,23 @@
package hep.dataforge.meta
import kotlin.test.Test
import kotlin.test.assertEquals
class SpecificationTest {
class TestSpecific(override val config: Config) : Specific {
var list by numberList(1, 2, 3)
companion object : Specification<TestSpecific> {
override fun wrap(config: Config): TestSpecific = TestSpecific(config)
}
}
@Test
fun testSpecific(){
val testObject = TestSpecific.build {
list = emptyList()
}
assertEquals(emptyList(), testObject.list)
}
}

8
dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/StyledTest.kt
View File

@ -9,8 +9,8 @@ class StyledTest{
fun testSNS(){
val meta = buildMeta {
repeat(10){
"b.a[$it]" to {
"d" to it
"b.a[$it]" put {
"d" put it
}
}
}.seal().withStyle()
@ -18,9 +18,9 @@ class StyledTest{
val bNode = meta["b"].node
val aNodes = bNode?.getAll("a")
val aNodes = bNode?.getIndexed("a")
val allNodes = meta.getAll("b.a")
val allNodes = meta.getIndexed("b.a")
assertEquals(3, aNodes?.get("3").node["d"].int)
assertEquals(3, allNodes["3"].node["d"].int)

41
dataforge-meta/src/jsMain/kotlin/hep/dataforge/meta/DynamicMeta.kt
View File

@ -3,18 +3,27 @@ package hep.dataforge.meta
import hep.dataforge.names.NameToken
import hep.dataforge.values.Null
import hep.dataforge.values.Value
import hep.dataforge.values.isList
//TODO add Meta wrapper for dynamic
fun Value.toDynamic(): dynamic {
return if (isList()) {
list.map { it.toDynamic() }.toTypedArray().asDynamic()
} else {
value.asDynamic()
}
}
/**
* Represent or copy this [Meta] to dynamic object to be passed to JS libraries
*/
fun Meta.toDynamic(): dynamic {
if(this is DynamicMeta) return this.obj
if (this is DynamicMeta) return this.obj
fun MetaItem<*>.toDynamic(): dynamic = when (this) {
is MetaItem.ValueItem -> this.value.value.asDynamic()
is MetaItem.ValueItem -> this.value.toDynamic()
is MetaItem.NodeItem -> this.node.toDynamic()
}
@ -35,15 +44,21 @@ class DynamicMeta(val obj: dynamic) : MetaBase() {
private fun isArray(@Suppress("UNUSED_PARAMETER") obj: dynamic): Boolean =
js("Array.isArray(obj)") as Boolean
private fun isPrimitive(obj: dynamic): Boolean =
(jsTypeOf(obj) != "object")
@Suppress("UNCHECKED_CAST")
private fun asItem(obj: dynamic): MetaItem<DynamicMeta>? {
if (obj == null) return MetaItem.ValueItem(Null)
return when (jsTypeOf(obj as? Any)) {
"boolean" -> MetaItem.ValueItem(Value.of(obj as Boolean))
"number" -> MetaItem.ValueItem(Value.of(obj as Number))
"string" -> MetaItem.ValueItem(Value.of(obj as String))
"object" -> MetaItem.NodeItem(DynamicMeta(obj))
else -> null
return when {
obj == null -> MetaItem.ValueItem(Null)
isArray(obj) && (obj as Array<Any?>).all { isPrimitive(it) } -> MetaItem.ValueItem(Value.of(obj as Array<Any?>))
else -> when (jsTypeOf(obj)) {
"boolean" -> MetaItem.ValueItem(Value.of(obj as Boolean))
"number" -> MetaItem.ValueItem(Value.of(obj as Number))
"string" -> MetaItem.ValueItem(Value.of(obj as String))
"object" -> MetaItem.NodeItem(DynamicMeta(obj))
else -> null
}
}
}
@ -51,11 +66,15 @@ class DynamicMeta(val obj: dynamic) : MetaBase() {
get() = keys().flatMap<String, Pair<NameToken, MetaItem<DynamicMeta>>> { key ->
val value = obj[key] ?: return@flatMap emptyList()
if (isArray(value)) {
return@flatMap (value as Array<dynamic>)
.mapIndexedNotNull() { index, it ->
val array = value as Array<Any?>
return@flatMap if (array.all { isPrimitive(it) }) {
listOf(NameToken(key) to MetaItem.ValueItem(Value.of(array)))
} else {
array.mapIndexedNotNull { index, it ->
val item = asItem(it) ?: return@mapIndexedNotNull null
NameToken(key, index.toString()) to item
}
}
} else {
val item = asItem(value) ?: return@flatMap emptyList()
listOf(NameToken(key) to item)

34
dataforge-meta/src/jsTest/kotlin/hep/dataforge/meta/DynamicMetaTest.kt
View File

@ -1,7 +1,9 @@
package hep.dataforge.meta
import hep.dataforge.values.int
import kotlin.test.Test
import kotlin.test.assertEquals
import kotlin.test.assertTrue
class DynamicMetaTest {
@ -10,7 +12,7 @@ class DynamicMetaTest {
fun testDynamicMeta() {
val d = js("{}")
d.a = 22
d.array = arrayOf(1,2,3)
d.array = arrayOf(1, 2, 3)
d.b = "myString"
d.ob = js("{}")
d.ob.childNode = 18
@ -18,7 +20,35 @@ class DynamicMetaTest {
val meta = DynamicMeta(d)
assertEquals(true, meta["ob.booleanNode"].boolean)
assertEquals(2,meta["array[1]"].int)
assertEquals(2, meta["array"].value?.list?.get(1)?.int)
assertEquals(4, meta.items.size)
}
@Test
fun testMetaToDynamic(){
val meta = buildMeta {
"a" put 22
"array" put listOf(1, 2, 3)
"b" put "myString"
"ob" put {
"childNode" put 18
"booleanNode" put true
}
}
val dynamic = meta.toDynamic()
assertEquals(2,dynamic.array[1])
assertEquals(22, dynamic.a)
val keys = js("Object.keys(dynamic)") as Array<String>
assertTrue { keys.contains("ob") }
assertEquals(18, dynamic.ob.childNode)
assertEquals<Meta>(meta, DynamicMeta(dynamic))
}
}

2
dataforge-output/src/commonMain/kotlin/hep/dataforge/output/OutputManager.kt
View File

@ -70,7 +70,7 @@ class ConsoleOutputManager : AbstractPlugin(), OutputManager {
override val type = ConsoleOutputManager::class
override fun invoke(meta:Meta) = ConsoleOutputManager()
override fun invoke(meta: Meta, context: Context) = ConsoleOutputManager()
}
}

8
dataforge-scripting/src/jvmTest/kotlin/hep/dataforge/scripting/BuildersKtTest.kt
View File

@ -3,7 +3,9 @@ package hep.dataforge.scripting
import hep.dataforge.context.Global
import hep.dataforge.meta.get
import hep.dataforge.meta.int
import hep.dataforge.workspace.*
import hep.dataforge.workspace.SimpleWorkspaceBuilder
import hep.dataforge.workspace.context
import hep.dataforge.workspace.target
import org.junit.Test
import kotlin.test.assertEquals
@ -17,7 +19,7 @@ class BuildersKtTest {
context("test")
target("testTarget"){
"a" to 12
"a" put 12
}
}
}
@ -30,7 +32,7 @@ class BuildersKtTest {
context("test")
target("testTarget"){
"a" to 12
"a" put 12
}
""".trimIndent()
val workspace = Builders.buildWorkspace(script)

65
dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/Dependency.kt
View File

@ -6,10 +6,7 @@ import hep.dataforge.data.filter
import hep.dataforge.meta.Meta
import hep.dataforge.meta.MetaRepr
import hep.dataforge.meta.buildMeta
import hep.dataforge.names.EmptyName
import hep.dataforge.names.Name
import hep.dataforge.names.get
import hep.dataforge.names.isEmpty
import hep.dataforge.names.*
/**
* A dependency of the task which allows to lazily create a data tree for single dependency
@ -24,13 +21,13 @@ class DataDependency(val filter: DataFilter, val placement: Name = EmptyName) :
return if (placement.isEmpty()) {
result
} else {
DataNode.build(Any::class) { this[placement] = result }
DataNode.invoke(Any::class) { this[placement] = result }
}
}
override fun toMeta(): Meta = buildMeta {
"data" to filter.config
"to" to placement
"data" put filter.config
"to" put placement.toString()
}
}
@ -38,31 +35,63 @@ class AllDataDependency(val placement: Name = EmptyName) : Dependency() {
override fun apply(workspace: Workspace): DataNode<Any> = if (placement.isEmpty()) {
workspace.data
} else {
DataNode.build(Any::class) { this[placement] = workspace.data }
DataNode.invoke(Any::class) { this[placement] = workspace.data }
}
override fun toMeta() = buildMeta {
"data" to "*"
"to" to placement
"data" put "@all"
"to" put placement.toString()
}
}
class TaskModelDependency(val name: String, val meta: Meta, val placement: Name = EmptyName) : Dependency() {
override fun apply(workspace: Workspace): DataNode<Any> {
val task = workspace.tasks[name] ?: error("Task with name $name is not found in the workspace")
abstract class TaskDependency<out T : Any>(
val meta: Meta,
val placement: Name = EmptyName
) : Dependency() {
abstract fun resolveTask(workspace: Workspace): Task<T>
/**
* A name of the dependency for logging and serialization
*/
abstract val name: Name
override fun apply(workspace: Workspace): DataNode<T> {
val task = resolveTask(workspace)
if (task.isTerminal) TODO("Support terminal task")
val result = workspace.run(task, meta)
return if (placement.isEmpty()) {
result
} else {
DataNode.build(Any::class) { this[placement] = result }
DataNode(task.type) { this[placement] = result }
}
}
override fun toMeta(): Meta = buildMeta {
"task" to name
"meta" to meta
"to" to placement
"task" put name.toString()
"meta" put meta
"to" put placement.toString()
}
}
class DirectTaskDependency<T : Any>(
val task: Task<T>,
meta: Meta,
placement: Name
) : TaskDependency<T>(meta, placement) {
override fun resolveTask(workspace: Workspace): Task<T> = task
override val name: Name get() = DIRECT_TASK_NAME + task.name
companion object {
val DIRECT_TASK_NAME = "@direct".asName()
}
}
class WorkspaceTaskDependency(
override val name: Name,
meta: Meta,
placement: Name
) : TaskDependency<Any>(meta, placement) {
override fun resolveTask(workspace: Workspace): Task<*> =
workspace.tasks[name] ?: error("Task with name $name is not found in the workspace")
}

23
dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/GenericTask.kt
View File

@ -1,37 +1,38 @@
package hep.dataforge.workspace
import hep.dataforge.data.*
import hep.dataforge.data.DataNode
import hep.dataforge.descriptors.NodeDescriptor
import hep.dataforge.meta.Meta
import hep.dataforge.meta.get
import hep.dataforge.meta.node
import hep.dataforge.names.Name
import kotlin.reflect.KClass
//data class TaskEnv(val workspace: Workspace, val model: TaskModel)
class GenericTask<R : Any>(
override val name: String,
override val name: Name,
override val type: KClass<out R>,
override val descriptor: NodeDescriptor,
private val modelTransform: TaskModelBuilder.(Meta) -> Unit,
private val dataTransform: Workspace.() -> TaskModel.(DataNode<Any>) -> DataNode<R>
) : Task<R> {
private fun gather(workspace: Workspace, model: TaskModel): DataNode<Any> {
return DataNode.build(Any::class) {
model.dependencies.forEach { dep ->
update(dep.apply(workspace))
}
}
}
// private fun gather(workspace: Workspace, model: TaskModel): DataNode<Any> {
// return DataNode.invoke(Any::class) {
// model.dependencies.forEach { dep ->
// update(dep.apply(workspace))
// }
// }
// }
override fun run(workspace: Workspace, model: TaskModel): DataNode<R> {
//validate model
validate(model)
// gather data
val input = gather(workspace, model)
val input = model.buildInput(workspace)// gather(workspace, model)
//execute
workspace.context.logger.info{"Starting task '$name' on ${model.target} with meta: \n${model.meta}"}
@ -54,7 +55,7 @@ class GenericTask<R : Any>(
override fun build(workspace: Workspace, taskConfig: Meta): TaskModel {
val taskMeta = taskConfig[name]?.node ?: taskConfig
val builder = TaskModelBuilder(name, taskMeta)
modelTransform.invoke(builder, taskMeta)
builder.modelTransform(taskMeta)
return builder.build()
}
//TODO add validation

8
dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/SimpleWorkspace.kt
View File

@ -1,12 +1,11 @@
package hep.dataforge.workspace
import hep.dataforge.context.Context
import hep.dataforge.context.Global
import hep.dataforge.context.content
import hep.dataforge.context.toMap
import hep.dataforge.data.DataNode
import hep.dataforge.meta.Meta
import hep.dataforge.names.Name
import hep.dataforge.names.toName
/**
@ -20,11 +19,10 @@ class SimpleWorkspace(
) : Workspace {
override val tasks: Map<Name, Task<*>> by lazy {
context.content<Task<*>>(Task.TYPE) + tasks.associate { it.name.toName() to it }
context.content<Task<*>>(Task.TYPE) + tasks.toMap()
}
companion object {
fun build(parent: Context = Global, block: SimpleWorkspaceBuilder.() -> Unit): SimpleWorkspace =
SimpleWorkspaceBuilder(parent).apply(block).build()
}
}

142
dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/TaskBuilder.kt → dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskBuilder.kt
View File

@ -6,54 +6,78 @@ import hep.dataforge.descriptors.NodeDescriptor
import hep.dataforge.meta.Meta
import hep.dataforge.meta.get
import hep.dataforge.meta.string
import hep.dataforge.names.EmptyName
import hep.dataforge.names.Name
import hep.dataforge.names.isEmpty
import hep.dataforge.names.toName
import kotlin.jvm.JvmName
import kotlin.reflect.KClass
@TaskBuildScope
class TaskBuilder(val name: String) {
private var modelTransform: TaskModelBuilder.(Meta) -> Unit = { data("*") }
class TaskBuilder<R : Any>(val name: Name, val type: KClass<out R>) {
private var modelTransform: TaskModelBuilder.(Meta) -> Unit = { allData() }
// private val additionalDependencies = HashSet<Dependency>()
var descriptor: NodeDescriptor? = null
private val dataTransforms: MutableList<DataTransformation> = ArrayList()
/**
* TODO will look better as extension class
*/
private class DataTransformation(
private inner class DataTransformation(
val from: String = "",
val to: String = "",
val transform: (Context, TaskModel, DataNode<Any>) -> DataNode<Any>
val transform: (Context, TaskModel, DataNode<Any>) -> DataNode<R>
) {
operator fun invoke(workspace: Workspace, model: TaskModel, node: DataNode<Any>): DataNode<Any>? {
operator fun invoke(workspace: Workspace, model: TaskModel, node: DataNode<Any>): DataNode<R>? {
val localData = if (from.isEmpty()) {
node
} else {
node[from.toName()].node ?: return null
node[from].node ?: return null
}
return transform(workspace.context, model, localData)
}
}
private val dataTransforms: MutableList<DataTransformation> = ArrayList();
// override fun add(dependency: Dependency) {
// additionalDependencies.add(dependency)
// }
fun model(modelTransform: TaskModelBuilder.(Meta) -> Unit) {
this.modelTransform = modelTransform
}
fun <T : Any> transform(
inputType: KClass<T>,
/**
* Add a transformation on untyped data
*/
@JvmName("rawTransform")
fun transform(
from: String = "",
to: String = "",
block: TaskModel.(Context, DataNode<T>) -> DataNode<Any>
block: TaskEnv.(DataNode<*>) -> DataNode<R>
) {
dataTransforms += DataTransformation(from, to) { context, model, data ->
block(model, context, data.cast(inputType))
val env = TaskEnv(EmptyName, model.meta, context, data)
env.block(data)
}
}
fun <T : Any> transform(
inputType: KClass<out T>,
from: String = "",
to: String = "",
block: TaskEnv.(DataNode<T>) -> DataNode<R>
) {
dataTransforms += DataTransformation(from, to) { context, model, data ->
data.ensureType(inputType)
val env = TaskEnv(EmptyName, model.meta, context, data)
env.block(data.cast(inputType))
}
}
inline fun <reified T : Any> transform(
from: String = "",
to: String = "",
noinline block: TaskModel.(Context, DataNode<T>) -> DataNode<Any>
noinline block: TaskEnv.(DataNode<T>) -> DataNode<R>
) {
transform(T::class, from, to, block)
}
@ -61,52 +85,57 @@ class TaskBuilder(val name: String) {
/**
* Perform given action on data elements in `from` node in input and put the result to `to` node
*/
inline fun <reified T : Any, reified R : Any> action(
inline fun <reified T : Any> action(
from: String = "",
to: String = "",
crossinline block: Context.() -> Action<T, R>
crossinline block: TaskEnv.() -> Action<T, R>
) {
transform(from, to) { context, data: DataNode<T> ->
block(context).invoke(data, meta)
transform(from, to) { data: DataNode<T> ->
block().invoke(data, meta)
}
}
class TaskEnv(val name: Name, val meta: Meta, val context: Context)
class TaskEnv(val name: Name, val meta: Meta, val context: Context, val data: DataNode<Any>) {
operator fun <T : Any> DirectTaskDependency<T>.invoke(): DataNode<T> = if(placement.isEmpty()){
data.cast(task.type)
} else {
data[placement].node?.cast(task.type)
?: error("Could not find results of direct task dependency $this at \"$placement\"")
}
}
/**
* A customized pipe action with ability to change meta and name
* A customized map action with ability to change meta and name
*/
inline fun <reified T : Any, reified R : Any> customPipe(
inline fun <reified T : Any> mapAction(
from: String = "",
to: String = "",
crossinline block: PipeBuilder<T, R>.(Context) -> Unit
crossinline block: MapActionBuilder<T, R>.(TaskEnv) -> Unit
) {
action(from, to) {
val context = this
PipeAction(
MapAction(
inputType = T::class,
outputType = R::class
) { block(context) }
outputType = type
) { block(this@action) }
}
}
/**
* A simple pipe action without changing meta or name
* A simple map action without changing meta or name
*/
inline fun <reified T : Any, reified R : Any> pipe(
inline fun <reified T : Any> map(
from: String = "",
to: String = "",
crossinline block: suspend TaskEnv.(T) -> R
) {
action(from, to) {
val context = this
PipeAction(
MapAction(
inputType = T::class,
outputType = R::class
outputType = type
) {
//TODO automatically append task meta
result = { data ->
TaskEnv(name, meta, context).block(data)
block(data)
}
}
}
@ -115,15 +144,15 @@ class TaskBuilder(val name: String) {
/**
* Join elements in gathered data by multiple groups
*/
inline fun <reified T : Any, reified R : Any> joinByGroup(
inline fun <reified T : Any> reduceByGroup(
from: String = "",
to: String = "",
crossinline block: JoinGroupBuilder<T, R>.(Context) -> Unit
crossinline block: ReduceGroupBuilder<T, R>.(TaskEnv) -> Unit //TODO needs KEEP-176
) {
action(from, to) {
JoinAction(
ReduceAction(
inputType = T::class,
outputType = R::class
outputType = type
) { block(this@action) }
}
}
@ -131,21 +160,20 @@ class TaskBuilder(val name: String) {
/**
* Join all elemlents in gathered data matching input type
*/
inline fun <reified T : Any, reified R : Any> join(
inline fun <reified T : Any> reduce(
from: String = "",
to: String = "",
crossinline block: suspend TaskEnv.(Map<Name, T>) -> R
) {
action(from, to) {
val context = this
JoinAction(
ReduceAction(
inputType = T::class,
outputType = R::class,
outputType = type,
action = {
result(
actionMeta[TaskModel.MODEL_TARGET_KEY]?.string ?: "@anonymous"
) { data ->
TaskEnv(name, meta, context).block(data)
block(data)
}
}
)
@ -155,15 +183,15 @@ class TaskBuilder(val name: String) {
/**
* Split each element in gathered data into fixed number of fragments
*/
inline fun <reified T : Any, reified R : Any> split(
inline fun <reified T : Any> split(
from: String = "",
to: String = "",
crossinline block: SplitBuilder<T, R>.(Context) -> Unit
crossinline block: SplitBuilder<T, R>.(TaskEnv) -> Unit //TODO needs KEEP-176
) {
action(from, to) {
SplitAction(
inputType = T::class,
outputType = R::class
outputType = type
) { block(this@action) }
}
}
@ -175,22 +203,28 @@ class TaskBuilder(val name: String) {
this.descriptor = NodeDescriptor.build(transform)
}
internal fun build(): GenericTask<Any> =
GenericTask(
internal fun build(): GenericTask<R> {
// val actualTransform: TaskModelBuilder.(Meta) -> Unit = {
// modelTransform
// dependencies.addAll(additionalDependencies)
// }
return GenericTask(
name,
Any::class,
type,
descriptor ?: NodeDescriptor.empty(),
modelTransform
) {
val workspace = this
{ data ->
return@GenericTask { data ->
val model = this
if (dataTransforms.isEmpty()) {
//return data node as is
logger.warn("No transformation present, returning input data")
data
logger.warn { "No transformation present, returning input data" }
data.ensureType(type)
data.cast(type)
} else {
val builder = DataTreeBuilder(Any::class)
val builder = DataTreeBuilder(type)
dataTransforms.forEach { transformation ->
val res = transformation(workspace, model, data)
if (res != null) {
@ -205,14 +239,6 @@ class TaskBuilder(val name: String) {
}
}
}
}
}
fun Workspace.Companion.task(name: String, builder: TaskBuilder.() -> Unit): GenericTask<Any> {
return TaskBuilder(name).apply(builder).build()
}
fun WorkspaceBuilder.task(name: String, builder: TaskBuilder.() -> Unit) {
task(TaskBuilder(name).apply(builder).build())
}
//TODO add delegates to build gradle-like tasks

129
dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskModel.kt
View File

@ -8,10 +8,10 @@ package hep.dataforge.workspace
import hep.dataforge.data.DataFilter
import hep.dataforge.data.DataTree
import hep.dataforge.data.DataTreeBuilder
import hep.dataforge.data.dataSequence
import hep.dataforge.meta.*
import hep.dataforge.names.EmptyName
import hep.dataforge.names.Name
import hep.dataforge.names.asName
import hep.dataforge.names.toName
import hep.dataforge.workspace.TaskModel.Companion.MODEL_TARGET_KEY
@ -23,7 +23,7 @@ import hep.dataforge.workspace.TaskModel.Companion.MODEL_TARGET_KEY
* @param dependencies a list of direct dependencies for this task
*/
data class TaskModel(
val name: String,
val name: Name,
val meta: Meta,
val dependencies: Collection<Dependency>
) : MetaRepr {
@ -31,19 +31,19 @@ data class TaskModel(
//TODO add pre-run check of task result type?
override fun toMeta(): Meta = buildMeta {
"name" to name
"meta" to meta
"dependsOn" to {
"name" put name.toString()
"meta" put meta
"dependsOn" put {
val dataDependencies = dependencies.filterIsInstance<DataDependency>()
val taskDependencies = dependencies.filterIsInstance<TaskModelDependency>()
val taskDependencies = dependencies.filterIsInstance<TaskDependency<*>>()
setIndexed("data".toName(), dataDependencies.map { it.toMeta() })
setIndexed("task".toName(), taskDependencies.map { it.toMeta() }) { taskDependencies[it].name }
setIndexed("task".toName(), taskDependencies.map { it.toMeta() }) { taskDependencies[it].name.toString() }
//TODO ensure all dependencies are listed
}
}
companion object {
const val MODEL_TARGET_KEY = "@target"
val MODEL_TARGET_KEY = "@target".asName()
}
}
@ -52,9 +52,8 @@ data class TaskModel(
*/
fun TaskModel.buildInput(workspace: Workspace): DataTree<Any> {
return DataTreeBuilder(Any::class).apply {
dependencies.asSequence().flatMap { it.apply(workspace).dataSequence() }.forEach { (name, data) ->
//TODO add concise error on replacement
this[name] = data
dependencies.forEach { dep ->
update(dep.apply(workspace))
}
}.build()
}
@ -62,54 +61,88 @@ fun TaskModel.buildInput(workspace: Workspace): DataTree<Any> {
@DslMarker
annotation class TaskBuildScope
interface TaskDependencyContainer {
val defaultMeta: Meta
fun add(dependency: Dependency)
}
/**
* Add dependency for a task defined in a workspace and resolved by
*/
fun TaskDependencyContainer.dependsOn(
name: Name,
placement: Name = EmptyName,
meta: Meta = defaultMeta
): WorkspaceTaskDependency =
WorkspaceTaskDependency(name, meta, placement).also { add(it) }
fun TaskDependencyContainer.dependsOn(
name: String,
placement: Name = EmptyName,
meta: Meta = defaultMeta
): WorkspaceTaskDependency =
dependsOn(name.toName(), placement, meta)
fun <T : Any> TaskDependencyContainer.dependsOn(
task: Task<T>,
placement: Name = EmptyName,
meta: Meta = defaultMeta
): DirectTaskDependency<T> =
DirectTaskDependency(task, meta, placement).also { add(it) }
fun <T : Any> TaskDependencyContainer.dependsOn(
task: Task<T>,
placement: String,
meta: Meta = defaultMeta
): DirectTaskDependency<T> =
DirectTaskDependency(task, meta, placement.toName()).also { add(it) }
fun <T : Any> TaskDependencyContainer.dependsOn(
task: Task<T>,
placement: Name = EmptyName,
metaBuilder: MetaBuilder.() -> Unit
): DirectTaskDependency<T> =
dependsOn(task, placement, buildMeta(metaBuilder))
/**
* Add custom data dependency
*/
fun TaskDependencyContainer.data(action: DataFilter.() -> Unit): DataDependency =
DataDependency(DataFilter.build(action)).also { add(it) }
/**
* User-friendly way to add data dependency
*/
fun TaskDependencyContainer.data(pattern: String? = null, from: String? = null, to: String? = null): DataDependency =
data {
pattern?.let { this.pattern = it }
from?.let { this.from = it }
to?.let { this.to = it }
}
/**
* Add all data as root node
*/
fun TaskDependencyContainer.allData(to: Name = EmptyName) = AllDataDependency(to).also { add(it) }
/**
* A builder for [TaskModel]
*/
@TaskBuildScope
class TaskModelBuilder(val name: String, meta: Meta = EmptyMeta) {
class TaskModelBuilder(val name: Name, meta: Meta = EmptyMeta) : TaskDependencyContainer {
/**
* Meta for current task. By default uses the whole input meta
*/
var meta: MetaBuilder = meta.builder()
val dependencies = HashSet<Dependency>()
override val defaultMeta: Meta get() = meta
override fun add(dependency: Dependency) {
dependencies.add(dependency)
}
var target: String by this.meta.string(key = MODEL_TARGET_KEY, default = "")
/**
* Add dependency for
*/
fun dependsOn(name: String, meta: Meta = this.meta, placement: Name = EmptyName) {
dependencies.add(TaskModelDependency(name, meta, placement))
}
fun dependsOn(task: Task<*>, meta: Meta = this.meta, placement: Name = EmptyName) =
dependsOn(task.name, meta, placement)
fun dependsOn(task: Task<*>, placement: Name = EmptyName, metaBuilder: MetaBuilder.() -> Unit) =
dependsOn(task.name, buildMeta(metaBuilder), placement)
/**
* Add custom data dependency
*/
fun data(action: DataFilter.() -> Unit) {
dependencies.add(DataDependency(DataFilter.build(action)))
}
/**
* User-friendly way to add data dependency
*/
fun data(pattern: String? = null, from: String? = null, to: String? = null) = data {
pattern?.let { this.pattern = it }
from?.let { this.from = it }
to?.let { this.to = it }
}
/**
* Add all data as root node
*/
fun allData(to: Name = EmptyName) {
dependencies.add(AllDataDependency(to))
}
fun build(): TaskModel = TaskModel(name, meta.seal(), dependencies)
}

7
dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/Workspace.kt
View File

@ -1,6 +1,8 @@
package hep.dataforge.workspace
import hep.dataforge.context.Context
import hep.dataforge.context.ContextAware
import hep.dataforge.context.Global
import hep.dataforge.data.Data
import hep.dataforge.data.DataNode
import hep.dataforge.data.dataSequence
@ -56,6 +58,8 @@ interface Workspace : ContextAware, Provider {
companion object {
const val TYPE = "workspace"
operator fun invoke(parent: Context = Global, block: SimpleWorkspaceBuilder.() -> Unit): SimpleWorkspace =
SimpleWorkspaceBuilder(parent).apply(block).build()
}
}
@ -73,3 +77,6 @@ fun Workspace.run(task: String, meta: Meta) =
fun Workspace.run(task: String, block: MetaBuilder.() -> Unit = {}) =
run(task, buildMeta(block))
fun <T: Any> Workspace.run(task: Task<T>, metaBuilder: MetaBuilder.() -> Unit = {}): DataNode<T> =
run(task, buildMeta(metaBuilder))

67
dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/WorkspaceBuilder.kt
View File

@ -2,12 +2,15 @@ package hep.dataforge.workspace
import hep.dataforge.context.Context
import hep.dataforge.context.ContextBuilder
import hep.dataforge.data.Data
import hep.dataforge.data.DataNode
import hep.dataforge.data.DataTreeBuilder
import hep.dataforge.meta.*
import hep.dataforge.names.EmptyName
import hep.dataforge.names.Name
import hep.dataforge.names.isEmpty
import hep.dataforge.names.toName
import kotlin.jvm.JvmName
import kotlin.reflect.KClass
@TaskBuildScope
interface WorkspaceBuilder {
@ -28,32 +31,26 @@ fun WorkspaceBuilder.context(name: String = "WORKSPACE", block: ContextBuilder.(
context = ContextBuilder(name, parentContext).apply(block).build()
}
fun WorkspaceBuilder.data(name: Name, data: Data<Any>) {
this.data[name] = data
inline fun <reified T : Any> WorkspaceBuilder.data(
name: Name = EmptyName,
noinline block: DataTreeBuilder<T>.() -> Unit
): DataNode<T> {
val node = DataTreeBuilder(T::class).apply(block)
if (name.isEmpty()) {
@Suppress("UNCHECKED_CAST")
data = node as DataTreeBuilder<Any>
} else {
data[name] = node
}
return node.build()
}
fun WorkspaceBuilder.data(name: String, data: Data<Any>) = data(name.toName(), data)
@JvmName("rawData")
fun WorkspaceBuilder.data(
name: Name = EmptyName,
block: DataTreeBuilder<Any>.() -> Unit
): DataNode<Any> = data<Any>(name, block)
fun WorkspaceBuilder.static(name: Name, data: Any, meta: Meta = EmptyMeta) =
data(name, Data.static(data, meta))
fun WorkspaceBuilder.static(name: Name, data: Any, block: MetaBuilder.() -> Unit = {}) =
data(name, Data.static(data, buildMeta(block)))
fun WorkspaceBuilder.static(name: String, data: Any, block: MetaBuilder.() -> Unit = {}) =
data(name, Data.static(data, buildMeta(block)))
fun WorkspaceBuilder.data(name: Name, node: DataNode<Any>) {
this.data[name] = node
}
fun WorkspaceBuilder.data(name: String, node: DataNode<Any>) = data(name.toName(), node)
fun WorkspaceBuilder.data(name: Name, block: DataTreeBuilder<Any>.() -> Unit) {
this.data[name] = DataNode.build(Any::class, block)
}
fun WorkspaceBuilder.data(name: String, block: DataTreeBuilder<Any>.() -> Unit) = data(name.toName(), block)
fun WorkspaceBuilder.target(name: String, block: MetaBuilder.() -> Unit) {
targets[name] = buildMeta(block).seal()
@ -65,22 +62,34 @@ fun WorkspaceBuilder.target(name: String, block: MetaBuilder.() -> Unit) {
fun WorkspaceBuilder.target(name: String, base: String, block: MetaBuilder.() -> Unit) {
val parentTarget = targets[base] ?: error("Base target with name $base not found")
targets[name] = parentTarget.builder()
.apply { "@baseTarget" to base }
.apply { "@baseTarget" put base }
.apply(block)
.seal()
}
fun WorkspaceBuilder.task(task: Task<Any>) {
this.tasks.add(task)
}
fun <T : Any> WorkspaceBuilder.task(
name: String,
type: KClass<out T>,
builder: TaskBuilder<T>.() -> Unit
): Task<T> = TaskBuilder(name.toName(), type).apply(builder).build().also { tasks.add(it) }
inline fun <reified T : Any> WorkspaceBuilder.task(
name: String,
noinline builder: TaskBuilder<T>.() -> Unit
): Task<T> = task(name, T::class, builder)
@JvmName("rawTask")
fun WorkspaceBuilder.task(
name: String,
builder: TaskBuilder<Any>.() -> Unit
): Task<Any> = task(name, Any::class, builder)
/**
* A builder for a simple workspace
*/
class SimpleWorkspaceBuilder(override val parentContext: Context) : WorkspaceBuilder {
override var context: Context = parentContext
override var data = DataTreeBuilder(Any::class)
override var data: DataTreeBuilder<Any> = DataTreeBuilder(Any::class)
override var tasks: MutableSet<Task<Any>> = HashSet()
override var targets: MutableMap<String, Meta> = HashMap()

29
dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/WorkspacePlugin.kt
View File

@ -1,19 +1,42 @@
package hep.dataforge.workspace
import hep.dataforge.context.AbstractPlugin
import hep.dataforge.context.toMap
import hep.dataforge.names.Name
import hep.dataforge.names.toName
import kotlin.reflect.KClass
/**
* An abstract plugin with some additional boilerplate to effectively work with workspace context
*/
abstract class WorkspacePlugin : AbstractPlugin() {
abstract val tasks: Collection<Task<*>>
private val _tasks = HashSet<Task<*>>()
val tasks: Collection<Task<*>> get() = _tasks
override fun provideTop(target: String): Map<Name, Any> {
return when(target){
Task.TYPE -> tasks.associateBy { it.name.toName() }
return when (target) {
Task.TYPE -> tasks.toMap()
else -> emptyMap()
}
}
fun task(task: Task<*>){
_tasks.add(task)
}
fun <T : Any> task(
name: String,
type: KClass<out T>,
builder: TaskBuilder<T>.() -> Unit
): GenericTask<T> = TaskBuilder(name.toName(), type).apply(builder).build().also {
_tasks.add(it)
}
inline fun <reified T : Any> task(
name: String,
noinline builder: TaskBuilder<T>.() -> Unit
) = task(name, T::class, builder)
//
////TODO add delegates to build gradle-like tasks
}

123
dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/fileData.kt
View File

@ -1,12 +1,14 @@
package hep.dataforge.workspace
import hep.dataforge.data.Data
import hep.dataforge.data.DataNode
import hep.dataforge.data.DataTreeBuilder
import hep.dataforge.data.datum
import hep.dataforge.descriptors.NodeDescriptor
import hep.dataforge.io.*
import hep.dataforge.meta.EmptyMeta
import hep.dataforge.meta.Meta
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.coroutineScope
import kotlinx.coroutines.withContext
import kotlinx.io.nio.asInput
import kotlinx.io.nio.asOutput
@ -16,71 +18,94 @@ import java.nio.file.StandardOpenOption
import kotlin.reflect.KClass
/**
* Read meta from file in a given [format]
* Read meta from file in a given [MetaFormat]
*/
suspend fun Path.readMeta(format: MetaFormat, descriptor: NodeDescriptor? = null): Meta {
return withContext(Dispatchers.IO) {
format.run {
Files.newByteChannel(this@readMeta, StandardOpenOption.READ)
.asInput()
.readMeta(descriptor)
}
}
fun MetaFormat.readMetaFile(path: Path, descriptor: NodeDescriptor? = null): Meta {
return Files.newByteChannel(path, StandardOpenOption.READ)
.asInput()
.readMeta(descriptor)
}
/**
* Write meta to file in a given [format]
* Write meta to file using given [MetaFormat]
*/
suspend fun Meta.write(path: Path, format: MetaFormat, descriptor: NodeDescriptor? = null) {
withContext(Dispatchers.IO) {
format.run {
Files.newByteChannel(path, StandardOpenOption.WRITE, StandardOpenOption.CREATE_NEW)
.asOutput()
.writeMeta(this@write, descriptor)
}
}
fun MetaFormat.writeMetaFile(path: Path, meta: Meta, descriptor: NodeDescriptor? = null) {
return Files.newByteChannel(path, StandardOpenOption.WRITE, StandardOpenOption.CREATE_NEW)
.asOutput()
.writeMeta(meta, descriptor)
}
/**
* Read data with supported envelope format and binary format. If envelope format is null, then read binary directly from file.
* The operation is blocking since it must read meta header. The reading of envelope body is lazy
* @param type explicit type of data read
* @param format binary format
* @param envelopeFormat the format of envelope. If null, file is read directly
* @param dataFormat binary format
* @param envelopeFormatFactory the format of envelope. If null, file is read directly
* @param metaFile the relative file for optional meta override
* @param metaFileFormat the meta format for override
*/
suspend fun <T : Any> Path.readData(
fun <T : Any> IOPlugin.readData(
path: Path,
type: KClass<out T>,
format: IOFormat<T>,
envelopeFormat: EnvelopeFormat? = null,
metaFile: Path = resolveSibling("$fileName.meta"),
metaFileFormat: MetaFormat = JsonMetaFormat
dataFormat: IOFormat<T>,
envelopeFormatFactory: EnvelopeFormatFactory? = null,
metaFile: Path = path.resolveSibling("${path.fileName}.meta"),
metaFileFormat: MetaFormat = JsonMetaFormat.default
): Data<T> {
return coroutineScope {
val externalMeta = if (Files.exists(metaFile)) {
metaFile.readMeta(metaFileFormat)
} else {
null
}
if (envelopeFormat == null) {
Data(type, externalMeta ?: EmptyMeta) {
withContext(Dispatchers.IO) {
format.run {
Files.newByteChannel(this@readData, StandardOpenOption.READ)
.asInput()
.readThis()
}
val externalMeta = if (Files.exists(metaFile)) {
metaFileFormat.readMetaFile(metaFile)
} else {
null
}
return if (envelopeFormatFactory == null) {
Data(type, externalMeta ?: EmptyMeta) {
withContext(Dispatchers.IO) {
dataFormat.run {
Files.newByteChannel(path, StandardOpenOption.READ)
.asInput()
.readObject()
}
}
} else {
withContext(Dispatchers.IO) {
readEnvelope(envelopeFormat).let {
if (externalMeta == null) {
it
} else {
it.withMetaLayers(externalMeta)
}
}.toData(type, format)
}
} else {
readEnvelopeFile(path, envelopeFormatFactory).let {
if (externalMeta == null) {
it
} else {
it.withMetaLayers(externalMeta)
}
}.toData(type, dataFormat)
}
}
//TODO wants multi-receiver
fun <T : Any> DataTreeBuilder<T>.file(
plugin: IOPlugin,
path: Path,
dataFormat: IOFormat<T>,
envelopeFormatFactory: EnvelopeFormatFactory? = null
) {
plugin.run {
val data = readData(path, type, dataFormat, envelopeFormatFactory)
val name = path.fileName.toString().replace(".df", "")
datum(name, data)
}
}
/**
* Read the directory as a data node
*/
fun <T : Any> IOPlugin.readDataNode(
path: Path,
type: KClass<out T>,
dataFormat: IOFormat<T>,
envelopeFormatFactory: EnvelopeFormatFactory? = null
): DataNode<T> {
if (!Files.isDirectory(path)) error("Provided path $this is not a directory")
return DataNode(type) {
Files.list(path).forEach { path ->
if (!path.fileName.toString().endsWith(".meta")) {
file(this@readDataNode,path, dataFormat, envelopeFormatFactory)
}
}
}

95
dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/DataPropagationTest.kt Normal file
View File

@ -0,0 +1,95 @@
package hep.dataforge.workspace
import hep.dataforge.context.Context
import hep.dataforge.context.PluginFactory
import hep.dataforge.context.PluginTag
import hep.dataforge.data.*
import hep.dataforge.meta.Meta
import hep.dataforge.names.asName
import org.junit.Test
import kotlin.reflect.KClass
import kotlin.test.assertEquals
class DataPropagationTestPlugin : WorkspacePlugin() {
override val tag: PluginTag = Companion.tag
val testAllData = task("allData", Int::class) {
model {
allData()
}
transform<Int> { data ->
return@transform DataNode {
val result = data.dataSequence().map { it.second.get() }.reduce { acc, pair -> acc + pair }
set("result".asName(), Data { result })
}
}
}
val testSingleData = task("singleData", Int::class) {
model {
data("myData\\[12\\]")
}
transform<Int> { data ->
return@transform DataNode {
val result = data.dataSequence().map { it.second.get() }.reduce { acc, pair -> acc + pair }
set("result".asName(), Data { result })
}
}
}
val testAllRegexData = task("allRegexData", Int::class) {
model {
data(pattern = "myData.*")
}
transform<Int> { data ->
return@transform DataNode {
val result = data.dataSequence().map { it.second.get() }.reduce { acc, pair -> acc + pair }
set("result".asName(), Data { result })
}
}
}
companion object : PluginFactory<DataPropagationTestPlugin> {
override val type: KClass<out DataPropagationTestPlugin> = DataPropagationTestPlugin::class
override fun invoke(meta: Meta, context: Context): DataPropagationTestPlugin =
DataPropagationTestPlugin(meta)
override val tag: PluginTag = PluginTag("Test")
}
}
class DataPropagationTest {
val testWorkspace = Workspace {
context {
plugin(DataPropagationTestPlugin())
}
data {
repeat(100) {
static("myData[$it]", it)
}
}
}
@Test
fun testAllData() {
val node = testWorkspace.run("Test.allData")
assertEquals(4950, node.first()!!.get())
}
@Test
fun testAllRegexData() {
val node = testWorkspace.run("Test.allRegexData")
assertEquals(4950, node.first()!!.get())
}
@Test
fun testSingleData() {
val node = testWorkspace.run("Test.singleData")
assertEquals(12, node.first()!!.get())
}
}

109
dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/SimpleWorkspaceTest.kt
View File

@ -1,10 +1,12 @@
package hep.dataforge.workspace
import hep.dataforge.context.PluginTag
import hep.dataforge.data.first
import hep.dataforge.data.get
import hep.dataforge.data.*
import hep.dataforge.meta.boolean
import hep.dataforge.meta.builder
import hep.dataforge.meta.get
import hep.dataforge.meta.int
import hep.dataforge.names.plus
import org.junit.Test
import kotlin.test.assertEquals
import kotlin.test.assertTrue
@ -14,30 +16,36 @@ class SimpleWorkspaceTest {
val testPlugin = object : WorkspacePlugin() {
override val tag: PluginTag = PluginTag("test")
val contextTask = Workspace.task("test") {
pipe<Any, Unit> {
val contextTask = task("test", Any::class) {
map<Any> {
context.logger.info { "Test: $it" }
}
}
override val tasks: Collection<Task<*>> = listOf(contextTask)
}
val workspace = SimpleWorkspace.build {
val workspace = Workspace {
context {
plugin(testPlugin)
}
repeat(100) {
static("myData[$it]", it)
data {
repeat(100) {
static("myData[$it]", it)
}
}
task("square") {
val filterTask = task<Int>("filterOne") {
model {
allData()
data("myData\\[12\\]")
}
pipe<Int, Int> { data ->
map<Int>{
it
}
}
val square = task<Int>("square") {
map<Int> { data ->
if (meta["testFlag"].boolean == true) {
println("flag")
}
@ -46,30 +54,55 @@ class SimpleWorkspaceTest {
}
}
task("sum") {
model {
dependsOn("square")
val linear = task<Int>("linear") {
map<Int> { data ->
context.logger.info { "Starting linear on $data" }
data * 2 + 1
}
join<Int, Int> { data ->
}
val fullSquare = task<Int>("fullsquare") {
model {
val squareDep = dependsOn(square, placement = "square")
val linearDep = dependsOn(linear, placement = "linear")
}
transform { data ->
val squareNode = data["square"].node!!.cast<Int>()//squareDep()
val linearNode = data["linear"].node!!.cast<Int>()//linearDep()
return@transform DataNode(Int::class) {
squareNode.dataSequence().forEach { (name, _) ->
val newData = Data {
val squareValue = squareNode[name].data!!.get()
val linearValue = linearNode[name].data!!.get()
squareValue + linearValue
}
set(name, newData)
}
}
}
}
task<Int>("sum") {
model {
dependsOn(square)
}
reduce<Int> { data ->
context.logger.info { "Starting sum" }
data.values.sum()
}
}
task("average") {
model {
allData()
}
joinByGroup<Int, Double> { context ->
val average = task<Double>("average") {
reduceByGroup<Int> { env ->
group("even", filter = { name, _ -> name.toString().toInt() % 2 == 0 }) {
result { data ->
context.logger.info { "Starting even" }
env.context.logger.info { "Starting even" }
data.values.average()
}
}
group("odd", filter = { name, _ -> name.toString().toInt() % 2 == 1 }) {
result { data ->
context.logger.info { "Starting odd" }
env.context.logger.info { "Starting odd" }
data.values.average()
}
}
@ -78,13 +111,25 @@ class SimpleWorkspaceTest {
task("delta") {
model {
dependsOn("average")
dependsOn(average)
}
join<Double, Double> { data ->
reduce<Double> { data ->
data["even"]!! - data["odd"]!!
}
}
val customPipeTask = task<Int>("custom") {
mapAction<Int> {
meta = meta.builder().apply {
"newValue" put 22
}
name += "new"
result {
meta["value"].int ?: 0 + it
}
}
}
target("empty") {}
}
@ -97,7 +142,7 @@ class SimpleWorkspaceTest {
@Test
fun testMetaPropagation() {
val node = workspace.run("sum") { "testFlag" to true }
val node = workspace.run("sum") { "testFlag" put true }
val res = node.first()?.get()
}
@ -107,4 +152,16 @@ class SimpleWorkspaceTest {
assertTrue { tasks["test.test"] != null }
//val node = workspace.run("test.test", "empty")
}
@Test
fun testFullSquare() {
val node = workspace.run("fullsquare")
println(node.toMeta())
}
@Test
fun testGather() {
val node = workspace.run("filterOne")
assertEquals(12, node.first()?.get())
}
}

BIN
gradle/wrapper/gradle-wrapper.jar vendored
View File

Binary file not shown.

2
gradle/wrapper/gradle-wrapper.properties vendored
View File

@ -1,5 +1,5 @@
distributionBase=GRADLE_USER_HOME
distributionPath=wrapper/dists
distributionUrl=https\://services.gradle.org/distributions/gradle-5.1.1-bin.zip
distributionUrl=https\://services.gradle.org/distributions/gradle-5.6.2-bin.zip
zipStoreBase=GRADLE_USER_HOME
zipStorePath=wrapper/dists

Some files were not shown because too many files have changed in this diff Show More