kscience/dataforge-core
8
0
Fork 0
Code Issues Pull Requests 1 Packages Projects Releases 12 Wiki Activity

Merge pull request #7 from altavir/dev

Browse Source 
Dev
This commit is contained in:
Alexander Nozik 2019-05-09 13:58:11 +03:00 committed by GitHub
commit a7e1009d2b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
91 changed files with 3022 additions and 842 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.gitignore vendored
View File

@ -1,9 +1,9 @@
.idea/
*.iws
*/out/**
out/
.gradle
*/build/**
build/
!gradle-wrapper.jar

134
build.gradle.kts
View File

@ -1,142 +1,18 @@
import org.jetbrains.kotlin.gradle.dsl.KotlinMultiplatformExtension
import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
buildscript {
val kotlinVersion: String by rootProject.extra("1.3.21")
val ioVersion: String by rootProject.extra("0.1.5")
val coroutinesVersion: String by rootProject.extra("1.1.1")
val atomicfuVersion: String by rootProject.extra("0.12.1")
val dokkaVersion: String by rootProject.extra("0.9.17")
val serializationVersion: String by rootProject.extra("0.10.0")
repositories {
jcenter()
maven("https://dl.bintray.com/kotlin/kotlin-eap")
}
dependencies {
classpath("org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlinVersion")
classpath("org.jfrog.buildinfo:build-info-extractor-gradle:4+")
classpath("com.jfrog.bintray.gradle:gradle-bintray-plugin:1.8.4")
classpath("org.jetbrains.dokka:dokka-gradle-plugin:$dokkaVersion")
classpath("org.jetbrains.kotlin:kotlin-frontend-plugin:0.0.45")
classpath("org.openjfx:javafx-plugin:0.0.7")
}
}
plugins {
id("com.jfrog.artifactory") version "4.8.1" apply false
// id("org.jetbrains.kotlin.multiplatform") apply false
}
val dataforgeVersion by extra("0.1.2")
allprojects {
apply(plugin = "maven")
apply(plugin = "maven-publish")
apply(plugin = "com.jfrog.artifactory")
repositories {
jcenter()
maven("https://kotlin.bintray.com/kotlinx")
}
group = "hep.dataforge"
version = "0.1.1-dev-5"
// apply bintray configuration
apply(from = "${rootProject.rootDir}/gradle/bintray.gradle")
//apply artifactory configuration
apply(from = "${rootProject.rootDir}/gradle/artifactory.gradle")
version = dataforgeVersion
}
subprojects {
// dokka {
// outputFormat = "html"
// outputDirectory = javadoc.destinationDir
// }
//
// task dokkaJar (type: Jar, dependsOn: dokka) {
// from javadoc . destinationDir
// classifier = "javadoc"
// }
// Create empty jar for sources classifier to satisfy maven requirements
val stubSources by tasks.registering(Jar::class) {
archiveClassifier.set("sources")
//from(sourceSets.main.get().allSource)
if (name.startsWith("dataforge")) {
apply(plugin = "npm-bintray")
apply(plugin = "npm-artifactory")
}
// Create empty jar for javadoc classifier to satisfy maven requirements
val stubJavadoc by tasks.registering(Jar::class) {
archiveClassifier.set("javadoc")
}
tasks.withType<KotlinCompile> {
kotlinOptions{
jvmTarget = "1.8"
}
}
afterEvaluate {
extensions.findByType<KotlinMultiplatformExtension>()?.apply {
jvm {
compilations.all {
kotlinOptions {
jvmTarget = "1.8"
}
}
}
js {
compilations.all {
tasks.getByName(compileKotlinTaskName) {
kotlinOptions {
metaInfo = true
sourceMap = true
sourceMapEmbedSources = "always"
moduleKind = "umd"
}
}
}
configure(listOf(compilations["main"])) {
tasks.getByName(compileKotlinTaskName) {
kotlinOptions {
main = "call"
}
}
}
}
targets.all {
sourceSets.all {
languageSettings.progressiveMode = true
}
}
configure<PublishingExtension> {
publications.filterIsInstance<MavenPublication>().forEach { publication ->
if (publication.name == "kotlinMultiplatform") {
// for our root metadata publication, set artifactId with a package and project name
publication.artifactId = project.name
} else {
// for targets, set artifactId with a package, project name and target name (e.g. iosX64)
publication.artifactId = "${project.name}-${publication.name}"
}
}
targets.all {
val publication = publications.findByName(name) as MavenPublication
// Patch publications with fake javadoc
publication.artifact(stubJavadoc.get())
}
}
}
}
}

20
buildSrc/build.gradle.kts Normal file
View File

@ -0,0 +1,20 @@
plugins {
`kotlin-dsl`
}
repositories {
gradlePluginPortal()
jcenter()
}
val kotlinVersion = "1.3.31"
// Add plugins used in buildSrc as dependencies, also we should specify version only here
dependencies {
implementation("org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlinVersion")
implementation("org.jfrog.buildinfo:build-info-extractor-gradle:4.9.5")
implementation("com.jfrog.bintray.gradle:gradle-bintray-plugin:1.8.4")
implementation("org.jetbrains.dokka:dokka-gradle-plugin:0.9.18")
implementation("com.moowork.gradle:gradle-node-plugin:1.3.1")
implementation("org.openjfx:javafx-plugin:0.0.7")
}

0
buildSrc/settings.gradle.kts Normal file
View File

9
buildSrc/src/main/kotlin/Versions.kt Normal file
View File

@ -0,0 +1,9 @@
// Instead of defining runtime properties and use them dynamically
// define version in buildSrc and have autocompletion and compile-time check
// Also dependencies itself can be moved here
object Versions {
val ioVersion = "0.1.8"
val coroutinesVersion = "1.2.1"
val atomicfuVersion = "0.12.6"
val serializationVersion = "0.11.0"
}

59
buildSrc/src/main/kotlin/dokka-publish.gradle.kts Normal file
View File

@ -0,0 +1,59 @@
import org.jetbrains.dokka.gradle.DokkaTask
plugins {
kotlin("multiplatform")
id("org.jetbrains.dokka")
`maven-publish`
}
kotlin {
val dokka by tasks.getting(DokkaTask::class) {
outputFormat = "html"
outputDirectory = "$buildDir/javadoc"
jdkVersion = 8
kotlinTasks {
// dokka fails to retrieve sources from MPP-tasks so we only define the jvm task
listOf(tasks.getByPath("compileKotlinJvm"))
}
sourceRoot {
// assuming only single source dir
path = sourceSets["commonMain"].kotlin.srcDirs.first().toString()
platforms = listOf("Common")
}
// although the JVM sources are now taken from the task,
// we still define the jvm source root to get the JVM marker in the generated html
sourceRoot {
// assuming only single source dir
path = sourceSets["jvmMain"].kotlin.srcDirs.first().toString()
platforms = listOf("JVM")
}
}
val javadocJar by tasks.registering(Jar::class) {
dependsOn(dokka)
archiveClassifier.set("javadoc")
from("$buildDir/javadoc")
}
publishing {
// publications.filterIsInstance<MavenPublication>().forEach { publication ->
// if (publication.name == "kotlinMultiplatform") {
// // for our root metadata publication, set artifactId with a package and project name
// publication.artifactId = project.name
// } else {
// // for targets, set artifactId with a package, project name and target name (e.g. iosX64)
// publication.artifactId = "${project.name}-${publication.name}"
// }
// }
targets.all {
val publication = publications.findByName(name) as MavenPublication
// Patch publications with fake javadoc
publication.artifact(javadocJar.get())
}
}
}

44
buildSrc/src/main/kotlin/js-test.gradle.kts Normal file
View File

@ -0,0 +1,44 @@
import com.moowork.gradle.node.npm.NpmTask
import com.moowork.gradle.node.task.NodeTask
import org.gradle.kotlin.dsl.*
import org.jetbrains.kotlin.gradle.tasks.Kotlin2JsCompile
plugins {
id("com.moowork.node")
kotlin("multiplatform")
}
node {
nodeModulesDir = file("$buildDir/node_modules")
}
val compileKotlinJs by tasks.getting(Kotlin2JsCompile::class)
val compileTestKotlinJs by tasks.getting(Kotlin2JsCompile::class)
val populateNodeModules by tasks.registering(Copy::class) {
dependsOn(compileKotlinJs)
from(compileKotlinJs.destinationDir)
kotlin.js().compilations["test"].runtimeDependencyFiles.forEach {
if (it.exists() && !it.isDirectory) {
from(zipTree(it.absolutePath).matching { include("*.js") })
}
}
into("$buildDir/node_modules")
}
val installMocha by tasks.registering(NpmTask::class) {
setWorkingDir(buildDir)
setArgs(listOf("install", "mocha"))
}
val runMocha by tasks.registering(NodeTask::class) {
dependsOn(compileTestKotlinJs, populateNodeModules, installMocha)
setScript(file("$buildDir/node_modules/mocha/bin/mocha"))
setArgs(listOf(compileTestKotlinJs.outputFile))
}
tasks["jsTest"].dependsOn(runMocha)

38
buildSrc/src/main/kotlin/npm-artifactory.gradle.kts Normal file
View File

@ -0,0 +1,38 @@
import groovy.lang.GroovyObject
import org.jfrog.gradle.plugin.artifactory.dsl.PublisherConfig
import org.jfrog.gradle.plugin.artifactory.dsl.ResolverConfig
plugins {
id("com.jfrog.artifactory")
}
artifactory {
val artifactoryUser: String? by project
val artifactoryPassword: String? by project
val artifactoryContextUrl = "http://npm.mipt.ru:8081/artifactory"
setContextUrl(artifactoryContextUrl)//The base Artifactory URL if not overridden by the publisher/resolver
publish(delegateClosureOf<PublisherConfig> {
repository(delegateClosureOf<GroovyObject> {
setProperty("repoKey", "gradle-dev-local")
setProperty("username", artifactoryUser)
setProperty("password", artifactoryPassword)
})
defaults(delegateClosureOf<GroovyObject>{
invokeMethod("publications", arrayOf("jvm", "js", "kotlinMultiplatform", "metadata"))
//TODO: This property is not available for ArtifactoryTask
//setProperty("publishBuildInfo", false)
setProperty("publishArtifacts", true)
setProperty("publishPom", true)
setProperty("publishIvy", false)
})
})
resolve(delegateClosureOf<ResolverConfig> {
repository(delegateClosureOf<GroovyObject> {
setProperty("repoKey", "gradle-dev")
setProperty("username", artifactoryUser)
setProperty("password", artifactoryPassword)
})
})
}

97
buildSrc/src/main/kotlin/npm-bintray.gradle.kts Normal file
View File

@ -0,0 +1,97 @@
@file:Suppress("UnstableApiUsage")
import com.jfrog.bintray.gradle.BintrayExtension.PackageConfig
import com.jfrog.bintray.gradle.BintrayExtension.VersionConfig
// Old bintray.gradle script converted to real Gradle plugin (precompiled script plugin)
// It now has own dependencies and support type safe accessors
// Syntax is pretty close to what we had in Groovy
// (excluding Property.set and bintray dynamic configs)
plugins {
id("com.jfrog.bintray")
`maven-publish`
}
val vcs = "https://github.com/mipt-npm/kmath"
// Configure publishing
publishing {
repositories {
maven("https://bintray.com/mipt-npm/scientifik")
}
// Process each publication we have in this project
publications.filterIsInstance<MavenPublication>().forEach { publication ->
// use type safe pom config GSL insterad of old dynamic
publication.pom {
name.set(project.name)
description.set(project.description)
url.set(vcs)
licenses {
license {
name.set("The Apache Software License, Version 2.0")
url.set("http://www.apache.org/licenses/LICENSE-2.0.txt")
distribution.set("repo")
}
}
developers {
developer {
id.set("MIPT-NPM")
name.set("MIPT nuclear physics methods laboratory")
organization.set("MIPT")
organizationUrl.set("http://npm.mipt.ru")
}
}
scm {
url.set(vcs)
}
}
}
}
bintray {
// delegates for runtime properties
val bintrayUser: String? by project
val bintrayApiKey: String? by project
user = bintrayUser ?: System.getenv("BINTRAY_USER")
key = bintrayApiKey ?: System.getenv("BINTRAY_API_KEY")
publish = true
override = true // for multi-platform Kotlin/Native publishing
// We have to use delegateClosureOf because bintray supports only dynamic groovy syntax
// this is a problem of this plugin
pkg(delegateClosureOf<PackageConfig> {
userOrg = "mipt-npm"
repo = "scientifik"
name = "scientifik.kmath"
issueTrackerUrl = "https://github.com/mipt-npm/kmath/issues"
setLicenses("Apache-2.0")
vcsUrl = vcs
version(delegateClosureOf<VersionConfig> {
name = project.version.toString()
vcsTag = project.version.toString()
released = java.util.Date().toString()
})
})
tasks {
bintrayUpload {
dependsOn(publishToMavenLocal)
doFirst {
setPublications(project.publishing.publications
.filterIsInstance<MavenPublication>()
.filter { !it.name.contains("-test") && it.name != "kotlinMultiplatform" }
.map {
println("""Uploading artifact "${it.groupId}:${it.artifactId}:${it.version}" from publication "${it.name}""")
it.name //https://github.com/bintray/gradle-bintray-plugin/issues/256
})
}
}
}
}

86
buildSrc/src/main/kotlin/npm-multiplatform.gradle.kts Normal file
View File

@ -0,0 +1,86 @@
import org.gradle.kotlin.dsl.*
plugins {
kotlin("multiplatform")
`maven-publish`
}
kotlin {
jvm {
compilations.all {
kotlinOptions {
jvmTarget = "1.8"
}
}
}
js {
compilations.all {
kotlinOptions {
metaInfo = true
sourceMap = true
sourceMapEmbedSources = "always"
moduleKind = "commonjs"
}
}
compilations.named("main") {
kotlinOptions {
main = "call"
}
}
}
sourceSets {
val commonMain by getting {
dependencies {
api(kotlin("stdlib"))
}
}
val commonTest by getting {
dependencies {
implementation(kotlin("test-common"))
implementation(kotlin("test-annotations-common"))
}
}
val jvmMain by getting {
dependencies {
api(kotlin("stdlib-jdk8"))
}
}
val jvmTest by getting {
dependencies {
implementation(kotlin("test"))
implementation(kotlin("test-junit"))
}
}
val jsMain by getting {
dependencies {
api(kotlin("stdlib-js"))
}
}
val jsTest by getting {
dependencies {
implementation(kotlin("test-js"))
}
}
}
targets.all {
sourceSets.all {
languageSettings.progressiveMode = true
languageSettings.enableLanguageFeature("InlineClasses")
}
}
apply(plugin = "dokka-publish")
// Apply JS test configuration
val runJsTests by ext(false)
if (runJsTests) {
apply(plugin = "js-test")
}
}

5
dataforge-context/build.gradle.kts
View File

@ -1,10 +1,10 @@
plugins {
kotlin("multiplatform")
`npm-multiplatform`
}
description = "Context and provider definitions"
val coroutinesVersion: String by rootProject.extra
val coroutinesVersion: String = Versions.coroutinesVersion
kotlin {
jvm()
@ -22,6 +22,7 @@ kotlin {
val jvmMain by getting {
dependencies {
api("io.github.microutils:kotlin-logging:1.6.10")
api("ch.qos.logback:logback-classic:1.2.3")
api("org.jetbrains.kotlinx:kotlinx-coroutines-core:$coroutinesVersion")
}
}

11
dataforge-context/src/commonMain/kotlin/hep/dataforge/context/AbstractPlugin.kt
View File

@ -1,16 +1,15 @@
package hep.dataforge.context
import hep.dataforge.meta.Config
import hep.dataforge.meta.EmptyMeta
import hep.dataforge.meta.Meta
import hep.dataforge.names.Name
abstract class AbstractPlugin : Plugin {
abstract class AbstractPlugin(override val meta: Meta = EmptyMeta) : Plugin {
private var _context: Context? = null
override val context: Context
get() = _context ?: error("Plugin $tag is not attached")
override val config = Config()
override fun attach(context: Context) {
this._context = context
}
@ -19,9 +18,7 @@ abstract class AbstractPlugin : Plugin {
this._context = null
}
//TODO make configuration activation-safe
override fun provideTop(target: String, name: Name): Any? = null
override fun listTop(target: String): Sequence<Name> = emptySequence()
override fun listNames(target: String): Sequence<Name> = emptySequence()
}

18
dataforge-context/src/commonMain/kotlin/hep/dataforge/context/Context.kt
View File

@ -2,9 +2,10 @@ package hep.dataforge.context
import hep.dataforge.meta.*
import hep.dataforge.names.Name
import hep.dataforge.names.appendLeft
import hep.dataforge.names.toName
import hep.dataforge.provider.Provider
import hep.dataforge.provider.provideAll
import hep.dataforge.provider.top
import hep.dataforge.values.Value
import kotlinx.coroutines.CoroutineScope
import mu.KLogger
@ -66,10 +67,10 @@ open class Context(final override val name: String, val parent: Context? = Globa
}
}
override fun listTop(target: String): Sequence<Name> {
override fun listNames(target: String): Sequence<Name> {
return when (target) {
Plugin.PLUGIN_TARGET -> plugins.asSequence().map { it.name.toName() }
Value.TYPE -> properties.asValueSequence().map { it.first }
Value.TYPE -> properties.values().map { it.first }
else -> emptySequence()
}
}
@ -118,12 +119,13 @@ open class Context(final override val name: String, val parent: Context? = Globa
/**
* A sequences of all objects provided by plugins with given target and type
*/
fun Context.members(target: String): Sequence<Any> =
plugins.asSequence().flatMap { it.provideAll(target) }
fun Context.content(target: String): Map<Name, Any> = content<Any>(target)
@JvmName("typedMembers")
inline fun <reified T : Any> Context.members(target: String) =
members(target).filterIsInstance<T>()
@JvmName("typedContent")
inline fun <reified T : Any> Context.content(target: String): Map<Name, T> =
plugins.flatMap { plugin ->
plugin.top<T>(target).entries.map { (it.key.appendLeft(plugin.name)) to it.value }
}.associate { it }
/**

9
dataforge-context/src/commonMain/kotlin/hep/dataforge/context/ContextBuilder.kt
View File

@ -1,8 +1,7 @@
package hep.dataforge.context
import hep.dataforge.meta.Config
import hep.dataforge.meta.MetaBuilder
import hep.dataforge.meta.configure
import hep.dataforge.meta.buildMeta
/**
* A convenience builder for context
@ -19,11 +18,11 @@ class ContextBuilder(var name: String = "@anonimous", val parent: Context = Glob
plugins.add(plugin)
}
fun plugin(tag: PluginTag, action: Config.() -> Unit) {
plugins.add(PluginRepository.fetch(tag).configure(action))
fun plugin(tag: PluginTag, action: MetaBuilder.() -> Unit) {
plugins.add(PluginRepository.fetch(tag, buildMeta(action)))
}
fun plugin(name: String, group: String = "", version: String = "", action: Config.() -> Unit) {
fun plugin(name: String, group: String = "", version: String = "", action: MetaBuilder.() -> Unit) {
plugin(PluginTag(name, group, version), action)
}

12
dataforge-context/src/commonMain/kotlin/hep/dataforge/context/Plugin.kt
View File

@ -1,6 +1,5 @@
package hep.dataforge.context
import hep.dataforge.meta.Configurable
import hep.dataforge.meta.Meta
import hep.dataforge.meta.MetaRepr
import hep.dataforge.meta.buildMeta
@ -22,7 +21,7 @@ import hep.dataforge.provider.Provider
*
* @author Alexander Nozik
*/
interface Plugin : Named, ContextAware, Provider, MetaRepr, Configurable {
interface Plugin : Named, ContextAware, Provider, MetaRepr {
/**
* Get tag for this plugin
@ -31,13 +30,14 @@ interface Plugin : Named, ContextAware, Provider, MetaRepr, Configurable {
*/
val tag: PluginTag
val meta: Meta
/**
* The name of this plugin ignoring version and group
*
* @return
*/
override val name: String
get() = tag.name
override val name: String get() = tag.name
/**
* Plugin dependencies which are required to attach this plugin. Plugin
@ -46,7 +46,7 @@ interface Plugin : Named, ContextAware, Provider, MetaRepr, Configurable {
*
* @return
*/
fun dependsOn(): List<PluginTag> = emptyList()
fun dependsOn(): List<PluginFactory<*>> = emptyList()
/**
* Start this plugin and attach registration info to the context. This method
@ -67,7 +67,7 @@ interface Plugin : Named, ContextAware, Provider, MetaRepr, Configurable {
"context" to context.name
"type" to this::class.simpleName
"tag" to tag
"meta" to config
"meta" to meta
}
companion object {

22
dataforge-context/src/commonMain/kotlin/hep/dataforge/context/PluginManager.kt
View File

@ -1,6 +1,9 @@
package hep.dataforge.context
import hep.dataforge.meta.*
import hep.dataforge.meta.EmptyMeta
import hep.dataforge.meta.Meta
import hep.dataforge.meta.MetaBuilder
import hep.dataforge.meta.buildMeta
import kotlin.reflect.KClass
/**
@ -112,12 +115,21 @@ class PluginManager(override val context: Context) : ContextAware, Iterable<Plug
fun load(tag: PluginTag, meta: Meta = EmptyMeta): Plugin {
val loaded = get(tag, false)
return when {
loaded == null -> load(PluginRepository.fetch(tag)).configure(meta)
loaded.config == meta -> loaded // if meta is the same, return existing plugin
loaded == null -> load(PluginRepository.fetch(tag,meta))
loaded.meta == meta -> loaded // if meta is the same, return existing plugin
else -> throw RuntimeException("Can't load plugin with tag $tag. Plugin with this tag and different configuration already exists in context.")
}
}
fun load(factory: PluginFactory<*>, meta: Meta = EmptyMeta): Plugin{
val loaded = get(factory.tag, false)
return when {
loaded == null -> load(factory(meta))
loaded.meta == meta -> loaded // if meta is the same, return existing plugin
else -> throw RuntimeException("Can't load plugin with tag ${factory.tag}. Plugin with this tag and different configuration already exists in context.")
}
}
/**
* Load plugin by its class and meta. Ignore if plugin with this meta is already loaded.
* Throw an exception if there exists plugin with the same type, but different meta
@ -126,7 +138,7 @@ class PluginManager(override val context: Context) : ContextAware, Iterable<Plug
val loaded = get(type, false)
return when {
loaded == null -> {
val plugin = PluginRepository.list().first { it.type == type }.build(meta)
val plugin = PluginRepository.list().first { it.type == type }.invoke(meta)
if (type.isInstance(plugin)) {
@Suppress("UNCHECKED_CAST")
load(plugin as T)
@ -134,7 +146,7 @@ class PluginManager(override val context: Context) : ContextAware, Iterable<Plug
error("Corrupt type information in plugin repository")
}
}
loaded.config == meta -> loaded // if meta is the same, return existing plugin
loaded.meta == meta -> loaded // if meta is the same, return existing plugin
else -> throw RuntimeException("Can't load plugin with type $type. Plugin with this type and different configuration already exists in context.")
}
}

37
dataforge-context/src/commonMain/kotlin/hep/dataforge/context/PluginRepository.kt
View File

@ -1,48 +1,49 @@
package hep.dataforge.context
import hep.dataforge.meta.EmptyMeta
import hep.dataforge.meta.Meta
import hep.dataforge.meta.configure
import kotlin.reflect.KClass
interface PluginFactory {
interface PluginFactory<T : Plugin> {
val tag: PluginTag
val type: KClass<out Plugin>
fun build(): Plugin
val type: KClass<out T>
operator fun invoke(meta: Meta = EmptyMeta): T
}
fun PluginFactory.build(meta: Meta) = build().configure(meta)
expect object PluginRepository {
fun register(factory: PluginFactory)
fun register(factory: PluginFactory<*>)
/**
* List plugins available in the repository
*/
fun list(): Sequence<PluginFactory>
fun list(): Sequence<PluginFactory<*>>
}
/**
* Fetch specific plugin and instantiate it with given meta
*/
fun PluginRepository.fetch(tag: PluginTag): Plugin =
PluginRepository.list().find { it.tag.matches(tag) }?.build()
?: error("Plugin with tag $tag not found in the repository")
fun PluginRepository.fetch(tag: PluginTag, meta: Meta = EmptyMeta): Plugin =
list().find { it.tag.matches(tag) }?.invoke(meta) ?: error("Plugin with tag $tag not found in the repository")
fun PluginRepository.register(tag: PluginTag, type: KClass<out Plugin>, constructor: () -> Plugin) {
val factory = object : PluginFactory {
fun <T : Plugin> PluginRepository.register(
tag: PluginTag,
type: KClass<out T>,
constructor: (Meta) -> T
): PluginFactory<T> {
val factory = object : PluginFactory<T> {
override val tag: PluginTag = tag
override val type: KClass<out Plugin> = type
override val type: KClass<out T> = type
override fun build(): Plugin = constructor()
override fun invoke(meta: Meta): T = constructor(meta)
}
PluginRepository.register(factory)
register(factory)
return factory
}
inline fun <reified T : Plugin> PluginRepository.register(tag: PluginTag, noinline constructor: () -> T) =
inline fun <reified T : Plugin> PluginRepository.register(tag: PluginTag, noinline constructor: (Meta) -> T) =
register(tag, T::class, constructor)
fun PluginRepository.register(plugin: Plugin) = register(plugin.tag, plugin::class) { plugin }

21
dataforge-context/src/commonMain/kotlin/hep/dataforge/provider/Provider.kt
View File

@ -17,6 +17,7 @@ package hep.dataforge.provider
import hep.dataforge.names.Name
import hep.dataforge.names.toName
import kotlin.jvm.JvmName
/**
* A marker utility interface for providers.
@ -51,7 +52,7 @@ interface Provider {
* @param target
* @return
*/
fun listTop(target: String): Sequence<Name>
fun listNames(target: String): Sequence<Name>
}
fun Provider.provide(path: Path, targetOverride: String? = null): Any? {
@ -77,15 +78,23 @@ inline fun <reified T : Any> Provider.provide(path: String): T? {
return provide(Path.parse(path)) as? T
}
inline fun <reified T : Any> Provider.provide(target: String, name: String): T? {
return provide(PathToken(name.toName(), target).toPath()) as? T
inline fun <reified T : Any> Provider.provide(target: String, name: Name): T? {
return provide(PathToken(name, target).toPath()) as? T
}
inline fun <reified T : Any> Provider.provide(target: String, name: String): T? =
provide(target, name.toName())
/**
* [Sequence] of all elements with given target
* A top level content with names
*/
fun Provider.provideAll(target: String): Sequence<Any> {
return listTop(target).map { provideTop(target, it) ?: error("The element $it is declared but not provided") }
fun Provider.top(target: String): Map<Name, Any> = top<Any>(target)
@JvmName("typedTop")
inline fun <reified T : Any> Provider.top(target: String): Map<Name, T> {
return listNames(target).associate {
it to (provideTop(target, it) as? T ?: error("The element $it is declared but not provided"))
}
}

40
dataforge-context/src/commonTest/kotlin/hep/dataforge/context/ContextTest.kt Normal file
View File

@ -0,0 +1,40 @@
package hep.dataforge.context
import hep.dataforge.names.Name
import hep.dataforge.names.appendLeft
import hep.dataforge.names.toName
import kotlin.test.Test
import kotlin.test.assertEquals
import kotlin.test.assertTrue
class ContextTest {
class DummyPlugin : AbstractPlugin() {
override val tag get() = PluginTag("test")
override fun provideTop(target: String, name: Name): Any? {
return when (target) {
"test" -> return name
else -> super.provideTop(target, name)
}
}
override fun listNames(target: String): Sequence<Name> {
return when (target) {
"test" -> sequenceOf("a", "b", "c.d").map { it.toName() }
else -> super.listNames(target)
}
}
}
@Test
fun testPluginManager() {
Global.plugins.load(DummyPlugin())
val members = Global.content<Name>("test")
assertEquals(3, members.count())
members.forEach {
assertTrue{it.key == it.value.appendLeft("test")}
}
}
}

6
dataforge-context/src/jsMain/kotlin/hep/dataforge/context/PluginRepository.kt
View File

@ -3,14 +3,14 @@ package hep.dataforge.context
actual object PluginRepository {
private val factories: MutableSet<PluginFactory> = HashSet()
private val factories: MutableSet<PluginFactory<*>> = HashSet()
actual fun register(factory: PluginFactory) {
actual fun register(factory: PluginFactory<*>) {
factories.add(factory)
}
/**
* List plugins available in the repository
*/
actual fun list(): Sequence<PluginFactory> = factories.asSequence()
actual fun list(): Sequence<PluginFactory<*>> = factories.asSequence()
}

6
dataforge-context/src/jvmMain/kotlin/hep/dataforge/context/PluginRepository.kt
View File

@ -2,16 +2,16 @@ package hep.dataforge.context
actual object PluginRepository {
private val factories: MutableSet<PluginFactory> = HashSet()
private val factories: MutableSet<PluginFactory<*>> = HashSet()
actual fun register(factory: PluginFactory) {
actual fun register(factory: PluginFactory<*>) {
factories.add(factory)
}
/**
* List plugins available in the repository
*/
actual fun list(): Sequence<PluginFactory> =
actual fun list(): Sequence<PluginFactory<*>> =
factories.asSequence() + Global.services()
}

20
dataforge-context/src/jvmMain/kotlin/hep/dataforge/provider/Types.kt
View File

@ -1,11 +1,14 @@
package hep.dataforge.provider
import hep.dataforge.context.Context
import hep.dataforge.context.members
import hep.dataforge.context.content
import hep.dataforge.names.Name
import kotlin.reflect.KClass
import kotlin.reflect.full.findAnnotation
/**
*
*/
object Types {
operator fun get(cl: KClass<*>): String {
return cl.findAnnotation<Type>()?.id ?: cl.simpleName ?: ""
@ -24,13 +27,20 @@ inline fun <reified T : Any> Provider.provideByType(name: String): T? {
return provide(target, name)
}
inline fun <reified T : Any> Provider.provideAllByType(): Sequence<T> {
inline fun <reified T : Any> Provider.provideByType(name: Name): T? {
val target = Types[T::class]
return provideAll(target).filterIsInstance<T>()
return provide(target, name)
}
inline fun <reified T : Any> Provider.top(): Map<Name, T> {
val target = Types[T::class]
return listNames(target).associate { name ->
name to (provideByType<T>(name) ?: error("The element $name is declared but not provided"))
}
}
/**
* A sequences of all objects provided by plugins with given target and type
*/
inline fun <reified T : Any> Context.members(): Sequence<T> = members<T>(Types[T::class])
inline fun <reified T : Any> Context.content(): Map<Name, T> = content<T>(Types[T::class])

5
dataforge-data/build.gradle.kts
View File

@ -1,8 +1,8 @@
plugins {
kotlin("multiplatform")
`npm-multiplatform`
}
val coroutinesVersion: String by rootProject.extra
val coroutinesVersion: String = Versions.coroutinesVersion
kotlin {
jvm()
@ -11,6 +11,7 @@ kotlin {
val commonMain by getting{
dependencies {
api(project(":dataforge-meta"))
api(kotlin("reflect"))
api("org.jetbrains.kotlinx:kotlinx-coroutines-core-common:$coroutinesVersion")
}
}

40
dataforge-data/src/commonMain/kotlin/hep/dataforge/data/Action.kt
View File

@ -20,9 +20,10 @@ interface Action<in T : Any, out R : Any> {
}
/**
* Action composition. The result is terminal if one of parts is terminal
* Action composition. The result is terminal if one of its parts is terminal
*/
infix fun <T : Any, I : Any, R : Any> Action<T, I>.then(action: Action<I, R>): Action<T, R> {
// TODO introduce composite action and add optimize by adding action to the list
return object : Action<T, R> {
override fun invoke(node: DataNode<T>, meta: Meta): DataNode<R> {
return action(this@then.invoke(node, meta), meta)
@ -33,28 +34,19 @@ infix fun <T : Any, I : Any, R : Any> Action<T, I>.then(action: Action<I, R>): A
}
}
/**
* An action that performs the same transformation on each of input data nodes. Null results are ignored.
*/
class PipeAction<in T : Any, out R : Any>(val transform: (Name, Data<T>, Meta) -> Data<R>?) : Action<T, R> {
override fun invoke(node: DataNode<T>, meta: Meta): DataNode<R> = DataNode.build {
node.dataSequence().forEach { (name, data) ->
val res = transform(name, data, meta)
if (res != null) {
set(name, res)
}
}
}
companion object {
/**
* A simple pipe that performs transformation on the data and copies input meta into the output
*/
inline fun <T : Any, reified R : Any> simple(noinline transform: suspend (Name, T, Meta) -> R) =
PipeAction { name, data: Data<T>, meta ->
val goal = data.goal.pipe { transform(name, it, meta) }
return@PipeAction Data.of(goal, data.meta)
}
}
}
///**
// * An action that performs the same transformation on each of input data nodes. Null results are ignored.
// * The transformation is non-suspending because it is lazy.
// */
//class PipeAction<in T : Any, out R : Any>(val transform: (Name, Data<T>, Meta) -> Data<R>?) : Action<T, R> {
// override fun invoke(node: DataNode<T>, meta: Meta): DataNode<R> = DataNode.build {
// node.data().forEach { (name, data) ->
// val res = transform(name, data, meta)
// if (res != null) {
// set(name, res)
// }
// }
// }
//}

21
dataforge-data/src/commonMain/kotlin/hep/dataforge/data/Data.kt
View File

@ -2,7 +2,7 @@ package hep.dataforge.data
import hep.dataforge.meta.Meta
import hep.dataforge.meta.MetaRepr
import kotlin.coroutines.CoroutineContext
import kotlinx.coroutines.CoroutineScope
import kotlin.reflect.KClass
/**
@ -29,19 +29,32 @@ interface Data<out T : Any> : MetaRepr {
const val TYPE = "data"
fun <T : Any> of(type: KClass<out T>, goal: Goal<T>, meta: Meta): Data<T> = DataImpl(type, goal, meta)
inline fun <reified T : Any> of(goal: Goal<T>, meta: Meta): Data<T> = of(T::class, goal, meta)
fun <T : Any> of(name: String, type: KClass<out T>, goal: Goal<T>, meta: Meta): Data<T> =
NamedData(name, of(type, goal, meta))
inline fun <reified T : Any> of(name: String, goal: Goal<T>, meta: Meta): Data<T> =
of(name, T::class, goal, meta)
fun <T : Any> static(context: CoroutineContext, value: T, meta: Meta): Data<T> =
DataImpl(value::class, Goal.static(context, value), meta)
fun <T : Any> static(scope: CoroutineScope, value: T, meta: Meta): Data<T> =
DataImpl(value::class, Goal.static(scope, value), meta)
}
}
suspend fun <T: Any> Data<T>.await(): T = goal.await()
/**
* Upcast a [Data] to a supertype
*/
inline fun <reified R : Any, reified T : R> Data<T>.cast(): Data<R> {
return Data.of(R::class, goal, meta)
}
fun <R : Any, T : R> Data<T>.cast(type: KClass<R>): Data<R> {
return Data.of(type, goal, meta)
}
suspend fun <T : Any> Data<T>.await(): T = goal.await()
/**
* Generic Data implementation

10
dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataFilter.kt
View File

@ -4,14 +4,14 @@ import hep.dataforge.meta.*
import hep.dataforge.names.toName
class DataFilter(override val config: Config) : Specification {
class DataFilter(override val config: Config) : Specific {
var from by string()
var to by string()
var pattern by string("*.")
// val prefix by string()
// val suffix by string()
companion object : SpecificationCompanion<DataFilter> {
companion object : Specification<DataFilter> {
override fun wrap(config: Config): DataFilter = DataFilter(config)
}
}
@ -22,15 +22,15 @@ class DataFilter(override val config: Config) : Specification {
fun <T : Any> DataNode<T>.filter(filter: DataFilter): DataNode<T> {
val sourceNode = filter.from?.let { getNode(it.toName()) } ?: this@filter
val regex = filter.pattern.toRegex()
val targetNode = DataTreeBuilder<T>().apply {
sourceNode.dataSequence().forEach { (name, data) ->
val targetNode = DataTreeBuilder(type).apply {
sourceNode.data().forEach { (name, data) ->
if (name.toString().matches(regex)) {
this[name] = data
}
}
}
return filter.to?.let {
DataTreeBuilder<T>().apply { this[it.toName()] = targetNode }.build()
DataTreeBuilder(type).apply { this[it.toName()] = targetNode }.build()
} ?: targetNode.build()
}

78
dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataNode.kt
View File

@ -1,14 +1,18 @@
package hep.dataforge.data
import hep.dataforge.names.Name
import hep.dataforge.names.NameToken
import hep.dataforge.names.plus
import hep.dataforge.names.toName
import hep.dataforge.names.*
import kotlin.reflect.KClass
/**
* A tree-like data structure grouped into the node. All data inside the node must inherit its type
*/
interface DataNode<out T : Any> {
/**
* The minimal common ancestor to all data in the node
*/
val type: KClass<out T>
/**
* Get the specific data if it exists
*/
@ -22,21 +26,23 @@ interface DataNode<out T : Any> {
/**
* Walk the tree upside down and provide all data nodes with full names
*/
fun dataSequence(): Sequence<Pair<Name, Data<T>>>
fun data(): Sequence<Pair<Name, Data<T>>>
/**
* A sequence of all nodes in the tree walking upside down, excluding self
*/
fun nodeSequence(): Sequence<Pair<Name, DataNode<T>>>
fun nodes(): Sequence<Pair<Name, DataNode<T>>>
operator fun iterator(): Iterator<Pair<Name, Data<T>>> = dataSequence().iterator()
operator fun iterator(): Iterator<Pair<Name, Data<T>>> = data().iterator()
companion object {
const val TYPE = "dataNode"
fun <T : Any> build(block: DataTreeBuilder<T>.() -> Unit) = DataTreeBuilder<T>().apply(block).build()
}
fun <T : Any> build(type: KClass<out T>, block: DataTreeBuilder<T>.() -> Unit) =
DataTreeBuilder<T>(type).apply(block).build()
fun <T : Any> builder(type: KClass<out T>) = DataTreeBuilder(type)
}
}
internal sealed class DataTreeItem<out T : Any> {
@ -44,29 +50,32 @@ internal sealed class DataTreeItem<out T : Any> {
class Value<out T : Any>(val value: Data<T>) : DataTreeItem<T>()
}
class DataTree<out T : Any> internal constructor(private val items: Map<NameToken, DataTreeItem<T>>) : DataNode<T> {
class DataTree<out T : Any> internal constructor(
override val type: KClass<out T>,
private val items: Map<NameToken, DataTreeItem<T>>
) : DataNode<T> {
//TODO add node-level meta?
override fun get(name: Name): Data<T>? = when (name.length) {
0 -> error("Empty name")
1 -> (items[name.first()] as? DataTreeItem.Value)?.value
else -> getNode(name.first()!!.toName())?.get(name.cutFirst())
else -> getNode(name.first()!!.asName())?.get(name.cutFirst())
}
override fun getNode(name: Name): DataTree<T>? = when (name.length) {
0 -> this
1 -> (items[name.first()] as? DataTreeItem.Node)?.tree
else -> getNode(name.first()!!.toName())?.getNode(name.cutFirst())
else -> getNode(name.first()!!.asName())?.getNode(name.cutFirst())
}
override fun dataSequence(): Sequence<Pair<Name, Data<T>>> {
override fun data(): Sequence<Pair<Name, Data<T>>> {
return sequence {
items.forEach { (head, tree) ->
when (tree) {
is DataTreeItem.Value -> yield(head.toName() to tree.value)
is DataTreeItem.Value -> yield(head.asName() to tree.value)
is DataTreeItem.Node -> {
val subSequence =
tree.tree.dataSequence().map { (name, data) -> (head.toName() + name) to data }
tree.tree.data().map { (name, data) -> (head.asName() + name) to data }
yieldAll(subSequence)
}
}
@ -74,13 +83,13 @@ class DataTree<out T : Any> internal constructor(private val items: Map<NameToke
}
}
override fun nodeSequence(): Sequence<Pair<Name, DataNode<T>>> {
override fun nodes(): Sequence<Pair<Name, DataNode<T>>> {
return sequence {
items.forEach { (head, tree) ->
if (tree is DataTreeItem.Node) {
yield(head.toName() to tree.tree)
yield(head.asName() to tree.tree)
val subSequence =
tree.tree.nodeSequence().map { (name, node) -> (head.toName() + name) to node }
tree.tree.nodes().map { (name, node) -> (head.asName() + name) to node }
yieldAll(subSequence)
}
}
@ -96,7 +105,7 @@ private sealed class DataTreeBuilderItem<out T : Any> {
/**
* A builder for a DataTree.
*/
class DataTreeBuilder<T : Any> {
class DataTreeBuilder<T : Any>(private val type: KClass<out T>) {
private val map = HashMap<NameToken, DataTreeBuilderItem<T>>()
operator fun set(token: NameToken, node: DataTreeBuilder<T>) {
@ -111,7 +120,7 @@ class DataTreeBuilder<T : Any> {
private fun buildNode(token: NameToken): DataTreeBuilder<T> {
return if (!map.containsKey(token)) {
DataTreeBuilder<T>().also { map[token] = DataTreeBuilderItem.Node(it) }
DataTreeBuilder<T>(type).also { map[token] = DataTreeBuilderItem.Node(it) }
} else {
(map[token] as? DataTreeBuilderItem.Node ?: error("The node with name $token is occupied by leaf")).tree
}
@ -156,7 +165,15 @@ class DataTreeBuilder<T : Any> {
/**
* Build and append node
*/
infix fun String.to(block: DataTreeBuilder<T>.() -> Unit) = set(toName(), DataTreeBuilder<T>().apply(block))
infix fun String.to(block: DataTreeBuilder<T>.() -> Unit) = set(toName(), DataTreeBuilder<T>(type).apply(block))
fun update(node: DataNode<T>){
node.data().forEach {
//TODO check if the place is occupied
this[it.first] = it.second
}
}
fun build(): DataTree<T> {
val resMap = map.mapValues { (_, value) ->
@ -165,28 +182,35 @@ class DataTreeBuilder<T : Any> {
is DataTreeBuilderItem.Node -> DataTreeItem.Node(value.tree.build())
}
}
return DataTree(resMap)
return DataTree(type, resMap)
}
}
/**
* Generate a mutable builder from this node. Node content is not changed
*/
fun <T : Any> DataNode<T>.builder(): DataTreeBuilder<T> = DataTreeBuilder<T>().apply {
dataSequence().forEach { (name, data) -> this[name] = data }
fun <T : Any> DataNode<T>.builder(): DataTreeBuilder<T> = DataTreeBuilder(type).apply {
data().forEach { (name, data) -> this[name] = data }
}
/**
* Start computation for all goals in data node
*/
fun DataNode<*>.startAll() = dataSequence().forEach { (_, data) -> data.goal.start() }
fun DataNode<*>.startAll() = data().forEach { (_, data) -> data.goal.start() }
fun <T : Any> DataNode<T>.filter(predicate: (Name, Data<T>) -> Boolean): DataNode<T> = DataNode.build {
dataSequence().forEach { (name, data) ->
fun <T : Any> DataNode<T>.filter(predicate: (Name, Data<T>) -> Boolean): DataNode<T> = DataNode.build(type) {
data().forEach { (name, data) ->
if (predicate(name, data)) {
this[name] = data
}
}
}
fun <T: Any> DataNode<T>.first(): Data<T> = data().first().second
/**
* Check that node is compatible with given type meaning that each element could be cast to the type
*/
expect fun DataNode<*>.checkType(type: KClass<*>)
//fun <T : Any, R: T> DataNode<T>.filterIsInstance(type: KClass<R>): DataNode<R> = filter{_,data -> type.}

87
dataforge-data/src/commonMain/kotlin/hep/dataforge/data/Goal.kt
View File

@ -2,33 +2,37 @@ package hep.dataforge.data
import kotlinx.coroutines.*
import kotlin.coroutines.CoroutineContext
import kotlin.coroutines.EmptyCoroutineContext
/**
* A special deferred with explicit dependencies and some additional information like progress and unique id
*/
interface Goal<out T> : Deferred<T>, CoroutineScope {
val scope: CoroutineScope
override val coroutineContext get() = scope.coroutineContext
val dependencies: Collection<Goal<*>>
val status: String
val totalWork: Double
val workDone: Double
val totalWork: Double get() = dependencies.sumByDouble { totalWork } + (monitor?.totalWork ?: 0.0)
val workDone: Double get() = dependencies.sumByDouble { workDone } + (monitor?.workDone ?: 0.0)
val status: String get() = monitor?.status ?: ""
val progress: Double get() = workDone / totalWork
companion object {
/**
* Create goal wrapping static value. This goal is always completed
*/
fun <T> static(context: CoroutineContext, value: T): Goal<T> =
StaticGoalImpl(context, CompletableDeferred(value))
fun <T> static(scope: CoroutineScope, value: T): Goal<T> =
StaticGoalImpl(scope, CompletableDeferred(value))
}
}
/**
* A monitor of goal state that could be accessed only form inside the goal
*/
class GoalMonitor {
class GoalMonitor : CoroutineContext.Element {
override val key: CoroutineContext.Key<*> get() = GoalMonitor
var totalWork: Double = 1.0
var workDone: Double = 0.0
var status: String = ""
@ -46,26 +50,24 @@ class GoalMonitor {
fun finish() {
workDone = totalWork
}
companion object : CoroutineContext.Key<GoalMonitor>
}
val CoroutineScope.monitor: GoalMonitor? get() = coroutineContext[GoalMonitor]
private class GoalImpl<T>(
override val scope: CoroutineScope,
override val dependencies: Collection<Goal<*>>,
val monitor: GoalMonitor,
deferred: Deferred<T>
) : Goal<T>, Deferred<T> by deferred {
override val coroutineContext: CoroutineContext get() = this
override val totalWork: Double get() = dependencies.sumByDouble { totalWork } + monitor.totalWork
override val workDone: Double get() = dependencies.sumByDouble { workDone } + monitor.workDone
override val status: String get() = monitor.status
}
) : Goal<T>, Deferred<T> by deferred
private class StaticGoalImpl<T>(val context: CoroutineContext, deferred: CompletableDeferred<T>) : Goal<T>,
private class StaticGoalImpl<T>(override val scope: CoroutineScope, deferred: CompletableDeferred<T>) : Goal<T>,
Deferred<T> by deferred {
override val dependencies: Collection<Goal<*>> get() = emptyList()
override val status: String get() = ""
override val totalWork: Double get() = 0.0
override val workDone: Double get() = 0.0
override val coroutineContext: CoroutineContext get() = context
}
@ -75,23 +77,32 @@ private class StaticGoalImpl<T>(val context: CoroutineContext, deferred: Complet
*
* **Important:** Unlike regular deferred, the [Goal] is started lazily, so the actual calculation is called only when result is requested.
*/
fun <R> CoroutineScope.createGoal(dependencies: Collection<Goal<*>>, block: suspend GoalMonitor.() -> R): Goal<R> {
val monitor = GoalMonitor()
val deferred = async(start = CoroutineStart.LAZY) {
fun <R> CoroutineScope.createGoal(
dependencies: Collection<Goal<*>>,
context: CoroutineContext = EmptyCoroutineContext,
block: suspend CoroutineScope.() -> R
): Goal<R> {
val deferred = async(context + GoalMonitor(), start = CoroutineStart.LAZY) {
dependencies.forEach { it.start() }
monitor.start()
return@async supervisorScope { monitor.block() }
}.also {
monitor.finish()
monitor?.start()
//Running in supervisor scope in order to allow manual error handling
return@async supervisorScope {
block().also {
monitor?.finish()
}
}
}
return GoalImpl(dependencies, monitor, deferred)
return GoalImpl(this, dependencies, deferred)
}
/**
* Create a one-to-one goal based on existing goal
*/
fun <T, R> Goal<T>.pipe(block: suspend GoalMonitor.(T) -> R): Goal<R> = createGoal(listOf(this)) { block(await()) }
fun <T, R> Goal<T>.pipe(
context: CoroutineContext = EmptyCoroutineContext,
block: suspend CoroutineScope.(T) -> R
): Goal<R> = createGoal(listOf(this), context) { block(await()) }
/**
* Create a joining goal.
@ -99,8 +110,22 @@ fun <T, R> Goal<T>.pipe(block: suspend GoalMonitor.(T) -> R): Goal<R> = createGo
*/
fun <T, R> Collection<Goal<T>>.join(
scope: CoroutineScope = first(),
block: suspend GoalMonitor.(Collection<T>) -> R
): Goal<R> =
scope.createGoal(this) {
block(map { it.await() })
}
context: CoroutineContext = EmptyCoroutineContext,
block: suspend CoroutineScope.(Collection<T>) -> R
): Goal<R> = scope.createGoal(this, context) {
block(map { it.await() })
}
/**
* A joining goal for a map
* @param K type of the map key
* @param T type of the input goal
* @param R type of the result goal
*/
fun <K, T, R> Map<K, Goal<T>>.join(
scope: CoroutineScope = values.first(),
context: CoroutineContext = EmptyCoroutineContext,
block: suspend CoroutineScope.(Map<K, T>) -> R
): Goal<R> = scope.createGoal(this.values, context) {
block(mapValues { it.value.await() })
}

75
dataforge-data/src/commonMain/kotlin/hep/dataforge/data/GroupBuilder.kt Normal file
View File

@ -0,0 +1,75 @@
/*
* Copyright 2015 Alexander Nozik.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package hep.dataforge.data
import hep.dataforge.meta.Meta
import hep.dataforge.meta.get
import hep.dataforge.meta.string
interface GroupRule {
operator fun <T : Any> invoke(node: DataNode<T>): Map<String, DataNode<T>>
}
/**
* The class to builder groups of content with annotation defined rules
*
* @author Alexander Nozik
*/
object GroupBuilder {
/**
* Create grouping rule that creates groups for different values of value
* field with name [key]
*
* @param key
* @param defaultTagValue
* @return
*/
fun byValue(key: String, defaultTagValue: String): GroupRule = object :
GroupRule {
override fun <T : Any> invoke(node: DataNode<T>): Map<String, DataNode<T>> {
val map = HashMap<String, DataTreeBuilder<T>>()
node.data().forEach { (name, data) ->
val tagValue = data.meta[key]?.string ?: defaultTagValue
map.getOrPut(tagValue) { DataNode.builder(node.type) }[name] = data
}
return map.mapValues { it.value.build() }
}
}
// @ValueDef(key = "byValue", required = true, info = "The name of annotation value by which grouping should be made")
// @ValueDef(
// key = "defaultValue",
// def = "default",
// info = "Default value which should be used for content in which the grouping value is not presented"
// )
fun byMeta(config: Meta): GroupRule {
//TODO expand grouping options
return config["byValue"]?.string?.let {
byValue(
it,
config["defaultValue"]?.string ?: "default"
)
}
?: object : GroupRule {
override fun <T : Any> invoke(node: DataNode<T>): Map<String, DataNode<T>> = mapOf("" to node)
}
}
}

111
dataforge-data/src/commonMain/kotlin/hep/dataforge/data/JoinAction.kt Normal file
View File

@ -0,0 +1,111 @@
package hep.dataforge.data
import hep.dataforge.meta.Laminate
import hep.dataforge.meta.Meta
import hep.dataforge.meta.MetaBuilder
import hep.dataforge.meta.builder
import hep.dataforge.names.Name
import hep.dataforge.names.toName
import kotlin.coroutines.CoroutineContext
import kotlin.coroutines.EmptyCoroutineContext
import kotlin.reflect.KClass
class JoinGroup<T : Any, R : Any>(var name: String, internal val node: DataNode<T>) {
var meta: MetaBuilder = MetaBuilder()
lateinit var result: suspend ActionEnv.(Map<Name, T>) -> R
fun result(f: suspend ActionEnv.(Map<Name, T>) -> R) {
this.result = f;
}
}
class JoinGroupBuilder<T : Any, R : Any>(val actionMeta: Meta) {
private val groupRules: MutableList<(DataNode<T>) -> List<JoinGroup<T, R>>> = ArrayList();
/**
* introduce grouping by value name
*/
fun byValue(tag: String, defaultTag: String = "@default", action: JoinGroup<T, R>.() -> Unit) {
groupRules += { node ->
GroupBuilder.byValue(tag, defaultTag).invoke(node).map {
JoinGroup<T, R>(it.key, it.value).apply(action)
}
}
}
/**
* Add a single fixed group to grouping rules
*/
fun group(groupName: String, filter: DataFilter, action: JoinGroup<T, R>.() -> Unit) {
groupRules += { node ->
listOf(
JoinGroup<T, R>(groupName, node.filter(filter)).apply(action)
)
}
}
fun group(groupName: String, filter: (Name, Data<T>) -> Boolean, action: JoinGroup<T, R>.() -> Unit) {
groupRules += { node ->
listOf(
JoinGroup<T, R>(groupName, node.filter(filter)).apply(action)
)
}
}
/**
* Apply transformation to the whole node
*/
fun result(resultName: String, f: suspend ActionEnv.(Map<Name, T>) -> R) {
groupRules += { node ->
listOf(JoinGroup<T, R>(resultName, node).apply { result(f) })
}
}
internal fun buildGroups(input: DataNode<T>): List<JoinGroup<T, R>> {
return groupRules.flatMap { it.invoke(input) }
}
}
/**
* The same rules as for KPipe
*/
class JoinAction<T : Any, R : Any>(
val inputType: KClass<T>,
val outputType: KClass<R>,
val context: CoroutineContext = EmptyCoroutineContext,
private val action: JoinGroupBuilder<T, R>.() -> Unit
) : Action<T, R> {
override fun invoke(node: DataNode<T>, meta: Meta): DataNode<R> {
node.checkType(inputType)
return DataNode.build(outputType) {
JoinGroupBuilder<T, R>(meta).apply(action).buildGroups(node).forEach { group ->
val laminate = Laminate(group.meta, meta)
val goalMap: Map<Name, Goal<T>> = group.node
.data()
.associate { it.first to it.second.goal }
val groupName: String = group.name;
val env = ActionEnv(groupName.toName(), laminate.builder())
val goal = goalMap.join(context = context) { group.result.invoke(env, it) }
val res = Data.of(outputType, goal, env.meta)
set(env.name, res)
}
}
}
}
operator fun <T> Map<Name,T>.get(name:String) = get(name.toName())

65
dataforge-data/src/commonMain/kotlin/hep/dataforge/data/PipeAction.kt Normal file
View File

@ -0,0 +1,65 @@
package hep.dataforge.data
import hep.dataforge.meta.*
import hep.dataforge.names.Name
import kotlin.coroutines.CoroutineContext
import kotlin.coroutines.EmptyCoroutineContext
import kotlin.reflect.KClass
class ActionEnv(val name: Name, val meta: Meta)
/**
* Action environment
*/
class PipeBuilder<T, R>(var name: Name, var meta: MetaBuilder) {
lateinit var result: suspend ActionEnv.(T) -> R
/**
* Calculate the result of goal
*/
fun result(f: suspend ActionEnv.(T) -> R) {
result = f;
}
}
class PipeAction<T : Any, R : Any>(
val inputType: KClass<T>,
val outputType: KClass<R>,
val context: CoroutineContext = EmptyCoroutineContext,
private val block: PipeBuilder<T, R>.() -> Unit
) : Action<T, R> {
override fun invoke(node: DataNode<T>, meta: Meta): DataNode<R> {
node.checkType(inputType)
return DataNode.build(outputType) {
node.data().forEach { (name, data) ->
//merging data meta with action meta (data meta is primary)
val oldMeta = meta.builder().apply { update(data.meta) }
// creating environment from old meta and name
val env = ActionEnv(name, oldMeta)
//applying transformation from builder
val builder = PipeBuilder<T, R>(name, oldMeta).apply(block)
//getting new name
val newName = builder.name
//getting new meta
val newMeta = builder.meta.seal()
//creating a goal with custom context if provided
val goal = data.goal.pipe(context) { builder.result(env, it) }
//setting the data node
this[newName] = Data.of(outputType, goal, newMeta)
}
}
}
}
inline fun <reified T : Any, reified R : Any> DataNode<T>.pipe(
meta: Meta,
context: CoroutineContext = EmptyCoroutineContext,
noinline action: PipeBuilder<T, R>.() -> Unit
): DataNode<R> = PipeAction(T::class, R::class, context, action).invoke(this, meta)

69
dataforge-data/src/commonMain/kotlin/hep/dataforge/data/SplitAction.kt Normal file
View File

@ -0,0 +1,69 @@
package hep.dataforge.data
import hep.dataforge.meta.Laminate
import hep.dataforge.meta.Meta
import hep.dataforge.meta.MetaBuilder
import hep.dataforge.meta.builder
import hep.dataforge.names.Name
import hep.dataforge.names.toName
import kotlin.collections.set
import kotlin.coroutines.CoroutineContext
import kotlin.coroutines.EmptyCoroutineContext
import kotlin.reflect.KClass
class FragmentRule<T : Any, R : Any>(val name: Name, var meta: MetaBuilder) {
lateinit var result: suspend (T) -> R
fun result(f: suspend (T) -> R) {
result = f;
}
}
class SplitBuilder<T : Any, R : Any>(val name: Name, val meta: Meta) {
internal val fragments: MutableMap<Name, FragmentRule<T, R>.() -> Unit> = HashMap()
/**
* Add new fragment building rule. If the framgent not defined, result won't be available even if it is present in the map
* @param name the name of a fragment
* @param rule the rule to transform fragment name and meta using
*/
fun fragment(name: String, rule: FragmentRule<T, R>.() -> Unit) {
fragments[name.toName()] = rule
}
}
class SplitAction<T : Any, R : Any>(
val inputType: KClass<T>,
val outputType: KClass<R>,
val context: CoroutineContext = EmptyCoroutineContext,
private val action: SplitBuilder<T, R>.() -> Unit
) : Action<T, R> {
override fun invoke(node: DataNode<T>, meta: Meta): DataNode<R> {
node.checkType(inputType)
return DataNode.build(outputType) {
node.data().forEach { (name, data) ->
val laminate = Laminate(data.meta, meta)
val split = SplitBuilder<T, R>(name, data.meta).apply(action)
// apply individual fragment rules to result
split.fragments.forEach { (fragmentName, rule) ->
val env = FragmentRule<T, R>(fragmentName, laminate.builder())
rule(env)
val goal = data.goal.pipe(context = context) { env.result(it) }
val res = Data.of(outputType, goal, env.meta)
set(env.name, res)
}
}
}
}
}

10
dataforge-data/src/jsMain/kotlin/hep/dataforge/data/checkType.kt Normal file
View File

@ -0,0 +1,10 @@
package hep.dataforge.data
import kotlin.reflect.KClass
/**
* Check that node is compatible with given type meaning that each element could be cast to the type
*/
actual fun DataNode<*>.checkType(type: KClass<*>) {
//Not supported in js yet
}

46
dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/CastDataNode.kt Normal file
View File

@ -0,0 +1,46 @@
package hep.dataforge.data
import hep.dataforge.names.Name
import kotlin.reflect.KClass
import kotlin.reflect.full.isSubclassOf
fun <T : Any, R : Any> Data<T>.safeCast(type: KClass<R>): Data<R>? {
return if (type.isSubclassOf(type)) {
@Suppress("UNCHECKED_CAST")
Data.of(type, goal as Goal<R>, meta)
} else {
null
}
}
/**
* Filter a node by data and node type. Resulting node and its subnodes is guaranteed to have border type [type],
* but could contain empty nodes
*/
fun <T : Any, R : Any> DataNode<T>.cast(type: KClass<R>): DataNode<R> {
return if (this is CastDataNode) {
origin.cast(type)
} else {
CastDataNode(this, type)
}
}
inline fun <T : Any, reified R : Any> DataNode<T>.cast(): DataNode<R> = cast(R::class)
class CastDataNode<out T : Any>(val origin: DataNode<Any>, override val type: KClass<out T>) : DataNode<T> {
override fun get(name: Name): Data<T>? =
origin[name]?.safeCast(type)
override fun getNode(name: Name): DataNode<T>? {
return origin.getNode(name)?.cast(type)
}
override fun data(): Sequence<Pair<Name, Data<T>>> =
origin.data().mapNotNull { pair ->
pair.second.safeCast(type)?.let { pair.first to it }
}
override fun nodes(): Sequence<Pair<Name, DataNode<T>>> =
origin.nodes().map { it.first to it.second.cast(type) }
}

13
dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/checkType.kt Normal file
View File

@ -0,0 +1,13 @@
package hep.dataforge.data
import kotlin.reflect.KClass
import kotlin.reflect.full.isSuperclassOf
/**
* Check that node is compatible with given type meaning that each element could be cast to the type
*/
actual fun DataNode<*>.checkType(type: KClass<*>) {
if (!type.isSuperclassOf(type)) {
error("$type expected, but $type received")
}
}

47
dataforge-io/build.gradle.kts
View File

@ -1,16 +1,57 @@
plugins {
kotlin("multiplatform")
`npm-multiplatform`
}
description = "IO for meta"
val ioVersion: String = Versions.ioVersion
val serializationVersion: String = Versions.serializationVersion
kotlin {
jvm()
js()
sourceSets {
val commonMain by getting{
dependencies {
api(project(":dataforge-context"))
api(project(":dataforge-meta-io"))
api(project(":dataforge-meta"))
//implementation 'org.jetbrains.kotlin:kotlin-reflect'
api("org.jetbrains.kotlinx:kotlinx-serialization-runtime-common:$serializationVersion")
api("org.jetbrains.kotlinx:kotlinx-io:$ioVersion")
}
}
val commonTest by getting {
dependencies {
implementation("org.jetbrains.kotlin:kotlin-test-common")
implementation("org.jetbrains.kotlin:kotlin-test-annotations-common")
}
}
val jvmMain by getting {
dependencies {
api("org.jetbrains.kotlinx:kotlinx-serialization-runtime:$serializationVersion")
api("org.jetbrains.kotlinx:kotlinx-io-jvm:$ioVersion")
}
}
val jvmTest by getting {
dependencies {
implementation("org.jetbrains.kotlin:kotlin-test")
implementation("org.jetbrains.kotlin:kotlin-test-junit")
}
}
val jsMain by getting {
dependencies {
api("org.jetbrains.kotlinx:kotlinx-serialization-runtime-js:$serializationVersion")
api("org.jetbrains.kotlinx:kotlinx-io-js:$ioVersion")
}
}
val jsTest by getting {
dependencies {
implementation("org.jetbrains.kotlin:kotlin-test-js")
}
}
// iosMain {
// }
// iosTest {
// }
}
}

6
dataforge-meta-io/src/commonMain/kotlin/hep/dataforge/meta/io/BinaryMetaFormat.kt → dataforge-io/src/commonMain/kotlin/hep/dataforge/io/BinaryMetaFormat.kt
View File

@ -1,4 +1,4 @@
package hep.dataforge.meta.io
package hep.dataforge.io
import hep.dataforge.meta.*
import hep.dataforge.values.*
@ -8,8 +8,8 @@ import kotlinx.io.core.readText
import kotlinx.io.core.writeText
object BinaryMetaFormat : MetaFormat {
override fun write(meta: Meta, out: Output) {
out.writeMeta(meta)
override fun write(obj: Meta, out: Output) {
out.writeMeta(obj)
}
override fun read(input: Input): Meta {

20
dataforge-meta-io/src/commonMain/kotlin/hep/dataforge/meta/io/Envelope.kt → dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Envelope.kt
View File

@ -1,21 +1,15 @@
package hep.dataforge.meta.io
package hep.dataforge.io
import hep.dataforge.meta.Meta
import hep.dataforge.meta.get
import hep.dataforge.meta.string
import kotlinx.io.core.Input
interface Envelope {
val meta: Meta
val data: Binary?
val data: Input?
companion object {
// /**
// * Property keys
// */
// const val TYPE_PROPERTY = "type"
// const val META_TYPE_PROPERTY = "metaType"
// const val META_LENGTH_PROPERTY = "metaLength"
// const val DATA_LENGTH_PROPERTY = "dataLength"
/**
* meta keys
@ -25,9 +19,16 @@ interface Envelope {
const val ENVELOPE_DATA_TYPE_KEY = "$ENVELOPE_NODE.dataType"
const val ENVELOPE_DESCRIPTION_KEY = "$ENVELOPE_NODE.description"
//const val ENVELOPE_TIME_KEY = "@envelope.time"
}
}
class SimpleEnvelope(override val meta: Meta, val dataProvider: () -> Input?) : Envelope{
override val data: Input?
get() = dataProvider()
}
/**
* The purpose of the envelope
*
@ -48,3 +49,4 @@ val Envelope.dataType: String? get() = meta[Envelope.ENVELOPE_DATA_TYPE_KEY].str
* @return
*/
val Envelope.description: String? get() = meta[Envelope.ENVELOPE_DESCRIPTION_KEY].string

10
dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOFormat.kt Normal file
View File

@ -0,0 +1,10 @@
package hep.dataforge.io
import kotlinx.io.core.Input
import kotlinx.io.core.Output
interface IOFormat<T : Any> {
fun write(obj: T, out: Output)
fun read(input: Input): T
}

107
dataforge-io/src/commonMain/kotlin/hep/dataforge/io/JsonMetaFormat.kt Normal file
View File

@ -0,0 +1,107 @@
package hep.dataforge.io
import hep.dataforge.meta.Meta
import hep.dataforge.meta.MetaItem
import hep.dataforge.names.NameToken
import hep.dataforge.names.toName
import hep.dataforge.values.*
import kotlinx.io.core.Input
import kotlinx.io.core.Output
import kotlinx.io.core.readText
import kotlinx.io.core.writeText
import kotlinx.serialization.json.*
object JsonMetaFormat : MetaFormat {
override fun write(obj: Meta, out: Output) {
val str = obj.toJson().toString()
out.writeText(str)
}
override fun read(input: Input): Meta {
val str = input.readText()
val json = Json.plain.parseJson(str)
if (json is JsonObject) {
return json.toMeta()
} else {
TODO("Non-object root not supported")
}
}
}
fun Value.toJson(): JsonElement {
return if(isList()){
JsonArray(list.map { it.toJson() })
} else {
when (type) {
ValueType.NUMBER -> JsonPrimitive(number)
ValueType.STRING -> JsonPrimitive(string)
ValueType.BOOLEAN -> JsonPrimitive(boolean)
ValueType.NULL -> JsonNull
}
}
}
fun Meta.toJson(): JsonObject {
val map = this.items.mapValues { entry ->
val value = entry.value
when (value) {
is MetaItem.ValueItem -> value.value.toJson()
is MetaItem.NodeItem -> value.node.toJson()
}
}.mapKeys { it.key.toString() }
return JsonObject(map)
}
fun JsonObject.toMeta() = JsonMeta(this)
class JsonMeta(val json: JsonObject) : Meta {
private fun JsonPrimitive.toValue(): Value {
return when (this) {
JsonNull -> Null
else -> this.content.parseValue() // Optimize number and boolean parsing
}
}
private operator fun MutableMap<String, MetaItem<JsonMeta>>.set(key: String, value: JsonElement) = when (value) {
is JsonPrimitive -> this[key] = MetaItem.ValueItem(value.toValue())
is JsonObject -> this[key] = MetaItem.NodeItem(value.toMeta())
is JsonArray -> {
when {
value.all { it is JsonPrimitive } -> {
val listValue = ListValue(
value.map {
//We already checked that all values are primitives
(it as JsonPrimitive).toValue()
}
)
this[key] = MetaItem.ValueItem(listValue)
}
else -> value.forEachIndexed { index, jsonElement ->
when (jsonElement) {
is JsonObject -> this["$key[$index]"] = MetaItem.NodeItem(JsonMeta(jsonElement))
is JsonPrimitive -> this["$key[$index]"] = MetaItem.ValueItem(jsonElement.toValue())
is JsonArray -> TODO("Nested arrays not supported")
}
}
}
}
}
override val items: Map<NameToken, MetaItem<JsonMeta>> by lazy {
val map = HashMap<String, MetaItem<JsonMeta>>()
json.forEach { (key, value) -> map[key] = value }
map.mapKeys { it.key.toName().first()!! }
}
}
class JsonMetaFormatFactory : MetaFormatFactory {
override val name: String = "json"
override val key: Short = 0x4a53//"JS"
override fun build() = JsonMetaFormat
}

11
dataforge-meta-io/src/commonMain/kotlin/hep/dataforge/meta/io/MetaFormat.kt → dataforge-io/src/commonMain/kotlin/hep/dataforge/io/MetaFormat.kt
View File

@ -1,15 +1,14 @@
package hep.dataforge.meta.io
package hep.dataforge.io
import hep.dataforge.meta.Meta
import kotlinx.io.core.*
import kotlinx.io.core.BytePacketBuilder
import kotlinx.io.core.ByteReadPacket
import kotlinx.io.core.toByteArray
/**
* A format for meta serialization
*/
interface MetaFormat {
fun write(meta: Meta, out: Output)
fun read(input: Input): Meta
}
interface MetaFormat: IOFormat<Meta>
/**
* ServiceLoader compatible factory

63
dataforge-io/src/commonMain/kotlin/hep/dataforge/io/OutputManager.kt
View File

@ -1,63 +0,0 @@
package hep.dataforge.io
import hep.dataforge.context.AbstractPlugin
import hep.dataforge.context.Plugin
import hep.dataforge.context.PluginTag
import hep.dataforge.context.PluginTag.Companion.DATAFORGE_GROUP
import hep.dataforge.meta.EmptyMeta
import hep.dataforge.meta.Meta
import hep.dataforge.names.EmptyName
import hep.dataforge.names.Name
import kotlinx.coroutines.CoroutineDispatcher
import kotlin.reflect.KClass
/**
* A manager for outputs
*/
interface OutputManager : Plugin {
/**
* Provide an output for given name and stage.
*
* @param stage represents the node or directory for the output. Empty means root node.
* @param name represents the name inside the node.
* @param meta configuration for [Output] (not for rendered object)
*
*/
operator fun get(name: Name, stage: Name = EmptyName, meta: Meta = EmptyMeta): Output<Any>
/**
* Get an output specialized for giver ntype
*/
fun <T : Any> typed(type: KClass<T>, name: Name, stage: Name = EmptyName, meta: Meta = EmptyMeta): Output<T>
}
/**
* Get an output with given [name], [stage] and reified content type
*/
inline fun <reified T : Any> OutputManager.typed(
name: Name,
stage: Name = EmptyName,
meta: Meta = EmptyMeta
): Output<T> {
return typed(T::class, name, stage, meta)
}
/**
* System console output.
* The [ConsoleOutput] is used when no other [OutputManager] is provided.
*/
expect val ConsoleOutput: Output<Any>
object ConsoleOutputManager : AbstractPlugin(), OutputManager {
override val tag: PluginTag = PluginTag("output.console", group = DATAFORGE_GROUP)
override fun get(name: Name, stage: Name, meta: Meta): Output<Any> = ConsoleOutput
override fun <T : Any> typed(type: KClass<T>, name: Name, stage: Name, meta: Meta): Output<T> = ConsoleOutput
}
/**
* A dispatcher for output tasks.
*/
expect val OutputDispatcher : CoroutineDispatcher

3
dataforge-meta-io/src/commonTest/kotlin/hep/dataforge/meta/io/MetaFormatTest.kt → dataforge-io/src/commonTest/kotlin/hep/dataforge/io/MetaFormatTest.kt
View File

@ -1,4 +1,4 @@
package hep.dataforge.meta.io
package hep.dataforge.io
import hep.dataforge.meta.buildMeta
import kotlin.test.Test
@ -26,6 +26,7 @@ class MetaFormatTest {
"node" to {
"b" to "DDD"
"c" to 11.1
"array" to doubleArrayOf(1.0,2.0,3.0)
}
}
val string = meta.asString(JsonMetaFormat)

2
dataforge-io/src/jvmMain/resources/META-INF/services/hep.dataforge.io.MetaFormatFactory Normal file
View File

@ -0,0 +1,2 @@
hep.dataforge.io.BinaryMetaFormatFactory
hep.dataforge.io.JsonMetaFormatFactory

57
dataforge-meta-io/build.gradle.kts
View File

@ -1,57 +0,0 @@
plugins {
kotlin("multiplatform")
}
description = "IO for meta"
val ioVersion: String by rootProject.extra
val serializationVersion: String by rootProject.extra
kotlin {
jvm()
js()
sourceSets {
val commonMain by getting{
dependencies {
api(project(":dataforge-meta"))
//implementation 'org.jetbrains.kotlin:kotlin-reflect'
api("org.jetbrains.kotlinx:kotlinx-serialization-runtime-common:$serializationVersion")
api("org.jetbrains.kotlinx:kotlinx-io:$ioVersion")
}
}
val commonTest by getting {
dependencies {
implementation("org.jetbrains.kotlin:kotlin-test-common")
implementation("org.jetbrains.kotlin:kotlin-test-annotations-common")
}
}
val jvmMain by getting {
dependencies {
api("org.jetbrains.kotlinx:kotlinx-serialization-runtime:$serializationVersion")
api("org.jetbrains.kotlinx:kotlinx-io-jvm:$ioVersion")
}
}
val jvmTest by getting {
dependencies {
implementation("org.jetbrains.kotlin:kotlin-test")
implementation("org.jetbrains.kotlin:kotlin-test-junit")
}
}
val jsMain by getting {
dependencies {
api("org.jetbrains.kotlinx:kotlinx-serialization-runtime-js:$serializationVersion")
api("org.jetbrains.kotlinx:kotlinx-io-js:$ioVersion")
}
}
val jsTest by getting {
dependencies {
implementation("org.jetbrains.kotlin:kotlin-test-js")
}
}
// iosMain {
// }
// iosTest {
// }
}
}

7
dataforge-meta-io/src/commonMain/kotlin/hep/dataforge/meta/io/Binary.kt
View File

@ -1,7 +0,0 @@
package hep.dataforge.meta.io
import kotlinx.io.ByteBuffer
import kotlinx.io.core.Input
//TODO replace by abstraction
typealias Binary = Input

89
dataforge-meta-io/src/commonMain/kotlin/hep/dataforge/meta/io/JsonMetaFormat.kt
View File

@ -1,89 +0,0 @@
package hep.dataforge.meta.io
import hep.dataforge.meta.Meta
import hep.dataforge.meta.MetaItem
import hep.dataforge.names.NameToken
import hep.dataforge.values.*
import kotlinx.io.core.Input
import kotlinx.io.core.Output
import kotlinx.io.core.readText
import kotlinx.io.core.writeText
import kotlinx.serialization.json.*
object JsonMetaFormat : MetaFormat {
override fun write(meta: Meta, out: Output) {
val str = meta.toJson().toString()
out.writeText(str)
}
override fun read(input: Input): Meta {
val str = input.readText()
val json = Json.plain.parseJson(str)
if(json is JsonObject) {
return json.toMeta()
} else {
TODO("non-object root")
}
}
}
fun Value.toJson(): JsonElement {
return when (type) {
ValueType.NUMBER -> JsonPrimitive(number)
ValueType.STRING -> JsonPrimitive(string)
ValueType.BOOLEAN -> JsonPrimitive(boolean)
ValueType.NULL -> JsonNull
}
}
fun Meta.toJson(): JsonObject {
val map = this.items.mapValues { entry ->
val value = entry.value
when (value) {
is MetaItem.ValueItem -> value.value.toJson()
is MetaItem.NodeItem -> value.node.toJson()
}
}.mapKeys { it.key.toString() }
return JsonObject(map)
}
fun JsonElement.toMetaItem() = when (this) {
is JsonPrimitive -> MetaItem.ValueItem<JsonMeta>(this.toValue())
is JsonObject -> MetaItem.NodeItem(this.toMeta())
is JsonArray -> {
if (this.all { it is JsonPrimitive }) {
val value = ListValue(this.map { (it as JsonPrimitive).toValue() })
MetaItem.ValueItem<JsonMeta>(value)
} else {
TODO("mixed nodes json")
}
}
}
fun JsonObject.toMeta() = JsonMeta(this)
private fun JsonPrimitive.toValue(): Value {
return when (this) {
JsonNull -> Null
else -> this.content.parseValue() // Optimize number and boolean parsing
}
}
class JsonMeta(val json: JsonObject) : Meta {
override val items: Map<NameToken, MetaItem<out Meta>> by lazy {
json.mapKeys { NameToken(it.key) }.mapValues { entry ->
entry.value.toMetaItem()
}
}
}
class JsonMetaFormatFactory : MetaFormatFactory {
override val name: String = "json"
override val key: Short = 0x4a53//"JS"
override fun build() = JsonMetaFormat
}

2
dataforge-meta-io/src/jvmMain/resources/META-INF/services/hep.dataforge.meta.io.MetaFormatFactory
View File

@ -1,2 +0,0 @@
hep.dataforge.meta.io.BinaryMetaFormatFactory
hep.dataforge.meta.io.JsonMetaFormatFactory

52
dataforge-meta/build.gradle.kts
View File

@ -1,5 +1,5 @@
plugins {
kotlin("multiplatform")
`npm-multiplatform`
}
description = "Meta definition and basic operations on meta"
@ -7,54 +7,4 @@ description = "Meta definition and basic operations on meta"
kotlin {
jvm()
js()
sourceSets {
val commonMain by getting {
dependencies {
api(kotlin("stdlib"))
}
}
val commonTest by getting {
dependencies {
implementation(kotlin("test-common"))
implementation(kotlin("test-annotations-common"))
}
}
val jvmMain by getting {
dependencies {
api(kotlin("stdlib-jdk8"))
}
}
val jvmTest by getting {
dependencies {
implementation(kotlin("test"))
implementation(kotlin("test-junit"))
}
}
val jsMain by getting {
dependencies {
api(kotlin("stdlib-js"))
}
}
val jsTest by getting {
dependencies {
implementation(kotlin("test-js"))
}
}
// mingwMain {
// }
// mingwTest {
// }
}
}
//tasks.withType<Kotlin2JsCompile>{
// kotlinOptions{
// metaInfo = true
// outputFile = "${project.buildDir.path}/js/${project.name}.js"
// sourceMap = true
// moduleKind = "umd"
// main = "call"
// }
//}

5
dataforge-meta/src/commonMain/kotlin/hep/dataforge/descriptors/Described.kt Normal file
View File

@ -0,0 +1,5 @@
package hep.dataforge.descriptors
interface Described {
val descriptor: NodeDescriptor
}

128
dataforge-meta/src/commonMain/kotlin/hep/dataforge/descriptors/NodeDescriptor.kt Normal file
View File

@ -0,0 +1,128 @@
/*
* Copyright 2018 Alexander Nozik.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/*
* To change this license header, choose License Headers in Project Properties.
* To change this template file, choose Tools | Templates
* and open the template in the editor.
*/
package hep.dataforge.descriptors
import hep.dataforge.meta.*
import hep.dataforge.names.NameToken
import hep.dataforge.names.toName
/**
* Descriptor for meta node. Could contain additional information for viewing
* and editing.
*
* @author Alexander Nozik
*/
class NodeDescriptor(override val config: Config) : Specific {
/**
* The name of this node
*
* @return
*/
var name: String by string { error("Anonymous descriptors are not allowed") }
/**
* The default for this node. Null if there is no default.
*
* @return
*/
var default: Meta? by node()
/**
* True if multiple children with this nodes name are allowed. Anonymous
* nodes are always single
*
* @return
*/
var multiple: Boolean by boolean(false)
/**
* True if the node is required
*
* @return
*/
var required: Boolean by boolean { default == null }
/**
* The node description
*
* @return
*/
var info: String? by string()
/**
* A list of tags for this node. Tags used to customize node usage
*
* @return
*/
var tags: List<String> by value{ value ->
value?.list?.map { it.string } ?: emptyList()
}
/**
* The list of value descriptors
*/
val values: Map<String, ValueDescriptor>
get() = config.getAll("value".toName()).entries.associate { (name, node) ->
name to ValueDescriptor.wrap(node.node ?: error("Value descriptor must be a node"))
}
fun value(name: String, descriptor: ValueDescriptor) {
val token = NameToken("value", name)
config[token] = descriptor.config
}
/**
* Add a value descriptor using block for
*/
fun value(name: String, block: ValueDescriptor.() -> Unit) {
value(name, ValueDescriptor.build { this.name = name }.apply(block))
}
/**
* The map of children node descriptors
*/
val nodes: Map<String, NodeDescriptor>
get() = config.getAll("node".toName()).entries.associate { (name, node) ->
name to NodeDescriptor.wrap(node.node ?: error("Node descriptor must be a node"))
}
fun node(name: String, descriptor: NodeDescriptor) {
val token = NameToken("node", name)
config[token] = descriptor.config
}
fun node(name: String, block: NodeDescriptor.() -> Unit) {
node(name, NodeDescriptor.build { this.name = name }.apply(block))
}
//override val descriptor: NodeDescriptor = empty("descriptor")
companion object : Specification<NodeDescriptor> {
override fun wrap(config: Config): NodeDescriptor = NodeDescriptor(config)
}
}

193
dataforge-meta/src/commonMain/kotlin/hep/dataforge/descriptors/ValueDescriptor.kt Normal file
View File

@ -0,0 +1,193 @@
/*
* Copyright 2018 Alexander Nozik.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package hep.dataforge.descriptors
import hep.dataforge.meta.*
import hep.dataforge.values.False
import hep.dataforge.values.True
import hep.dataforge.values.Value
import hep.dataforge.values.ValueType
/**
* A descriptor for meta value
*
* Descriptor can have non-atomic path. It is resolved when descriptor is added to the node
*
* @author Alexander Nozik
*/
class ValueDescriptor(override val config: Config) : Specific {
/**
* The default for this value. Null if there is no default.
*
* @return
*/
var default: Value? by value()
fun default(v: Any) {
this.default = Value.of(v)
}
/**
* True if multiple values with this name are allowed.
*
* @return
*/
var multiple: Boolean by boolean(false)
/**
* True if the value is required
*
* @return
*/
var required: Boolean by boolean { default == null }
/**
* Value name
*
* @return
*/
var name: String by string { error("Anonymous descriptors are not allowed") }
/**
* The value info
*
* @return
*/
var info: String? by string()
/**
* A list of allowed ValueTypes. Empty if any value type allowed
*
* @return
*/
var type: List<ValueType> by value {
it?.list?.map { v -> ValueType.valueOf(v.string) } ?: emptyList()
}
fun type(vararg t: ValueType) {
this.type = listOf(*t)
}
var tags: List<String> by value { value ->
value?.list?.map { it.string } ?: emptyList()
}
/**
* Check if given value is allowed for here. The type should be allowed and
* if it is value should be within allowed values
*
* @param value
* @return
*/
fun isAllowedValue(value: Value): Boolean {
return (type.isEmpty() || type.contains(ValueType.STRING) || type.contains(value.type)) && (allowedValues.isEmpty() || allowedValues.contains(
value
))
}
/**
* A list of allowed values with descriptions. If empty than any value is
* allowed.
*
* @return
*/
var allowedValues: List<Value> by value {
it?.list ?: if (type.size == 1 && type[0] === ValueType.BOOLEAN) {
listOf(True, False)
} else {
emptyList()
}
}
/**
* Allow given list of value and forbid others
*/
fun allow(vararg v: Any) {
this.allowedValues = v.map { Value.of(it) }
}
companion object : Specification<ValueDescriptor> {
override fun wrap(config: Config): ValueDescriptor = ValueDescriptor(config)
inline fun <reified E : Enum<E>> enum(name: String) =
ValueDescriptor.build {
this.name = name
type(ValueType.STRING)
this.allowedValues = enumValues<E>().map { Value.of(it.name) }
}
// /**
// * Build a value descriptor from annotation
// */
// fun build(def: ValueDef): ValueDescriptor {
// val builder = MetaBuilder("value")
// .setValue("name", def.key)
//
// if (def.type.isNotEmpty()) {
// builder.setValue("type", def.type)
// }
//
// if (def.multiple) {
// builder.setValue("multiple", def.multiple)
// }
//
// if (!def.info.isEmpty()) {
// builder.setValue("info", def.info)
// }
//
// if (def.allowed.isNotEmpty()) {
// builder.setValue("allowedValues", def.allowed)
// } else if (def.enumeration != Any::class) {
// if (def.enumeration.java.isEnum) {
// val values = def.enumeration.java.enumConstants
// builder.setValue("allowedValues", values.map { it.toString() })
// } else {
// throw RuntimeException("Only enumeration classes are allowed in 'enumeration' annotation property")
// }
// }
//
// if (def.def.isNotEmpty()) {
// builder.setValue("default", def.def)
// } else if (!def.required) {
// builder.setValue("required", def.required)
// }
//
// if (def.tags.isNotEmpty()) {
// builder.setValue("tags", def.tags)
// }
// return ValueDescriptor(builder)
// }
//
// /**
// * Build empty value descriptor
// */
// fun empty(valueName: String): ValueDescriptor {
// val builder = MetaBuilder("value")
// .setValue("name", valueName)
// return ValueDescriptor(builder)
// }
//
// /**
// * Merge two separate value descriptors
// */
// fun merge(primary: ValueDescriptor, secondary: ValueDescriptor): ValueDescriptor {
// return ValueDescriptor(Laminate(primary.meta, secondary.meta))
// }
}
}

149
dataforge-meta/src/commonMain/kotlin/hep/dataforge/descriptors/annotations.kt Normal file
View File

@ -0,0 +1,149 @@
/*
* Copyright 2018 Alexander Nozik.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package hep.dataforge.descriptors
import hep.dataforge.values.ValueType
import kotlin.reflect.KClass
@Target(AnnotationTarget.PROPERTY)
@MustBeDocumented
annotation class ValueDef(
val key: String,
val type: Array<ValueType> = arrayOf(ValueType.STRING),
val multiple: Boolean = false,
val def: String = "",
val info: String = "",
val required: Boolean = true,
val allowed: Array<String> = emptyArray(),
val enumeration: KClass<*> = Any::class,
val tags: Array<String> = emptyArray()
)
@MustBeDocumented
annotation class NodeDef(
val key: String,
val info: String = "",
val multiple: Boolean = false,
val required: Boolean = false,
val tags: Array<String> = emptyArray(),
/**
* A list of child value descriptors
*/
val values: Array<ValueDef> = emptyArray(),
/**
* A target class for this node to describe
* @return
*/
val type: KClass<*> = Any::class,
/**
* The DataForge path to the resource containing the description. Following targets are supported:
*
* 1. resource
* 1. file
* 1. class
* 1. method
* 1. property
*
*
* Does not work if [type] is provided
*
* @return
*/
val descriptor: String = ""
)
/**
* Description text for meta property, node or whole object
*/
@Target(AnnotationTarget.CLASS, AnnotationTarget.PROPERTY)
@Retention(AnnotationRetention.RUNTIME)
@MustBeDocumented
annotation class Description(val value: String)
/**
* Annotation for value property which states that lists are expected
*/
@Target(AnnotationTarget.PROPERTY)
@Retention(AnnotationRetention.RUNTIME)
@MustBeDocumented
annotation class Multiple
/**
* Descriptor target
* The DataForge path to the resource containing the description. Following targets are supported:
* 1. resource
* 1. file
* 1. class
* 1. method
* 1. property
*
*
* Does not work if [type] is provided
*/
@Target(AnnotationTarget.CLASS, AnnotationTarget.PROPERTY, AnnotationTarget.VALUE_PARAMETER)
@Retention(AnnotationRetention.RUNTIME)
@MustBeDocumented
annotation class Descriptor(val value: String)
/**
* Aggregator class for descriptor nodes
*/
@Target(AnnotationTarget.CLASS, AnnotationTarget.PROPERTY, AnnotationTarget.FUNCTION, AnnotationTarget.VALUE_PARAMETER)
@Retention(AnnotationRetention.RUNTIME)
@MustBeDocumented
annotation class DescriptorNodes(vararg val nodes: NodeDef)
/**
* Aggregator class for descriptor values
*/
@Target(AnnotationTarget.CLASS, AnnotationTarget.PROPERTY, AnnotationTarget.FUNCTION, AnnotationTarget.VALUE_PARAMETER)
@Retention(AnnotationRetention.RUNTIME)
@MustBeDocumented
annotation class DescriptorValues(vararg val nodes: ValueDef)
/**
* Alternative name for property descriptor declaration
*/
@Target(AnnotationTarget.PROPERTY, AnnotationTarget.VALUE_PARAMETER)
@Retention(AnnotationRetention.RUNTIME)
@MustBeDocumented
annotation class DescriptorName(val name: String)
@Target(AnnotationTarget.PROPERTY)
@Retention(AnnotationRetention.RUNTIME)
@MustBeDocumented
annotation class DescriptorValue(val def: ValueDef)
//TODO enter fields directly?
@Target(AnnotationTarget.PROPERTY)
@Retention(AnnotationRetention.RUNTIME)
@MustBeDocumented
annotation class ValueProperty(
val name: String = "",
val type: Array<ValueType> = arrayOf(ValueType.STRING),
val multiple: Boolean = false,
val def: String = "",
val enumeration: KClass<*> = Any::class,
val tags: Array<String> = emptyArray()
)
@Target(AnnotationTarget.PROPERTY)
@Retention(AnnotationRetention.RUNTIME)
@MustBeDocumented
annotation class NodeProperty(val name: String = "")

4
dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Config.kt
View File

@ -2,7 +2,7 @@ package hep.dataforge.meta
import hep.dataforge.names.Name
import hep.dataforge.names.NameToken
import hep.dataforge.names.toName
import hep.dataforge.names.asName
//TODO add validator to configuration
@ -28,7 +28,7 @@ operator fun Config.get(token: NameToken): MetaItem<Config>? = items[token]
fun Meta.toConfig(): Config = this as? Config ?: Config().also { builder ->
this.items.mapValues { entry ->
val item = entry.value
builder[entry.key.toName()] = when (item) {
builder[entry.key.asName()] = when (item) {
is MetaItem.ValueItem -> MetaItem.ValueItem(item.value)
is MetaItem.NodeItem -> MetaItem.NodeItem(item.node.toConfig())
}

93
dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/ConfigDelegates.kt
View File

@ -10,33 +10,104 @@ import kotlin.jvm.JvmName
/**
* A property delegate that uses custom key
*/
fun Configurable.value(default: Value = Null, key: String? = null) =
ValueConfigDelegate(config, key, default)
fun Configurable.value(default: Any = Null, key: String? = null) =
MutableValueDelegate(config, key, Value.of(default))
fun <T> Configurable.value(default: T? = null, key: String? = null, transform: (Value?) -> T) =
MutableValueDelegate(config, key, Value.of(default)).transform(reader = transform)
fun Configurable.stringList(key: String? = null) =
value(key) { it?.list?.map { value -> value.string } ?: emptyList() }
fun Configurable.numberList(key: String? = null) =
value(key) { it?.list?.map { value -> value.number } ?: emptyList() }
fun Configurable.string(default: String? = null, key: String? = null) =
StringConfigDelegate(config, key, default)
MutableStringDelegate(config, key, default)
fun Configurable.boolean(default: Boolean? = null, key: String? = null) =
BooleanConfigDelegate(config, key, default)
MutableBooleanDelegate(config, key, default)
fun Configurable.number(default: Number? = null, key: String? = null) =
NumberConfigDelegate(config, key, default)
MutableNumberDelegate(config, key, default)
fun Configurable.child(key: String? = null) = MetaNodeDelegate(config, key)
/* Number delegates*/
fun Configurable.int(default: Int? = null, key: String? = null) =
number(default, key).int
fun Configurable.double(default: Double? = null, key: String? = null) =
number(default, key).double
fun Configurable.long(default: Long? = null, key: String? = null) =
number(default, key).long
fun Configurable.short(default: Short? = null, key: String? = null) =
number(default, key).short
fun Configurable.float(default: Float? = null, key: String? = null) =
number(default, key).float
//fun <T : Configurable> Configurable.spec(spec: Specification<T>, key: String? = null) = ChildConfigDelegate<T>(key) { spec.wrap(this) }
@JvmName("safeString")
fun Configurable.string(default: String, key: String? = null) =
SafeStringConfigDelegate(config, key, default)
MutableSafeStringDelegate(config, key) { default }
@JvmName("safeBoolean")
fun Configurable.boolean(default: Boolean, key: String? = null) =
SafeBooleanConfigDelegate(config, key, default)
MutableSafeBooleanDelegate(config, key) { default }
@JvmName("safeNumber")
fun Configurable.number(default: Number, key: String? = null) =
SafeNumberConfigDelegate(config, key, default)
MutableSafeNumberDelegate(config, key) { default }
@JvmName("safeString")
fun Configurable.string(key: String? = null, default: () -> String) =
MutableSafeStringDelegate(config, key, default)
@JvmName("safeBoolean")
fun Configurable.boolean(key: String? = null, default: () -> Boolean) =
MutableSafeBooleanDelegate(config, key, default)
@JvmName("safeNumber")
fun Configurable.number(key: String? = null, default: () -> Number) =
MutableSafeNumberDelegate(config, key, default)
/* Safe number delegates*/
@JvmName("safeInt")
fun Configurable.int(default: Int, key: String? = null) =
number(default, key).int
@JvmName("safeDouble")
fun Configurable.double(default: Double, key: String? = null) =
number(default, key).double
@JvmName("safeLong")
fun Configurable.long(default: Long, key: String? = null) =
number(default, key).long
@JvmName("safeShort")
fun Configurable.short(default: Short, key: String? = null) =
number(default, key).short
@JvmName("safeFloat")
fun Configurable.float(default: Float, key: String? = null) =
number(default, key).float
/**
* Enum delegate
*/
inline fun <reified E : Enum<E>> Configurable.enum(default: E, key: String? = null) =
SafeEnumvConfigDelegate(config, key, default) { enumValueOf(it) }
MutableSafeEnumvDelegate(config, key, default) { enumValueOf(it) }
/* Node delegates */
fun Configurable.node(key: String? = null) = MutableNodeDelegate(config, key)
fun <T : Specific> Configurable.spec(spec: Specification<T>, key: String? = null) =
MutableMorphDelegate(config, key) { spec.wrap(it) }
fun <T : Specific> Configurable.spec(builder: (Config) -> T, key: String? = null) =
MutableMorphDelegate(config, key) { specification(builder).wrap(it) }

212
dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Delegates.kt
View File

@ -56,22 +56,40 @@ class DelegateWrapper<T, R>(val delegate: ReadOnlyProperty<Any?, T>, val reader:
//Delegates with non-null values
class SafeStringDelegate(val meta: Meta, private val key: String? = null, private val default: String) :
ReadOnlyProperty<Any?, String> {
class SafeStringDelegate(
val meta: Meta,
private val key: String? = null,
default: () -> String
) : ReadOnlyProperty<Any?, String> {
private val default: String by lazy(default)
override fun getValue(thisRef: Any?, property: KProperty<*>): String {
return meta[key ?: property.name]?.string ?: default
}
}
class SafeBooleanDelegate(val meta: Meta, private val key: String? = null, private val default: Boolean) :
ReadOnlyProperty<Any?, Boolean> {
class SafeBooleanDelegate(
val meta: Meta,
private val key: String? = null,
default: () -> Boolean
) : ReadOnlyProperty<Any?, Boolean> {
private val default: Boolean by lazy(default)
override fun getValue(thisRef: Any?, property: KProperty<*>): Boolean {
return meta[key ?: property.name]?.boolean ?: default
}
}
class SafeNumberDelegate(val meta: Meta, private val key: String? = null, private val default: Number) :
ReadOnlyProperty<Any?, Number> {
class SafeNumberDelegate(
val meta: Meta,
private val key: String? = null,
default: () -> Number
) : ReadOnlyProperty<Any?, Number> {
private val default: Number by lazy(default)
override fun getValue(thisRef: Any?, property: KProperty<*>): Number {
return meta[key ?: property.name]?.number ?: default
}
@ -118,96 +136,116 @@ fun Meta.number(default: Number? = null, key: String? = null) = NumberDelegate(t
fun Meta.child(key: String? = null) = ChildDelegate(this, key) { it }
@JvmName("safeString")
fun Meta.string(default: String, key: String? = null) = SafeStringDelegate(this, key, default)
fun Meta.string(default: String, key: String? = null) =
SafeStringDelegate(this, key) { default }
@JvmName("safeBoolean")
fun Meta.boolean(default: Boolean, key: String? = null) = SafeBooleanDelegate(this, key, default)
fun Meta.boolean(default: Boolean, key: String? = null) =
SafeBooleanDelegate(this, key) { default }
@JvmName("safeNumber")
fun Meta.number(default: Number, key: String? = null) = SafeNumberDelegate(this, key, default)
fun Meta.number(default: Number, key: String? = null) =
SafeNumberDelegate(this, key) { default }
@JvmName("safeString")
fun Meta.string(key: String? = null, default: () -> String) =
SafeStringDelegate(this, key, default)
@JvmName("safeBoolean")
fun Meta.boolean(key: String? = null, default: () -> Boolean) =
SafeBooleanDelegate(this, key, default)
@JvmName("safeNumber")
fun Meta.number(key: String? = null, default: () -> Number) =
SafeNumberDelegate(this, key, default)
inline fun <reified E : Enum<E>> Meta.enum(default: E, key: String? = null) =
SafeEnumDelegate(this, key, default) { enumValueOf(it) }
/* Config delegates */
/* Read-write delegates */
class ValueConfigDelegate<M : MutableMeta<M>>(
val config: M,
class MutableValueDelegate<M : MutableMeta<M>>(
val meta: M,
private val key: String? = null,
private val default: Value? = null
) : ReadWriteProperty<Any?, Value?> {
override fun getValue(thisRef: Any?, property: KProperty<*>): Value? {
return config[key ?: property.name]?.value ?: default
return meta[key ?: property.name]?.value ?: default
}
override fun setValue(thisRef: Any?, property: KProperty<*>, value: Value?) {
val name = key ?: property.name
if (value == null) {
config.remove(name)
meta.remove(name)
} else {
config.setValue(name, value)
meta.setValue(name, value)
}
}
fun <T> transform(writer: (T) -> Value? = { Value.of(it) }, reader: (Value?) -> T) =
ReadWriteDelegateWrapper(this, reader, writer)
}
class StringConfigDelegate<M : MutableMeta<M>>(
val config: M,
class MutableStringDelegate<M : MutableMeta<M>>(
val meta: M,
private val key: String? = null,
private val default: String? = null
) : ReadWriteProperty<Any?, String?> {
override fun getValue(thisRef: Any?, property: KProperty<*>): String? {
return config[key ?: property.name]?.string ?: default
return meta[key ?: property.name]?.string ?: default
}
override fun setValue(thisRef: Any?, property: KProperty<*>, value: String?) {
val name = key ?: property.name
if (value == null) {
config.remove(name)
meta.remove(name)
} else {
config.setValue(name, value.asValue())
meta.setValue(name, value.asValue())
}
}
}
class BooleanConfigDelegate<M : MutableMeta<M>>(
val config: M,
class MutableBooleanDelegate<M : MutableMeta<M>>(
val meta: M,
private val key: String? = null,
private val default: Boolean? = null
) : ReadWriteProperty<Any?, Boolean?> {
override fun getValue(thisRef: Any?, property: KProperty<*>): Boolean? {
return config[key ?: property.name]?.boolean ?: default
return meta[key ?: property.name]?.boolean ?: default
}
override fun setValue(thisRef: Any?, property: KProperty<*>, value: Boolean?) {
val name = key ?: property.name
if (value == null) {
config.remove(name)
meta.remove(name)
} else {
config.setValue(name, value.asValue())
meta.setValue(name, value.asValue())
}
}
}
class NumberConfigDelegate<M : MutableMeta<M>>(
val config: M,
class MutableNumberDelegate<M : MutableMeta<M>>(
val meta: M,
private val key: String? = null,
private val default: Number? = null
) : ReadWriteProperty<Any?, Number?> {
override fun getValue(thisRef: Any?, property: KProperty<*>): Number? {
return config[key ?: property.name]?.number ?: default
return meta[key ?: property.name]?.number ?: default
}
override fun setValue(thisRef: Any?, property: KProperty<*>, value: Number?) {
val name = key ?: property.name
if (value == null) {
config.remove(name)
meta.remove(name)
} else {
config.setValue(name, value.asValue())
meta.setValue(name, value.asValue())
}
}
val double get() = ReadWriteDelegateWrapper(this, reader = { it?.toDouble() }, writer = { it })
val float get() = ReadWriteDelegateWrapper(this, reader = { it?.toFloat() }, writer = { it })
val int get() = ReadWriteDelegateWrapper(this, reader = { it?.toInt() }, writer = { it })
val short get() = ReadWriteDelegateWrapper(this, reader = { it?.toShort() }, writer = { it })
val long get() = ReadWriteDelegateWrapper(this, reader = { it?.toLong() }, writer = { it })
@ -215,95 +253,108 @@ class NumberConfigDelegate<M : MutableMeta<M>>(
//Delegates with non-null values
class SafeStringConfigDelegate<M : MutableMeta<M>>(
val config: M,
class MutableSafeStringDelegate<M : MutableMeta<M>>(
val meta: M,
private val key: String? = null,
private val default: String
default: () -> String
) : ReadWriteProperty<Any?, String> {
private val default: String by lazy(default)
override fun getValue(thisRef: Any?, property: KProperty<*>): String {
return config[key ?: property.name]?.string ?: default
return meta[key ?: property.name]?.string ?: default
}
override fun setValue(thisRef: Any?, property: KProperty<*>, value: String) {
config.setValue(key ?: property.name, value.asValue())
meta.setValue(key ?: property.name, value.asValue())
}
}
class SafeBooleanConfigDelegate<M : MutableMeta<M>>(
val config: M,
class MutableSafeBooleanDelegate<M : MutableMeta<M>>(
val meta: M,
private val key: String? = null,
private val default: Boolean
default: () -> Boolean
) : ReadWriteProperty<Any?, Boolean> {
private val default: Boolean by lazy(default)
override fun getValue(thisRef: Any?, property: KProperty<*>): Boolean {
return config[key ?: property.name]?.boolean ?: default
return meta[key ?: property.name]?.boolean ?: default
}
override fun setValue(thisRef: Any?, property: KProperty<*>, value: Boolean) {
config.setValue(key ?: property.name, value.asValue())
meta.setValue(key ?: property.name, value.asValue())
}
}
class SafeNumberConfigDelegate<M : MutableMeta<M>>(
val config: M,
class MutableSafeNumberDelegate<M : MutableMeta<M>>(
val meta: M,
private val key: String? = null,
private val default: Number
default: () -> Number
) : ReadWriteProperty<Any?, Number> {
private val default: Number by lazy(default)
override fun getValue(thisRef: Any?, property: KProperty<*>): Number {
return config[key ?: property.name]?.number ?: default
return meta[key ?: property.name]?.number ?: default
}
override fun setValue(thisRef: Any?, property: KProperty<*>, value: Number) {
config.setValue(key ?: property.name, value.asValue())
meta.setValue(key ?: property.name, value.asValue())
}
val double get() = ReadWriteDelegateWrapper(this, reader = { it.toDouble() }, writer = { it })
val float get() = ReadWriteDelegateWrapper(this, reader = { it.toFloat() }, writer = { it })
val int get() = ReadWriteDelegateWrapper(this, reader = { it.toInt() }, writer = { it })
val short get() = ReadWriteDelegateWrapper(this, reader = { it.toShort() }, writer = { it })
val long get() = ReadWriteDelegateWrapper(this, reader = { it.toLong() }, writer = { it })
}
class SafeEnumvConfigDelegate<M : MutableMeta<M>, E : Enum<E>>(
val config: M,
class MutableSafeEnumvDelegate<M : MutableMeta<M>, E : Enum<E>>(
val meta: M,
private val key: String? = null,
private val default: E,
private val resolver: (String) -> E
) : ReadWriteProperty<Any?, E> {
override fun getValue(thisRef: Any?, property: KProperty<*>): E {
return (config[key ?: property.name]?.string)?.let { resolver(it) } ?: default
return (meta[key ?: property.name]?.string)?.let { resolver(it) } ?: default
}
override fun setValue(thisRef: Any?, property: KProperty<*>, value: E) {
config.setValue(key ?: property.name, value.name.asValue())
meta.setValue(key ?: property.name, value.name.asValue())
}
}
//Child node delegate
class MetaNodeDelegate<M : MutableMetaNode<M>>(
val config: M,
class MutableNodeDelegate<M : MutableMetaNode<M>>(
val meta: M,
private val key: String? = null
) : ReadWriteProperty<Any?, Meta> {
override fun getValue(thisRef: Any?, property: KProperty<*>): Meta {
return config[key ?: property.name]?.node ?: EmptyMeta
) : ReadWriteProperty<Any?, Meta?> {
override fun getValue(thisRef: Any?, property: KProperty<*>): Meta? {
return meta[key ?: property.name]?.node
}
override fun setValue(thisRef: Any?, property: KProperty<*>, value: Meta) {
config[key ?: property.name] = value
override fun setValue(thisRef: Any?, property: KProperty<*>, value: Meta?) {
meta[key ?: property.name] = value
}
}
class ChildConfigDelegate<M : MutableMetaNode<M>, T : Configurable>(
val config: M,
class MutableMorphDelegate<M : MutableMetaNode<M>, T : Configurable>(
val meta: M,
private val key: String? = null,
private val converter: (Meta) -> T
) :
ReadWriteProperty<Any?, T> {
override fun getValue(thisRef: Any?, property: KProperty<*>): T {
return converter(config[key ?: property.name]?.node ?: EmptyMeta)
) : ReadWriteProperty<Any?, T?> {
override fun getValue(thisRef: Any?, property: KProperty<*>): T? {
return meta[key ?: property.name]?.node?.let(converter)
}
override fun setValue(thisRef: Any?, property: KProperty<*>, value: T) {
config[key ?: property.name] = value.config
override fun setValue(thisRef: Any?, property: KProperty<*>, value: T?) {
if (value == null) {
meta.remove(key ?: property.name)
} else {
meta[key ?: property.name] = value.config
}
}
}
@ -328,32 +379,43 @@ class ReadWriteDelegateWrapper<T, R>(
* A property delegate that uses custom key
*/
fun <M : MutableMeta<M>> M.value(default: Value = Null, key: String? = null) =
ValueConfigDelegate(this, key, default)
MutableValueDelegate(this, key, default)
fun <M : MutableMeta<M>> M.string(default: String? = null, key: String? = null) =
StringConfigDelegate(this, key, default)
MutableStringDelegate(this, key, default)
fun <M : MutableMeta<M>> M.boolean(default: Boolean? = null, key: String? = null) =
BooleanConfigDelegate(this, key, default)
MutableBooleanDelegate(this, key, default)
fun <M : MutableMeta<M>> M.number(default: Number? = null, key: String? = null) =
NumberConfigDelegate(this, key, default)
MutableNumberDelegate(this, key, default)
fun <M : MutableMetaNode<M>> M.child(key: String? = null) = MetaNodeDelegate(this, key)
//fun <T : Configurable> Configurable.spec(spec: Specification<T>, key: String? = null) = ChildConfigDelegate<T>(key) { spec.wrap(this) }
fun <M : MutableMetaNode<M>> M.node(key: String? = null) = MutableNodeDelegate(this, key)
@JvmName("safeString")
fun <M : MutableMeta<M>> M.string(default: String, key: String? = null) =
SafeStringConfigDelegate(this, key, default)
MutableSafeStringDelegate(this, key) { default }
@JvmName("safeBoolean")
fun <M : MutableMeta<M>> M.boolean(default: Boolean, key: String? = null) =
SafeBooleanConfigDelegate(this, key, default)
MutableSafeBooleanDelegate(this, key) { default }
@JvmName("safeNumber")
fun <M : MutableMeta<M>> M.number(default: Number, key: String? = null) =
SafeNumberConfigDelegate(this, key, default)
MutableSafeNumberDelegate(this, key) { default }
@JvmName("safeString")
fun <M : MutableMeta<M>> M.string(key: String? = null, default: () -> String) =
MutableSafeStringDelegate(this, key, default)
@JvmName("safeBoolean")
fun <M : MutableMeta<M>> M.boolean(key: String? = null, default: () -> Boolean) =
MutableSafeBooleanDelegate(this, key, default)
@JvmName("safeNumber")
fun <M : MutableMeta<M>> M.number(key: String? = null, default: () -> Number) =
MutableSafeNumberDelegate(this, key, default)
inline fun <M : MutableMeta<M>, reified E : Enum<E>> M.enum(default: E, key: String? = null) =
SafeEnumvConfigDelegate(this, key, default) { enumValueOf(it) }
MutableSafeEnumvDelegate(this, key, default) { enumValueOf(it) }

2
dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/ExtraMetaDelegates.kt
View File

@ -33,4 +33,4 @@ fun Configurable.stringList(vararg default: String = emptyArray(), key: String?
fun <T : Metoid> Metoid.child(key: String? = null, converter: (Meta) -> T) = ChildDelegate(meta, key, converter)
fun <T : Configurable> Configurable.child(key: String? = null, converter: (Meta) -> T) =
ChildConfigDelegate(config, key, converter)
MutableMorphDelegate(config, key, converter)

59
dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Meta.kt
View File

@ -3,10 +3,7 @@ package hep.dataforge.meta
import hep.dataforge.meta.Meta.Companion.VALUE_KEY
import hep.dataforge.meta.MetaItem.NodeItem
import hep.dataforge.meta.MetaItem.ValueItem
import hep.dataforge.names.Name
import hep.dataforge.names.NameToken
import hep.dataforge.names.plus
import hep.dataforge.names.toName
import hep.dataforge.names.*
import hep.dataforge.values.EnumValue
import hep.dataforge.values.Value
import hep.dataforge.values.boolean
@ -76,30 +73,37 @@ operator fun Meta?.get(key: String): MetaItem<out Meta>? = get(key.toName())
* Get all items matching given name.
*/
fun Meta.getAll(name: Name): Map<String, MetaItem<out Meta>> {
if (name.length == 0) error("Can't use empty name for that")
val (body, query) = name.last()!!
val regex = query.toRegex()
return (this[name.cutLast()] as? NodeItem<*>)?.node?.items
?.filter { it.key.body == body && (query.isEmpty() || regex.matches(it.key.query)) }
?.mapKeys { it.key.query }
?: emptyMap()
val root = when (name.length) {
0 -> error("Can't use empty name for that")
1 -> this
else -> (this[name.cutLast()] as? NodeItem<*>)?.node
}
val (body, index) = name.last()!!
val regex = index.toRegex()
return root?.items
?.filter { it.key.body == body && (index.isEmpty() || regex.matches(it.key.index)) }
?.mapKeys { it.key.index }
?: emptyMap()
}
fun Meta.getAll(name: String): Map<String, MetaItem<out Meta>> = getAll(name.toName())
/**
* Transform meta to sequence of [Name]-[Value] pairs
* Get a sequence of [Name]-[Value] pairs
*/
fun Meta.asValueSequence(): Sequence<Pair<Name, Value>> {
fun Meta.values(): Sequence<Pair<Name, Value>> {
return items.asSequence().flatMap { entry ->
val item = entry.value
when (item) {
is ValueItem -> sequenceOf(entry.key.toName() to item.value)
is NodeItem -> item.node.asValueSequence().map { pair -> (entry.key.toName() + pair.first) to pair.second }
is ValueItem -> sequenceOf(entry.key.asName() to item.value)
is NodeItem -> item.node.values().map { pair -> (entry.key.asName() + pair.first) to pair.second }
}
}
}
operator fun Meta.iterator(): Iterator<Pair<Name, Value>> = asValueSequence().iterator()
operator fun Meta.iterator(): Iterator<Pair<Name, Value>> = values().iterator()
/**
* A meta node that ensures that all of its descendants has at least the same type
@ -108,6 +112,27 @@ interface MetaNode<M : MetaNode<M>> : Meta {
override val items: Map<NameToken, MetaItem<M>>
}
/**
* Get all items matching given name.
*/
fun <M : MetaNode<M>> MetaNode<M>.getAll(name: Name): Map<String, MetaItem<M>> {
val root: MetaNode<M>? = when (name.length) {
0 -> error("Can't use empty name for that")
1 -> this
else -> (this[name.cutLast()] as? NodeItem<M>)?.node
}
val (body, index) = name.last()!!
val regex = index.toRegex()
return root?.items
?.filter { it.key.body == body && (index.isEmpty() || regex.matches(it.key.index)) }
?.mapKeys { it.key.index }
?: emptyMap()
}
fun <M : MetaNode<M>> M.getAll(name: String): Map<String, MetaItem<M>> = getAll(name.toName())
operator fun <M : MetaNode<M>> MetaNode<M>.get(name: Name): MetaItem<M>? {
return name.first()?.let { token ->
val tail = name.cutFirst()
@ -118,7 +143,7 @@ operator fun <M : MetaNode<M>> MetaNode<M>.get(name: Name): MetaItem<M>? {
}
}
operator fun <M : MetaNode<M>> MetaNode<M>.get(key: String): MetaItem<M>? = get(key.toName())
operator fun <M : MetaNode<M>> MetaNode<M>?.get(key: String): MetaItem<M>? = this?.let { get(key.toName()) }
/**
* Equals and hash code implementation for meta node

4
dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MetaBuilder.kt
View File

@ -1,7 +1,7 @@
package hep.dataforge.meta
import hep.dataforge.names.Name
import hep.dataforge.names.toName
import hep.dataforge.names.asName
import hep.dataforge.values.Value
/**
@ -38,7 +38,7 @@ fun Meta.builder(): MetaBuilder {
return MetaBuilder().also { builder ->
items.mapValues { entry ->
val item = entry.value
builder[entry.key.toName()] = when (item) {
builder[entry.key.asName()] = when (item) {
is MetaItem.ValueItem -> MetaItem.ValueItem<MetaBuilder>(item.value)
is MetaItem.NodeItem -> MetaItem.NodeItem(item.node.builder())
}

42
dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MutableMeta.kt
View File

@ -1,9 +1,6 @@
package hep.dataforge.meta
import hep.dataforge.names.Name
import hep.dataforge.names.NameToken
import hep.dataforge.names.plus
import hep.dataforge.names.toName
import hep.dataforge.names.*
import hep.dataforge.values.Value
internal data class MetaListener(
@ -62,7 +59,7 @@ abstract class MutableMetaNode<M : MutableMetaNode<M>> : AbstractMetaNode<M>(),
}
}
}
itemChanged(key.toName(), oldItem, newItem)
itemChanged(key.asName(), oldItem, newItem)
}
/**
@ -103,7 +100,7 @@ fun <M : MutableMeta<M>> MutableMeta<M>.setItem(name: String, item: MetaItem<M>)
fun <M : MutableMeta<M>> MutableMeta<M>.setValue(name: String, value: Value) =
set(name.toName(), MetaItem.ValueItem(value))
fun <M : MutableMeta<M>> MutableMeta<M>.setItem(token: NameToken, item: MetaItem<M>?) = set(token.toName(), item)
fun <M : MutableMeta<M>> MutableMeta<M>.setItem(token: NameToken, item: MetaItem<M>?) = set(token.asName(), item)
fun <M : MutableMetaNode<M>> MutableMetaNode<M>.setNode(name: Name, node: Meta) =
set(name, MetaItem.NodeItem(wrap(name, node)))
@ -121,10 +118,13 @@ operator fun <M : MutableMetaNode<M>> M.set(name: Name, value: Any?) {
is MetaItem.NodeItem<*> -> setNode(name, value.node)
}
is Meta -> setNode(name, value)
is Specific -> setNode(name, value.config)
else -> setValue(name, Value.of(value))
}
}
operator fun <M : MutableMetaNode<M>> M.set(name: NameToken, value: Any?) = set(name.asName(), value)
operator fun <M : MutableMetaNode<M>> M.set(key: String, value: Any?) = set(key.toName(), value)
/**
@ -137,9 +137,9 @@ fun <M : MutableMetaNode<M>> M.update(meta: Meta) {
meta.items.forEach { entry ->
val value = entry.value
when (value) {
is MetaItem.ValueItem -> setValue(entry.key.toName(), value.value)
is MetaItem.NodeItem -> (this[entry.key.toName()] as? MetaItem.NodeItem)?.node?.update(value.node)
?: run { setNode(entry.key.toName(), value.node) }
is MetaItem.ValueItem -> setValue(entry.key.asName(), value.value)
is MetaItem.NodeItem -> (this[entry.key.asName()] as? MetaItem.NodeItem)?.node?.update(value.node)
?: run { setNode(entry.key.asName(), value.node) }
}
}
}
@ -149,12 +149,12 @@ fun <M : MutableMetaNode<M>> M.update(meta: Meta) {
fun <M : MutableMeta<M>> M.setIndexed(
name: Name,
items: Iterable<MetaItem<M>>,
queryFactory: (Int) -> String = { it.toString() }
indexFactory: MetaItem<M>.(index: Int) -> String = { it.toString() }
) {
val tokens = name.tokens.toMutableList()
val last = tokens.last()
items.forEachIndexed { index, meta ->
val indexedToken = NameToken(last.body, last.query + queryFactory(index))
val indexedToken = NameToken(last.body, last.index + meta.indexFactory(index))
tokens[tokens.lastIndex] = indexedToken
set(Name(tokens), meta)
}
@ -163,10 +163,26 @@ fun <M : MutableMeta<M>> M.setIndexed(
fun <M : MutableMetaNode<M>> M.setIndexed(
name: Name,
metas: Iterable<Meta>,
queryFactory: (Int) -> String = { it.toString() }
indexFactory: MetaItem<M>.(index: Int) -> String = { it.toString() }
) {
setIndexed(name, metas.map { MetaItem.NodeItem(wrap(name, it)) }, queryFactory)
setIndexed(name, metas.map { MetaItem.NodeItem(wrap(name, it)) }, indexFactory)
}
operator fun <M : MutableMetaNode<M>> M.set(name: Name, metas: Iterable<Meta>) = setIndexed(name, metas)
operator fun <M : MutableMetaNode<M>> M.set(name: String, metas: Iterable<Meta>) = setIndexed(name.toName(), metas)
/**
* Append the node with a same-name-sibling, automatically generating numerical index
*/
fun <M : MutableMetaNode<M>> M.append(name: Name, value: Any?) {
require(!name.isEmpty()) { "Name could not be empty for append operation" }
val newIndex = name.last()!!.index
if (newIndex.isNotEmpty()) {
set(name, value)
} else {
val index = (getAll(name).keys.mapNotNull { it.toIntOrNull() }.max() ?: -1) + 1
set(name.withIndex(index.toString()), value)
}
}
fun <M : MutableMetaNode<M>> M.append(name: String, value: Any?) = append(name.toName(), value)

26
dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Specification.kt → dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Specific.kt
View File

@ -3,16 +3,16 @@ package hep.dataforge.meta
/**
* Marker interface for specifications
*/
interface Specification : Configurable {
interface Specific : Configurable {
operator fun get(name: String): MetaItem<Config>? = config[name]
}
/**
* Allows to apply custom configuration in a type safe way to simple untyped configuration.
* By convention [Specification] companion should inherit this class
* By convention [Specific] companion should inherit this class
*
*/
interface SpecificationCompanion<T : Specification> {
interface Specification<T : Specific> {
/**
* Update given configuration using given type as a builder
*/
@ -22,41 +22,41 @@ interface SpecificationCompanion<T : Specification> {
fun build(action: T.() -> Unit) = update(Config(), action)
fun empty() = build { }
/**
* Wrap generic configuration producing instance of desired type
*/
fun wrap(config: Config): T
fun wrap(meta: Meta): T = wrap(meta.toConfig())
}
fun <T : Specification> specification(wrapper: (Config) -> T): SpecificationCompanion<T> =
object : SpecificationCompanion<T> {
fun <T : Specific> specification(wrapper: (Config) -> T): Specification<T> =
object : Specification<T> {
override fun wrap(config: Config): T = wrapper(config)
}
/**
* Apply specified configuration to configurable
*/
fun <T : Configurable, C : Specification, S : SpecificationCompanion<C>> T.configure(spec: S, action: C.() -> Unit) =
fun <T : Configurable, C : Specific, S : Specification<C>> T.configure(spec: S, action: C.() -> Unit) =
apply { spec.update(config, action) }
/**
* Update configuration using given specification
*/
fun <C : Specification, S : SpecificationCompanion<C>> Specification.update(spec: S, action: C.() -> Unit) =
fun <C : Specific, S : Specification<C>> Specific.update(spec: S, action: C.() -> Unit) =
apply { spec.update(config, action) }
/**
* Create a style based on given specification
*/
fun <C : Specification, S : SpecificationCompanion<C>> S.createStyle(action: C.() -> Unit): Meta =
fun <C : Specific, S : Specification<C>> S.createStyle(action: C.() -> Unit): Meta =
Config().also { update(it, action) }
fun <M : MutableMetaNode<M>, C : Specification> Specification.spec(
spec: SpecificationCompanion<C>,
fun <C : Specific> Specific.spec(
spec: Specification<C>,
key: String? = null
) =
ChildConfigDelegate(config, key) { spec.wrap(config) }
) = MutableMorphDelegate(config, key) { spec.wrap(it) }

2
dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Styled.kt
View File

@ -48,7 +48,7 @@ class Styled(val base: Meta, val style: Config = Config().empty()) : MutableMeta
}
}
fun Styled.configure(meta: Meta) = apply { style.update(style) }
fun Styled.configure(meta: Meta) = apply { style.update(meta) }
fun Meta.withStyle(style: Meta = EmptyMeta) = if (this is Styled) {
this.apply { this.configure(style) }

33
dataforge-meta/src/commonMain/kotlin/hep/dataforge/names/Name.kt
View File

@ -4,7 +4,7 @@ package hep.dataforge.names
/**
* The general interface for working with names.
* The name is a dot separated list of strings like `token1.token2.token3`.
* Each token could contain additional query in square brackets.
* Each token could contain additional index in square brackets.
*/
inline class Name constructor(val tokens: List<NameToken>) {
@ -43,24 +43,25 @@ inline class Name constructor(val tokens: List<NameToken>) {
/**
* A single name token. Body is not allowed to be empty.
* Following symbols are prohibited in name tokens: `{}.:\`.
* A name token could have appendix in square brackets called *query*
* A name token could have appendix in square brackets called *index*
*/
data class NameToken(val body: String, val query: String = "") {
data class NameToken(val body: String, val index: String = "") {
init {
if (body.isEmpty()) error("Syntax error: Name token body is empty")
}
override fun toString(): String = if (hasQuery()) {
"$body[$query]"
override fun toString(): String = if (hasIndex()) {
"$body[$index]"
} else {
body
}
fun hasQuery() = query.isNotEmpty()
fun hasIndex() = index.isNotEmpty()
}
fun String.toName(): Name {
if (isBlank()) return EmptyName
val tokens = sequence {
var bodyBuilder = StringBuilder()
var queryBuilder = StringBuilder()
@ -84,7 +85,7 @@ fun String.toName(): Name {
'[' -> bracketCount++
']' -> error("Syntax error: closing bracket ] not have not matching open bracket")
else -> {
if (queryBuilder.isNotEmpty()) error("Syntax error: only name end and name separator are allowed after query")
if (queryBuilder.isNotEmpty()) error("Syntax error: only name end and name separator are allowed after index")
bodyBuilder.append(it)
}
}
@ -101,8 +102,24 @@ operator fun Name.plus(other: Name): Name = Name(this.tokens + other.tokens)
operator fun Name.plus(other: String): Name = this + other.toName()
fun NameToken.toName() = Name(listOf(this))
fun Name.appendLeft(other: String): Name = NameToken(other) + this
fun NameToken.asName() = Name(listOf(this))
val EmptyName = Name(emptyList())
fun Name.isEmpty(): Boolean = this.length == 0
/**
* Set or replace last token index
*/
fun Name.withIndex(index: String): Name {
val tokens = ArrayList(tokens)
val last = NameToken(tokens.last().body, index)
tokens.removeAt(tokens.size - 1)
tokens.add(last)
return Name(tokens)
}
operator fun <T> Map<Name, T>.get(name: String) = get(name.toName())
operator fun <T> MutableMap<Name, T>.set(name: String, value: T) = set(name.toName(), value)

36
dataforge-meta/src/commonMain/kotlin/hep/dataforge/values/Value.kt
View File

@ -55,8 +55,15 @@ interface Value {
is Value -> value
true -> True
false -> False
is Number -> NumberValue(value)
is Number -> value.asValue()
is Iterable<*> -> ListValue(value.map { of(it) })
is DoubleArray -> value.asValue()
is IntArray -> value.asValue()
is FloatArray -> value.asValue()
is ShortArray -> value.asValue()
is LongArray -> value.asValue()
is ByteArray -> value.asValue()
is Array<*> -> ListValue(value.map { of(it) })
is Enum<*> -> EnumValue(value)
is CharSequence -> StringValue(value.toString())
else -> throw IllegalArgumentException("Unrecognized type of the object (${value::class}) converted to Value")
@ -171,6 +178,18 @@ class ListValue(override val list: List<Value>) : Value {
override val string: String get() = list.first().string
override fun toString(): String = value.toString()
override fun equals(other: Any?): Boolean {
if (this === other) return true
if (other !is Value) return false
return list == other.list
}
override fun hashCode(): Int {
return list.hashCode()
}
}
/**
@ -185,7 +204,20 @@ fun Boolean.asValue(): Value = if (this) True else False
fun String.asValue(): Value = StringValue(this)
fun Collection<Value>.asValue(): Value = ListValue(this.toList())
fun Iterable<Value>.asValue(): Value = ListValue(this.toList())
//TODO maybe optimized storage performance
fun DoubleArray.asValue(): Value = ListValue(map{NumberValue(it)})
fun IntArray.asValue(): Value = ListValue(map{NumberValue(it)})
fun LongArray.asValue(): Value = ListValue(map{NumberValue(it)})
fun ShortArray.asValue(): Value = ListValue(map{NumberValue(it)})
fun FloatArray.asValue(): Value = ListValue(map{NumberValue(it)})
fun ByteArray.asValue(): Value = ListValue(map{NumberValue(it)})
/**

14
dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/MetaBuilderTest.kt
View File

@ -23,4 +23,18 @@ class MetaBuilderTest {
assertEquals(true, meta["node.childNode.f"]?.boolean)
}
@Test
fun testSNS(){
val meta = buildMeta {
repeat(10){
"b.a[$it]" to it
}
}.seal()
assertEquals(10, meta.values().count())
val nodes = meta.getAll("b.a")
assertEquals(3, nodes["3"]?.int)
}
}

21
dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/MetaDelegateTest.kt
View File

@ -12,17 +12,30 @@ class MetaDelegateTest {
@Test
fun delegateTest() {
val testObject = object : Specification {
class InnerSpec(override val config: Config) : Specific {
var innerValue by string()
}
val innerSpec = specification(::InnerSpec)
val testObject = object : Specific {
override val config: Config = Config()
var myValue by config.string()
var safeValue by config.number(2.2)
var enumValue by config.enum(TestEnum.YES)
var myValue by string()
var safeValue by double(2.2)
var enumValue by enum(TestEnum.YES)
var inner by spec(innerSpec)
}
testObject.config["myValue"] = "theString"
testObject.enumValue = TestEnum.NO
testObject.inner = innerSpec.build { innerValue = "ddd"}
assertEquals("theString", testObject.myValue)
assertEquals(TestEnum.NO, testObject.enumValue)
assertEquals(2.2, testObject.safeValue)
assertEquals("ddd", testObject.inner?.innerValue)
}
}

29
dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/StyledTest.kt Normal file
View File

@ -0,0 +1,29 @@
package hep.dataforge.meta
import kotlin.test.Test
import kotlin.test.assertEquals
class StyledTest{
@Test
fun testSNS(){
val meta = buildMeta {
repeat(10){
"b.a[$it]" to {
"d" to it
}
}
}.seal().withStyle()
assertEquals(10, meta.values().count())
val bNode = meta["b"].node
val aNodes = bNode?.getAll("a")
val allNodes = meta.getAll("b.a")
assertEquals(3, aNodes?.get("3").node["d"].int)
assertEquals(3, allNodes["3"].node["d"].int)
}
}

63
dataforge-meta/src/jsMain/kotlin/hep/dataforge/meta/DynamicMeta.kt Normal file
View File

@ -0,0 +1,63 @@
package hep.dataforge.meta
import hep.dataforge.names.NameToken
import hep.dataforge.values.Null
import hep.dataforge.values.Value
//TODO add Meta wrapper for dynamic
/**
* Represent or copy this [Meta] to dynamic object to be passed to JS libraries
*/
fun Meta.toDynamic(): dynamic {
if(this is DynamicMeta) return this.obj
fun MetaItem<*>.toDynamic(): dynamic = when (this) {
is MetaItem.ValueItem -> this.value.value.asDynamic()
is MetaItem.NodeItem -> this.node.toDynamic()
}
val res = js("{}")
this.items.entries.groupBy { it.key.body }.forEach { (key, value) ->
val list = value.map { it.value }
res[key] = when (list.size) {
1 -> list.first().toDynamic()
else -> list.map { it.toDynamic() }
}
}
return res
}
class DynamicMeta(val obj: dynamic) : Meta {
private fun keys() = js("Object.keys(this.obj)") as Array<String>
private fun isArray(@Suppress("UNUSED_PARAMETER") obj: dynamic): Boolean =
js("Array.isArray(obj)") as Boolean
private fun asItem(obj: dynamic): MetaItem<DynamicMeta>? {
if (obj == null) return MetaItem.ValueItem(Null)
return when (jsTypeOf(obj)) {
"boolean" -> MetaItem.ValueItem(Value.of(obj as Boolean))
"number" -> MetaItem.ValueItem(Value.of(obj as Number))
"string" -> MetaItem.ValueItem(Value.of(obj as String))
"object" -> MetaItem.NodeItem(DynamicMeta(obj))
else -> null
}
}
override val items: Map<NameToken, MetaItem<DynamicMeta>>
get() = keys().flatMap<String, Pair<NameToken, MetaItem<DynamicMeta>>> { key ->
val value = obj[key] ?: return@flatMap emptyList()
if (isArray(value)) {
return@flatMap (value as Array<dynamic>)
.mapIndexedNotNull() { index, it ->
val item = asItem(it) ?: return@mapIndexedNotNull null
NameToken(key, index.toString()) to item
}
} else {
val item = asItem(value) ?: return@flatMap emptyList()
listOf(NameToken(key) to item)
}
}.associate { it }
}

24
dataforge-meta/src/jsTest/kotlin/hep/dataforge/meta/DynamicMetaTest.kt Normal file
View File

@ -0,0 +1,24 @@
package hep.dataforge.meta
import kotlin.test.Test
import kotlin.test.assertEquals
class DynamicMetaTest {
@Test
fun testDynamicMeta() {
val d = js("{}")
d.a = 22
d.array = arrayOf(1,2,3)
d.b = "myString"
d.ob = js("{}")
d.ob.childNode = 18
d.ob.booleanNode = true
val meta = DynamicMeta(d)
assertEquals(true, meta["ob.booleanNode"].boolean)
assertEquals(2,meta["array[1]"].int)
}
}

16
dataforge-output/build.gradle.kts Normal file
View File

@ -0,0 +1,16 @@
plugins {
`npm-multiplatform`
}
kotlin {
jvm()
js()
sourceSets {
val commonMain by getting{
dependencies {
api(project(":dataforge-context"))
api(project(":dataforge-io"))
}
}
}
}

28
dataforge-output/dataforge-output-html/build.gradle.kts Normal file
View File

@ -0,0 +1,28 @@
plugins {
`npm-multiplatform`
}
val htmlVersion by rootProject.extra("0.6.12")
kotlin {
jvm()
js()
sourceSets {
val commonMain by getting {
dependencies {
api(project(":dataforge-output"))
api("org.jetbrains.kotlinx:kotlinx-html-common:$htmlVersion")
}
}
val jsMain by getting {
dependencies {
api("org.jetbrains.kotlinx:kotlinx-html-js:$htmlVersion")
}
}
val jvmMain by getting{
dependencies {
api("org.jetbrains.kotlinx:kotlinx-html-jvm:$htmlVersion")
}
}
}
}

77
dataforge-output/dataforge-output-html/src/jvmMain/kotlin/hep/dataforge/output/html/HtmlOutput.kt Normal file
View File

@ -0,0 +1,77 @@
package hep.dataforge.output.html
import hep.dataforge.context.Context
import hep.dataforge.meta.Meta
import hep.dataforge.output.Output
import hep.dataforge.output.TextRenderer
import hep.dataforge.output.html.HtmlBuilder.Companion.HTML_CONVERTER_TYPE
import hep.dataforge.provider.Type
import hep.dataforge.provider.top
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.launch
import kotlinx.html.TagConsumer
import kotlinx.html.p
import kotlin.reflect.KClass
class HtmlOutput<T : Any>(override val context: Context, private val consumer: TagConsumer<*>) : Output<T> {
private val cache = HashMap<KClass<*>, HtmlBuilder<*>>()
/**
* Find the first [TextRenderer] matching the given object type.
*/
override fun render(obj: T, meta: Meta) {
val builder: HtmlBuilder<*> = if (obj is CharSequence) {
DefaultHtmlBuilder
} else {
val value = cache[obj::class]
if (value == null) {
val answer = context.top<HtmlBuilder<*>>().values
.filter { it.type.isInstance(obj) }.firstOrNull()
if (answer != null) {
cache[obj::class] = answer
answer
} else {
DefaultHtmlBuilder
}
} else {
value
}
}
context.launch(Dispatchers.Output) {
(builder as HtmlBuilder<T>).run { consumer.render(obj) }
}
}
}
/**
* A text or binary renderer based on [kotlinx.io.core.Output]
*/
@Type(HTML_CONVERTER_TYPE)
interface HtmlBuilder<T : Any> {
/**
* The priority of this renderer compared to other renderers
*/
val priority: Int
/**
* The type of the content served by this renderer
*/
val type: KClass<T>
suspend fun TagConsumer<*>.render(obj: T)
companion object {
const val HTML_CONVERTER_TYPE = "dataforge.htmlBuilder"
}
}
object DefaultHtmlBuilder : HtmlBuilder<Any> {
override val priority: Int = Int.MAX_VALUE
override val type: KClass<Any> = Any::class
override suspend fun TagConsumer<*>.render(obj: Any) {
p { +obj.toString() }
}
}

2
dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Output.kt → dataforge-output/src/commonMain/kotlin/hep/dataforge/output/Output.kt
View File

@ -1,4 +1,4 @@
package hep.dataforge.io
package hep.dataforge.output
import hep.dataforge.context.ContextAware
import hep.dataforge.meta.EmptyMeta

77
dataforge-output/src/commonMain/kotlin/hep/dataforge/output/OutputManager.kt Normal file
View File

@ -0,0 +1,77 @@
package hep.dataforge.output
import hep.dataforge.context.*
import hep.dataforge.context.PluginTag.Companion.DATAFORGE_GROUP
import hep.dataforge.meta.EmptyMeta
import hep.dataforge.meta.Meta
import hep.dataforge.names.EmptyName
import hep.dataforge.names.Name
import kotlinx.coroutines.CoroutineDispatcher
import kotlinx.coroutines.Dispatchers
import kotlin.reflect.KClass
/**
* A manager for outputs
*/
interface OutputManager : Plugin {
/**
* Get an output specialized for given type, name and stage.
* @param stage represents the node or directory for the output. Empty means root node.
* @param name represents the name inside the node.
* @param meta configuration for [Output] (not for rendered object)
*/
operator fun <T : Any> get(
type: KClass<out T>,
name: Name,
stage: Name = EmptyName,
meta: Meta = EmptyMeta
): Output<T>
}
/**
* Get an output manager for a context
*/
val Context.output: OutputManager get() = plugins.get() ?: ConsoleOutputManager()
/**
* Get an output with given [name], [stage] and reified content type
*/
inline operator fun <reified T : Any> OutputManager.get(
name: Name,
stage: Name = EmptyName,
meta: Meta = EmptyMeta
): Output<T> {
return get(T::class, name, stage, meta)
}
/**
* Directly render an object using the most suitable renderer
*/
fun OutputManager.render(obj: Any, name: Name, stage: Name = EmptyName, meta: Meta = EmptyMeta) =
get(obj::class, name, stage).render(obj, meta)
/**
* System console output.
* The [ConsoleOutput] is used when no other [OutputManager] is provided.
*/
expect val ConsoleOutput: Output<Any>
class ConsoleOutputManager : AbstractPlugin(), OutputManager {
override val tag: PluginTag get() = ConsoleOutputManager.tag
override fun <T : Any> get(type: KClass<out T>, name: Name, stage: Name, meta: Meta): Output<T> = ConsoleOutput
companion object : PluginFactory<ConsoleOutputManager> {
override val tag = PluginTag("output.console", group = DATAFORGE_GROUP)
override val type = ConsoleOutputManager::class
override fun invoke(meta:Meta) = ConsoleOutputManager()
}
}
/**
* A dispatcher for output tasks.
*/
expect val Dispatchers.Output: CoroutineDispatcher

16
dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TextOutput.kt → dataforge-output/src/commonMain/kotlin/hep/dataforge/output/TextOutput.kt
View File

@ -1,10 +1,11 @@
package hep.dataforge.io
package hep.dataforge.output
import hep.dataforge.context.Context
import hep.dataforge.io.TextRenderer.Companion.TEXT_RENDERER_TYPE
import hep.dataforge.meta.Meta
import hep.dataforge.output.TextRenderer.Companion.TEXT_RENDERER_TYPE
import hep.dataforge.provider.Type
import hep.dataforge.provider.provideAll
import hep.dataforge.provider.top
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.launch
import kotlin.reflect.KClass
@ -20,8 +21,8 @@ class TextOutput(override val context: Context, private val output: kotlinx.io.c
} else {
val value = cache[obj::class]
if (value == null) {
val answer = context.provideAll(TEXT_RENDERER_TYPE).filterIsInstance<TextRenderer>()
.filter { it.type.isInstance(obj) }.firstOrNull()
val answer =
context.top<TextRenderer>(TEXT_RENDERER_TYPE).values.firstOrNull { it.type.isInstance(obj) }
if (answer != null) {
cache[obj::class] = answer
answer
@ -32,12 +33,15 @@ class TextOutput(override val context: Context, private val output: kotlinx.io.c
value
}
}
context.launch(OutputDispatcher) {
context.launch(Dispatchers.Output) {
renderer.run { output.render(obj) }
}
}
}
/**
* A text or binary renderer based on [kotlinx.io.core.Output]
*/
@Type(TEXT_RENDERER_TYPE)
interface TextRenderer {
/**

4
dataforge-io/src/jsMain/kotlin/hep/dataforge/io/ConsoleOutput.kt → dataforge-output/src/jsMain/kotlin/hep/dataforge/output/ConsoleOutput.kt
View File

@ -1,4 +1,4 @@
package hep.dataforge.io
package hep.dataforge.output
import hep.dataforge.context.Context
import hep.dataforge.context.Global
@ -19,4 +19,4 @@ actual val ConsoleOutput: Output<Any> = object : Output<Any> {
}
actual val OutputDispatcher: CoroutineDispatcher = Dispatchers.Default
actual val Dispatchers.Output: CoroutineDispatcher get() = Dispatchers.Default

4
dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/ConsoleOutput.kt → dataforge-output/src/jvmMain/kotlin/hep/dataforge/output/ConsoleOutput.kt
View File

@ -1,4 +1,4 @@
package hep.dataforge.io
package hep.dataforge.output
import hep.dataforge.context.Global
import kotlinx.coroutines.Dispatchers
@ -10,4 +10,4 @@ import kotlinx.io.streams.asOutput
*/
actual val ConsoleOutput: Output<Any> = TextOutput(Global, System.out.asOutput())
actual val OutputDispatcher = Dispatchers.IO
actual val Dispatchers.Output get() = Dispatchers.IO

2
dataforge-scripting/build.gradle.kts
View File

@ -1,5 +1,5 @@
plugins {
kotlin("multiplatform")
`npm-multiplatform`
}
kotlin {

8
dataforge-scripting/src/jvmMain/kotlin/hep/dataforge/scripting/Builders.kt
View File

@ -2,6 +2,7 @@ package hep.dataforge.scripting
import hep.dataforge.context.Context
import hep.dataforge.context.Global
import hep.dataforge.workspace.SimpleWorkspaceBuilder
import hep.dataforge.workspace.Workspace
import hep.dataforge.workspace.WorkspaceBuilder
import java.io.File
@ -14,11 +15,12 @@ import kotlin.script.experimental.jvmhost.BasicJvmScriptingHost
object Builders {
fun buildWorkspace(source: SourceCode, context: Context = Global): Workspace {
val builder = WorkspaceBuilder(context)
val builder = SimpleWorkspaceBuilder(context)
val workspaceScriptConfiguration = ScriptCompilationConfiguration {
baseClass(Any::class)
implicitReceivers(WorkspaceBuilder::class)
defaultImports("hep.dataforge.workspace.*")
jvm {
dependenciesFromCurrentContext(wholeClasspath = true)
}
@ -31,8 +33,10 @@ object Builders {
BasicJvmScriptingHost().eval(source, workspaceScriptConfiguration, evaluationConfiguration).onFailure {
it.reports.forEach { scriptDiagnostic ->
when (scriptDiagnostic.severity) {
ScriptDiagnostic.Severity.FATAL, ScriptDiagnostic.Severity.ERROR ->
ScriptDiagnostic.Severity.FATAL, ScriptDiagnostic.Severity.ERROR -> {
context.logger.error(scriptDiagnostic.exception) { scriptDiagnostic.toString() }
error(scriptDiagnostic.toString())
}
ScriptDiagnostic.Severity.WARNING -> context.logger.warn { scriptDiagnostic.toString() }
ScriptDiagnostic.Severity.INFO -> context.logger.info { scriptDiagnostic.toString() }
ScriptDiagnostic.Severity.DEBUG -> context.logger.debug { scriptDiagnostic.toString() }

19
dataforge-scripting/src/jvmTest/kotlin/hep/dataforge/scripting/BuildersKtTest.kt
View File

@ -1,20 +1,33 @@
package hep.dataforge.scripting
import hep.dataforge.context.Global
import hep.dataforge.meta.get
import hep.dataforge.meta.int
import hep.dataforge.workspace.*
import org.junit.Test
import kotlin.test.assertEquals
class BuildersKtTest {
@Test
fun checkBuilder(){
val workspace = SimpleWorkspaceBuilder(Global).apply {
println("I am working")
context("test")
target("testTarget"){
"a" to 12
}
}
}
@Test
fun testWorkspaceBuilder() {
val script = """
println("I am working")
context{
name = "test"
}
context("test")
target("testTarget"){
"a" to 12

2
dataforge-workspace/build.gradle.kts
View File

@ -1,5 +1,5 @@
plugins {
kotlin("multiplatform")
`npm-multiplatform`
}
kotlin {

58
dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/Dependency.kt
View File

@ -2,13 +2,13 @@ package hep.dataforge.workspace
import hep.dataforge.data.DataFilter
import hep.dataforge.data.DataNode
import hep.dataforge.data.DataTreeBuilder
import hep.dataforge.data.filter
import hep.dataforge.meta.Meta
import hep.dataforge.meta.MetaRepr
import hep.dataforge.meta.buildMeta
import hep.dataforge.names.EmptyName
import hep.dataforge.names.Name
import hep.dataforge.names.get
import hep.dataforge.names.isEmpty
/**
@ -18,31 +18,51 @@ sealed class Dependency : MetaRepr {
abstract fun apply(workspace: Workspace): DataNode<Any>
}
class DataDependency(val filter: DataFilter) : Dependency() {
override fun apply(workspace: Workspace): DataNode<Any> =
workspace.data.filter(filter)
override fun toMeta(): Meta = filter.config
companion object {
val all: DataDependency = DataDependency(DataFilter.build { })
}
}
class TaskModelDependency(val name: String, val meta: Meta, val placement: Name = EmptyName) : Dependency() {
class DataDependency(val filter: DataFilter, val placement: Name = EmptyName) : Dependency() {
override fun apply(workspace: Workspace): DataNode<Any> {
val task = workspace.tasks[name] ?: error("Task with name ${name} is not found in the workspace")
if (task.isTerminal) TODO("Support terminal task")
val result = with(workspace) { task(meta) }
val result = workspace.data.filter(filter)
return if (placement.isEmpty()) {
result
} else {
DataTreeBuilder<Any>().apply { this[placement] = result }.build()
DataNode.build(Any::class) { this[placement] = result }
}
}
override fun toMeta(): Meta = buildMeta {
"name" to name
"meta" to meta
"data" to filter.config
"to" to placement
}
}
class AllDataDependency(val placement: Name = EmptyName) : Dependency() {
override fun apply(workspace: Workspace): DataNode<Any> = if (placement.isEmpty()) {
workspace.data
} else {
DataNode.build(Any::class) { this[placement] = workspace.data }
}
override fun toMeta() = buildMeta {
"data" to "*"
"to" to placement
}
}
class TaskModelDependency(val name: String, val meta: Meta, val placement: Name = EmptyName) : Dependency() {
override fun apply(workspace: Workspace): DataNode<Any> {
val task = workspace.tasks[name] ?: error("Task with name $name is not found in the workspace")
if (task.isTerminal) TODO("Support terminal task")
val result = workspace.run(task, meta)
return if (placement.isEmpty()) {
result
} else {
DataNode.build(Any::class) { this[placement] = result }
}
}
override fun toMeta(): Meta = buildMeta {
"task" to name
"meta" to meta
"to" to placement
}
}

61
dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/GenericTask.kt Normal file
View File

@ -0,0 +1,61 @@
package hep.dataforge.workspace
import hep.dataforge.data.*
import hep.dataforge.descriptors.NodeDescriptor
import hep.dataforge.meta.Meta
import hep.dataforge.meta.get
import hep.dataforge.meta.node
import kotlin.reflect.KClass
//data class TaskEnv(val workspace: Workspace, val model: TaskModel)
class GenericTask<R : Any>(
override val name: String,
override val type: KClass<out R>,
override val descriptor: NodeDescriptor,
private val modelTransform: TaskModelBuilder.(Meta) -> Unit,
private val dataTransform: Workspace.() -> TaskModel.(DataNode<Any>) -> DataNode<R>
) : Task<R> {
private fun gather(workspace: Workspace, model: TaskModel): DataNode<Any> {
return DataNode.build(Any::class) {
model.dependencies.forEach { dep ->
update(dep.apply(workspace))
}
}
}
override fun run(workspace: Workspace, model: TaskModel): DataNode<R> {
//validate model
validate(model)
// gather data
val input = gather(workspace, model)
//execute
workspace.context.logger.info{"Starting task '$name' on ${model.target} with meta: \n${model.meta}"}
val output = dataTransform(workspace).invoke(model, input)
//handle result
//output.handle(model.context.dispatcher) { this.handle(it) }
return output
}
/**
* Build new TaskModel and apply specific model transformation for this
* task. By default model uses the meta node with the same node as the name of the task.
*
* @param workspace
* @param taskConfig
* @return
*/
override fun build(workspace: Workspace, taskConfig: Meta): TaskModel {
val taskMeta = taskConfig[name]?.node ?: taskConfig
val builder = TaskModelBuilder(name, taskMeta)
modelTransform.invoke(builder, taskMeta)
return builder.build()
}
//TODO add validation
}

29
dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/SimpleWorkspace.kt Normal file
View File

@ -0,0 +1,29 @@
package hep.dataforge.workspace
import hep.dataforge.context.Context
import hep.dataforge.context.Global
import hep.dataforge.data.DataNode
import hep.dataforge.meta.Meta
import hep.dataforge.names.Name
import hep.dataforge.names.toName
import hep.dataforge.provider.top
/**
* A simple workspace without caching
*/
class SimpleWorkspace(
override val context: Context,
override val data: DataNode<Any>,
override val targets: Map<String, Meta>,
tasks: Collection<Task<Any>>
) : Workspace {
override val tasks: Map<Name, Task<*>> by lazy {
context.top<Task<*>>(Task.TYPE) + tasks.associate { it.name.toName() to it }
}
companion object {
fun build(parent: Context = Global, block: SimpleWorkspaceBuilder.() -> Unit): SimpleWorkspace =
SimpleWorkspaceBuilder(parent).apply(block).build()
}
}

8
dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/Task.kt
View File

@ -2,13 +2,14 @@ package hep.dataforge.workspace
import hep.dataforge.context.Named
import hep.dataforge.data.DataNode
import hep.dataforge.descriptors.Described
import hep.dataforge.meta.Meta
import hep.dataforge.provider.Type
import hep.dataforge.workspace.Task.Companion.TYPE
import kotlin.reflect.KClass
@Type(TYPE)
interface Task<out R : Any> : Named {
interface Task<out R : Any> : Named, Described {
/**
* Terminal task is the one that could not build model lazily
*/
@ -41,10 +42,11 @@ interface Task<out R : Any> : Named {
* Run given task model. Type check expected to be performed before actual
* calculation.
*
* @param model
* @param workspace - a workspace to run task model in
* @param model - a model to be executed
* @return
*/
fun run(model: TaskModel): DataNode<R>
fun run(workspace: Workspace, model: TaskModel): DataNode<R>
companion object {
const val TYPE = "task"

30
dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskModel.kt
View File

@ -12,6 +12,7 @@ import hep.dataforge.meta.*
import hep.dataforge.names.EmptyName
import hep.dataforge.names.Name
import hep.dataforge.names.toName
import hep.dataforge.workspace.TaskModel.Companion.MODEL_TARGET_KEY
/**
@ -39,23 +40,31 @@ data class TaskModel(
//TODO ensure all dependencies are listed
}
}
companion object {
const val MODEL_TARGET_KEY = "@target"
}
}
/**
* Build input for the task
*/
fun TaskModel.buildInput(workspace: Workspace): DataTree<Any> {
return DataTreeBuilder<Any>().apply {
dependencies.asSequence().flatMap { it.apply(workspace).dataSequence() }.forEach { (name, data) ->
return DataTreeBuilder(Any::class).apply {
dependencies.asSequence().flatMap { it.apply(workspace).data() }.forEach { (name, data) ->
//TODO add concise error on replacement
this[name] = data
}
}.build()
}
@DslMarker
annotation class TaskBuildScope
/**
* A builder for [TaskModel]
*/
@TaskBuildScope
class TaskModelBuilder(val name: String, meta: Meta = EmptyMeta) {
/**
* Meta for current task. By default uses the whole input meta
@ -63,13 +72,21 @@ class TaskModelBuilder(val name: String, meta: Meta = EmptyMeta) {
var meta: MetaBuilder = meta.builder()
val dependencies = HashSet<Dependency>()
var target: String by this.meta.string(key = MODEL_TARGET_KEY, default = "")
/**
* Add dependency for
*/
fun dependsOn(name: String, meta: Meta, placement: Name = EmptyName) {
fun dependsOn(name: String, meta: Meta = this.meta, placement: Name = EmptyName) {
dependencies.add(TaskModelDependency(name, meta, placement))
}
fun dependsOn(task: Task<*>, meta: Meta = this.meta, placement: Name = EmptyName) =
dependsOn(task.name, meta, placement)
fun dependsOn(task: Task<*>, placement: Name = EmptyName, metaBuilder: MetaBuilder.() -> Unit) =
dependsOn(task.name, buildMeta(metaBuilder), placement)
/**
* Add custom data dependency
*/
@ -89,9 +106,12 @@ class TaskModelBuilder(val name: String, meta: Meta = EmptyMeta) {
/**
* Add all data as root node
*/
fun allData() {
dependencies.add(DataDependency.all)
fun allData(to: Name = EmptyName) {
dependencies.add(AllDataDependency(to))
}
fun build(): TaskModel = TaskModel(name, meta.seal(), dependencies)
}
val TaskModel.target get() = meta[MODEL_TARGET_KEY]?.string ?: ""

66
dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/Workspace.kt
View File

@ -1,11 +1,11 @@
package hep.dataforge.workspace
import hep.dataforge.context.Context
import hep.dataforge.context.ContextAware
import hep.dataforge.context.members
import hep.dataforge.data.Data
import hep.dataforge.data.DataNode
import hep.dataforge.meta.Meta
import hep.dataforge.meta.MetaBuilder
import hep.dataforge.meta.buildMeta
import hep.dataforge.names.Name
import hep.dataforge.names.toName
import hep.dataforge.provider.Provider
@ -27,62 +27,68 @@ interface Workspace : ContextAware, Provider {
/**
* All tasks associated with the workspace
*/
val tasks: Map<String, Task<*>>
val tasks: Map<Name, Task<*>>
override fun provideTop(target: String, name: Name): Any? {
return when (target) {
"target", Meta.TYPE -> targets[name.toString()]
Task.TYPE -> tasks[name.toString()]
Task.TYPE -> tasks[name]
Data.TYPE -> data[name]
DataNode.TYPE -> data.getNode(name)
else -> null
}
}
override fun listTop(target: String): Sequence<Name> {
override fun listNames(target: String): Sequence<Name> {
return when (target) {
"target", Meta.TYPE -> targets.keys.asSequence().map { it.toName() }
Task.TYPE -> tasks.keys.asSequence().map { it.toName() }
Data.TYPE -> data.dataSequence().map { it.first }
DataNode.TYPE -> data.nodeSequence().map { it.first }
Task.TYPE -> tasks.keys.asSequence().map { it }
Data.TYPE -> data.data().map { it.first }
DataNode.TYPE -> data.nodes().map { it.first }
else -> emptySequence()
}
}
operator fun <R : Any> Task<R>.invoke(config: Meta): DataNode<R> {
/**
* Invoke a task in the workspace utilizing caching if possible
*/
fun <R : Any> run(task: Task<R>, config: Meta): DataNode<R> {
context.activate(this)
try {
val model = build(this@Workspace, config)
validate(model)
return run(model)
val model = task.build(this, config)
task.validate(model)
return task.run(this, model)
} finally {
context.deactivate(this)
}
}
/**
* Invoke a task in the workspace utilizing caching if possible
*/
operator fun <R : Any> Task<R>.invoke(targetName: String): DataNode<R> {
val target = targets[targetName] ?: error("A target with name $targetName not found in ${this@Workspace}")
return invoke(target)
}
// /**
// * Invoke a task in the workspace utilizing caching if possible
// */
// operator fun <R : Any> Task<R>.invoke(targetName: String): DataNode<R> {
// val target = targets[targetName] ?: error("A target with name $targetName not found in ${this@Workspace}")
// context.logger.info { "Running ${this.name} on $target" }
// return invoke(target)
// }
companion object {
const val TYPE = "workspace"
}
}
class SimpleWorkspace(
override val context: Context,
override val data: DataNode<Any>,
override val targets: Map<String, Meta>,
tasks: Collection<Task<Any>>
) : Workspace {
override val tasks: Map<String, Task<*>> by lazy {
(context.members<Task<*>>(Task.TYPE) + tasks).associate { it.name to it }
}
fun Workspace.run(task: Task<*>, target: String): DataNode<Any> {
val meta = targets[target] ?: error("A target with name $target not found in ${this}")
return run(task, meta)
}
fun Workspace.run(task: String, target: String) =
tasks[task.toName()]?.let { run(it, target) } ?: error("Task with name $task not found")
fun Workspace.run(task: String, meta: Meta) =
tasks[task.toName()]?.let { run(it, meta) } ?: error("Task with name $task not found")
fun Workspace.run(task: String, block: MetaBuilder.() -> Unit = {}) =
run(task, buildMeta(block))

114
dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/WorkspaceBuilder.kt
View File

@ -2,39 +2,91 @@ package hep.dataforge.workspace
import hep.dataforge.context.Context
import hep.dataforge.context.ContextBuilder
import hep.dataforge.data.Data
import hep.dataforge.data.DataNode
import hep.dataforge.data.DataTreeBuilder
import hep.dataforge.meta.Meta
import hep.dataforge.meta.MetaBuilder
import hep.dataforge.meta.buildMeta
import hep.dataforge.meta.*
import hep.dataforge.names.Name
import hep.dataforge.names.toName
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.GlobalScope
@TaskBuildScope
interface WorkspaceBuilder {
val parentContext: Context
var context: Context
var data: DataTreeBuilder<Any>
var tasks: MutableSet<Task<Any>>
var targets: MutableMap<String, Meta>
fun build(): Workspace
}
/**
* A builder for a workspace
* Set the context for future workspcace
*/
class WorkspaceBuilder(var context: Context) {
val data = DataTreeBuilder<Any>()
val targets = HashMap<String, Meta>()
val tasks = HashSet<Task<Any>>()
fun context(action: ContextBuilder.() -> Unit) {
this.context = ContextBuilder().apply(action).build()
}
fun data(action: DataTreeBuilder<Any>.() -> Unit) = data.apply(action)
fun target(name: String, meta: Meta) {
targets[name] = meta
}
fun target(name: String, action: MetaBuilder.() -> Unit) = target(name, buildMeta(action))
fun task(task: Task<*>) {
tasks.add(task)
}
fun build(): Workspace = SimpleWorkspace(
context,
data.build(),
targets,
tasks
)
fun WorkspaceBuilder.context(name: String, block: ContextBuilder.() -> Unit = {}) {
context = ContextBuilder(name, parentContext).apply(block).build()
}
fun WorkspaceBuilder.data(name: Name, data: Data<Any>) {
this.data[name] = data
}
fun WorkspaceBuilder.data(name: String, data: Data<Any>) = data(name.toName(), data)
fun WorkspaceBuilder.static(name: Name, data: Any, scope: CoroutineScope = GlobalScope, meta: Meta = EmptyMeta) =
data(name, Data.static(scope, data, meta))
fun WorkspaceBuilder.static(name: Name, data: Any, scope: CoroutineScope = GlobalScope, block: MetaBuilder.() -> Unit = {}) =
data(name, Data.static(scope, data, buildMeta(block)))
fun WorkspaceBuilder.static(name: String, data: Any, scope: CoroutineScope = GlobalScope, block: MetaBuilder.() -> Unit = {}) =
data(name, Data.static(scope, data, buildMeta(block)))
fun WorkspaceBuilder.data(name: Name, node: DataNode<Any>) {
this.data[name] = node
}
fun WorkspaceBuilder.data(name: String, node: DataNode<Any>) = data(name.toName(), node)
fun WorkspaceBuilder.data(name: Name, block: DataTreeBuilder<Any>.() -> Unit) {
this.data[name] = DataNode.build(Any::class, block)
}
fun WorkspaceBuilder.data(name: String, block: DataTreeBuilder<Any>.() -> Unit) = data(name.toName(), block)
fun WorkspaceBuilder.target(name: String, block: MetaBuilder.() -> Unit) {
targets[name] = buildMeta(block).seal()
}
/**
* Use existing target as a base updating it with the block
*/
fun WorkspaceBuilder.target(name: String, base: String, block: MetaBuilder.() -> Unit) {
val parentTarget = targets[base] ?: error("Base target with name $base not found")
targets[name] = parentTarget.builder()
.apply { "@baseTarget" to base }
.apply(block)
.seal()
}
fun WorkspaceBuilder.task(task: Task<Any>) {
this.tasks.add(task)
}
/**
* A builder for a simple workspace
*/
class SimpleWorkspaceBuilder(override val parentContext: Context) : WorkspaceBuilder {
override var context: Context = parentContext
override var data = DataTreeBuilder(Any::class)
override var tasks: MutableSet<Task<Any>> = HashSet()
override var targets: MutableMap<String, Meta> = HashMap()
override fun build(): SimpleWorkspace {
return SimpleWorkspace(context, data.build(), targets, tasks)
}
}

218
dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/TaskBuilder.kt Normal file
View File

@ -0,0 +1,218 @@
package hep.dataforge.workspace
import hep.dataforge.context.Context
import hep.dataforge.data.*
import hep.dataforge.descriptors.NodeDescriptor
import hep.dataforge.meta.Meta
import hep.dataforge.meta.get
import hep.dataforge.meta.string
import hep.dataforge.names.Name
import hep.dataforge.names.toName
import kotlin.reflect.KClass
@TaskBuildScope
class TaskBuilder(val name: String) {
private var modelTransform: TaskModelBuilder.(Meta) -> Unit = { data("*") }
var descriptor: NodeDescriptor? = null
/**
* TODO will look better as extension class
*/
private class DataTransformation(
val from: String = "",
val to: String = "",
val transform: (Context, TaskModel, DataNode<Any>) -> DataNode<Any>
) {
operator fun invoke(workspace: Workspace, model: TaskModel, node: DataNode<Any>): DataNode<Any>? {
val localData = if (from.isEmpty()) {
node
} else {
node.getNode(from.toName()) ?: return null
}
return transform(workspace.context, model, localData)
}
}
private val dataTransforms: MutableList<DataTransformation> = ArrayList();
fun model(modelTransform: TaskModelBuilder.(Meta) -> Unit) {
this.modelTransform = modelTransform
}
fun <T : Any> transform(
inputType: KClass<T>,
from: String = "",
to: String = "",
block: TaskModel.(Context, DataNode<T>) -> DataNode<Any>
) {
dataTransforms += DataTransformation(from, to) { context, model, data ->
block(model, context, data.cast(inputType))
}
}
inline fun <reified T : Any> transform(
from: String = "",
to: String = "",
noinline block: TaskModel.(Context, DataNode<T>) -> DataNode<Any>
) {
transform(T::class, from, to, block)
}
/**
* Perform given action on data elements in `from` node in input and put the result to `to` node
*/
inline fun <reified T : Any, reified R : Any> action(
from: String = "",
to: String = "",
crossinline block: Context.() -> Action<T, R>
) {
transform(from, to) { context, data: DataNode<T> ->
block(context).invoke(data, meta)
}
}
class TaskEnv(val name: Name, val meta: Meta, val context: Context)
/**
* A customized pipe action with ability to change meta and name
*/
inline fun <reified T : Any, reified R : Any> customPipe(
from: String = "",
to: String = "",
crossinline block: PipeBuilder<T, R>.(Context) -> Unit
) {
action(from, to) {
val context = this
PipeAction(
inputType = T::class,
outputType = R::class
) { block(context) }
}
}
/**
* A simple pipe action without changing meta or name
*/
inline fun <reified T : Any, reified R : Any> pipe(
from: String = "",
to: String = "",
crossinline block: suspend TaskEnv.(T) -> R
) {
action(from, to) {
val context = this
PipeAction(
inputType = T::class,
outputType = R::class
) {
//TODO automatically append task meta
result = { data ->
TaskEnv(name, meta, context).block(data)
}
}
}
}
/**
* Join elements in gathered data by multiple groups
*/
inline fun <reified T : Any, reified R : Any> joinByGroup(
from: String = "",
to: String = "",
crossinline block: JoinGroupBuilder<T, R>.(Context) -> Unit
) {
action(from, to) {
JoinAction(
inputType = T::class,
outputType = R::class
) { block(this@action) }
}
}
/**
* Join all elemlents in gathered data matching input type
*/
inline fun <reified T : Any, reified R : Any> join(
from: String = "",
to: String = "",
crossinline block: suspend TaskEnv.(Map<Name, T>) -> R
) {
action(from, to) {
val context = this
JoinAction(
inputType = T::class,
outputType = R::class,
action = {
result(
actionMeta[TaskModel.MODEL_TARGET_KEY]?.string ?: "@anonymous"
) { data ->
TaskEnv(name, meta, context).block(data)
}
}
)
}
}
/**
* Split each element in gathered data into fixed number of fragments
*/
inline fun <reified T : Any, reified R : Any> split(
from: String = "",
to: String = "",
crossinline block: SplitBuilder<T, R>.(Context) -> Unit
) {
action(from, to) {
SplitAction(
inputType = T::class,
outputType = R::class
) { block(this@action) }
}
}
/**
* Use DSL to create a descriptor for this task
*/
fun description(transform: NodeDescriptor.() -> Unit) {
this.descriptor = NodeDescriptor.build(transform)
}
internal fun build(): GenericTask<Any> =
GenericTask(
name,
Any::class,
descriptor ?: NodeDescriptor.empty(),
modelTransform
) {
val workspace = this
{ data ->
val model = this
if (dataTransforms.isEmpty()) {
//return data node as is
logger.warn("No transformation present, returning input data")
data
} else {
val builder = DataTreeBuilder(Any::class)
dataTransforms.forEach { transformation ->
val res = transformation(workspace, model, data)
if (res != null) {
if (transformation.to.isEmpty()) {
builder.update(res)
} else {
builder[transformation.to.toName()] = res
}
}
}
builder.build()
}
}
}
}
fun task(name: String, builder: TaskBuilder.() -> Unit): GenericTask<Any> {
return TaskBuilder(name).apply(builder).build()
}
fun WorkspaceBuilder.task(name: String, builder: TaskBuilder.() -> Unit) {
task(TaskBuilder(name).apply(builder).build())
}
//TODO add delegates to build gradle-like tasks

73
dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/SimpleWorkspaceTest.kt Normal file
View File

@ -0,0 +1,73 @@
package hep.dataforge.workspace
import hep.dataforge.data.first
import hep.dataforge.data.get
import org.junit.Test
import kotlin.test.assertEquals
class SimpleWorkspaceTest {
val workspace = SimpleWorkspace.build {
repeat(100) {
static("myData[$it]", it)
}
task("square") {
model {
allData()
}
pipe<Int, Int> { data ->
context.logger.info { "Starting square on $data" }
data * data
}
}
task("sum") {
model {
dependsOn("square")
}
join<Int, Int> { data ->
context.logger.info { "Starting sum" }
data.values.sum()
}
}
task("average") {
model {
allData()
}
joinByGroup<Int, Double> { context ->
group("even", filter = { name, data -> name.toString().toInt() % 2 == 0 }) {
result { data ->
context.logger.info { "Starting even" }
data.values.average()
}
}
group("odd", filter = { name, data -> name.toString().toInt() % 2 == 1 }) {
result { data ->
context.logger.info { "Starting odd" }
data.values.average()
}
}
}
}
task("delta"){
model{
dependsOn("average")
}
join<Double,Double> {data->
data["even"]!! - data["odd"]!!
}
}
}
@Test
fun testWorkspace() {
val node = workspace.run("sum")
val res = node.first()
assertEquals(328350, res.get())
}
}

7
settings.gradle.kts
View File

@ -18,13 +18,14 @@ pluginManagement {
enableFeaturePreview("GRADLE_METADATA")
rootProject.name = "dataforge-core"
//rootProject.name = "dataforge-core"
include(
":dataforge-meta",
":dataforge-meta-io",
":dataforge-io",
":dataforge-context",
":dataforge-data",
":dataforge-io",
":dataforge-output",
":dataforge-output:dataforge-output-html",
":dataforge-workspace",
":dataforge-scripting"
)